diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go new file mode 100644 index 000000000..44ee7a556 --- /dev/null +++ b/dropbox/dropbox.go @@ -0,0 +1,449 @@ +// Dropbox interface +package dropbox + +/* +Limitations of dropbox + +File system is case insensitive! + +Can only have 25,000 objects in a directory + +/delta might be more efficient than recursing with Metadata + +Setting metadata is problematic! Might have to use a database + +Md5sum has to download the file +*/ + +import ( + "crypto/md5" + "errors" + "fmt" + "io" + "log" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/stacktic/dropbox" +) + +// Constants +const ( + rcloneAppKey = "5jcck7diasz0rqy" + rcloneAppSecret = "1n9m04y2zx7bf26" + uploadChunkSize = 64 * 1024 // chunk size for upload + metadataLimit = dropbox.MetadataLimitDefault // max items to fetch at once +) + +// Register with Fs +func init() { + fs.Register(&fs.FsInfo{ + Name: "dropbox", + NewFs: NewFs, + Config: Config, + Options: []fs.Option{{ + Name: "app_key", + Help: "Dropbox App Key - leave blank to use rclone's.", + }, { + Name: "app_secret", + Help: "Dropbox App Secret - leave blank to use rclone's.", + }}, + }) +} + +// Configuration helper - called after the user has put in the defaults +func Config(name string) { + // See if already have a token + token := fs.ConfigFile.MustValue(name, "token") + if token != "" { + fmt.Printf("Already have a dropbox token - refresh?\n") + if !fs.Confirm() { + return + } + } + + // Get a dropbox + db := newDropbox(name) + + // This method will ask the user to visit an URL and paste the generated code. + if err := db.Auth(); err != nil { + log.Fatalf("Failed to authorize: %v", err) + } + + // Get the token + token = db.AccessToken() + + // Stuff it in the config file if it has changed + old := fs.ConfigFile.MustValue(name, "token") + if token != old { + fs.ConfigFile.SetValue(name, "token", token) + fs.SaveConfig() + } +} + +// FsDropbox represents a remote dropbox server +type FsDropbox struct { + db *dropbox.Dropbox // the connection to the dropbox server + root string // the path we are working on + slashRoot string // root with "/" prefix and postix +} + +// FsObjectDropbox describes a dropbox object +type FsObjectDropbox struct { + dropbox *FsDropbox // what this object is part of + remote string // The remote path + md5sum string // md5sum of the object + bytes int64 // size of the object + modTime time.Time // time it was last modified +} + +// ------------------------------------------------------------ + +// String converts this FsDropbox to a string +func (f *FsDropbox) String() string { + return fmt.Sprintf("Dropbox root '%s'", f.root) +} + +// parseParse parses a dropbox 'url' +func parseDropboxPath(path string) (root string, err error) { + root = strings.Trim(path, "/") + return +} + +// Makes a new dropbox from the config +func newDropbox(name string) *dropbox.Dropbox { + db := dropbox.NewDropbox() + + appKey := fs.ConfigFile.MustValue(name, "app_key") + if appKey == "" { + appKey = rcloneAppKey + } + appSecret := fs.ConfigFile.MustValue(name, "app_secret") + if appSecret == "" { + appSecret = rcloneAppSecret + } + + db.SetAppInfo(appKey, appSecret) + + return db +} + +// NewFs contstructs an FsDropbox from the path, container:path +func NewFs(name, path string) (fs.Fs, error) { + db := newDropbox(name) + + root, err := parseDropboxPath(path) + if err != nil { + return nil, err + } + slashRoot := "/" + root + if root != "" { + slashRoot += "/" + } + f := &FsDropbox{ + root: root, + slashRoot: slashRoot, + db: db, + } + + // Read the token from the config file + token := fs.ConfigFile.MustValue(name, "token") + + // Authorize the client + db.SetAccessToken(token) + + return f, nil +} + +// Return an FsObject from a path +func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) (fs.Object, error) { + fs := &FsObjectDropbox{ + dropbox: f, + remote: remote, + } + if info != nil { + fs.setMetaData(info) + } else { + err := fs.readMetaData() // reads info and meta, returning an error + if err != nil { + // logged already fs.Debug("Failed to read info: %s", err) + return nil, err + } + } + return fs, nil +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *FsDropbox) NewFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object { + fs, _ := f.newFsObjectWithInfo(remote, info) + // Errors have already been logged + return fs +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *FsDropbox) NewFsObject(remote string) fs.Object { + return f.NewFsObjectWithInfo(remote, nil) +} + +// Strips the root off entry and returns it +func (f *FsDropbox) stripRoot(entry *dropbox.Entry) string { + path := entry.Path + if strings.HasPrefix(path, f.slashRoot) { + path = path[len(f.slashRoot):] + } + return path +} + +// Walk the path returning a channel of FsObjects +// +// FIXME could do this in parallel but needs to be limited to Checkers +func (f *FsDropbox) list(path string, out fs.ObjectsChan) { + entry, err := f.db.Metadata(f.slashRoot+path, true, false, "", "", metadataLimit) + if err != nil { + fs.Stats.Error() + fs.Log(f, "Couldn't list %q: %s", path, err) + } else { + for i := range entry.Contents { + entry := &entry.Contents[i] + path = f.stripRoot(entry) + if entry.IsDir { + f.list(path, out) + } else { + out <- f.NewFsObjectWithInfo(path, entry) + } + } + } +} + +// Walk the path returning a channel of FsObjects +func (f *FsDropbox) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) + go func() { + defer close(out) + f.list("", out) + }() + return out +} + +// Walk the path returning a channel of FsObjects +func (f *FsDropbox) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) + go func() { + defer close(out) + entry, err := f.db.Metadata(f.root, true, false, "", "", metadataLimit) + if err != nil { + fs.Stats.Error() + fs.Log(f, "Couldn't list directories in root: %s", err) + } else { + for i := range entry.Contents { + entry := &entry.Contents[i] + if entry.IsDir { + out <- &fs.Dir{ + Name: f.stripRoot(entry), + When: time.Time(entry.ClientMtime), + Bytes: int64(entry.Bytes), + Count: -1, + } + } + } + } + }() + return out +} + +// A read closer which doesn't close the input +type readCloser struct { + in io.Reader +} + +// Read bytes from the object - see io.Reader +func (rc *readCloser) Read(p []byte) (n int, err error) { + return rc.in.Read(p) +} + +// Dummy close function +func (rc *readCloser) Close() error { + return nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *FsDropbox) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { + // Temporary FsObject under construction + o := &FsObjectDropbox{dropbox: f, remote: remote} + return o, o.Update(in, modTime, size) +} + +// Mkdir creates the container if it doesn't exist +func (f *FsDropbox) Mkdir() error { + _, err := f.db.CreateFolder(f.slashRoot) + return err +} + +// Rmdir deletes the container +// +// Returns an error if it isn't empty +func (f *FsDropbox) Rmdir() error { + entry, err := f.db.Metadata(f.slashRoot, true, false, "", "", metadataLimit) + if err != nil { + return err + } + if len(entry.Contents) != 0 { + return errors.New("Directory not empty") + } + return f.Purge() +} + +// Return the precision +func (fs *FsDropbox) Precision() time.Duration { + return time.Second +} + +// Purge deletes all the files and the container +// +// Returns an error if it isn't empty +func (f *FsDropbox) Purge() error { + _, err := f.db.Delete(f.slashRoot) + return err +} + +// ------------------------------------------------------------ + +// Return the parent Fs +func (o *FsObjectDropbox) Fs() fs.Fs { + return o.dropbox +} + +// Return a string version +func (o *FsObjectDropbox) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Return the remote path +func (o *FsObjectDropbox) Remote() string { + return o.remote +} + +// Md5sum returns the Md5sum of an object returning a lowercase hex string +// +// FIXME has to download the file! +func (o *FsObjectDropbox) Md5sum() (string, error) { + if o.md5sum != "" { + return o.md5sum, nil + } + in, err := o.Open() + if err != nil { + return "", err + } + defer in.Close() + hash := md5.New() + _, err = io.Copy(hash, in) + if err != nil { + return "", err + } + o.md5sum = fmt.Sprintf("%x", hash.Sum(nil)) + return o.md5sum, nil +} + +// Size returns the size of an object in bytes +func (o *FsObjectDropbox) Size() int64 { + return o.bytes +} + +// setMetaData sets the fs data from a dropbox.Entry +func (o *FsObjectDropbox) setMetaData(info *dropbox.Entry) { + o.bytes = int64(info.Bytes) + o.modTime = time.Time(info.ClientMtime) +} + +// Returns the remote path for the object +func (o *FsObjectDropbox) remotePath() string { + return o.dropbox.slashRoot + o.remote +} + +// readMetaData gets the info if it hasn't already been fetched +func (o *FsObjectDropbox) readMetaData() (err error) { + if !o.modTime.IsZero() { + return nil + } + entry, err := o.dropbox.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit) + if err != nil { + fs.Debug(o, "Couldn't find directory: %s", err) + return fmt.Errorf("Couldn't find directory: %s", err) + } + o.setMetaData(entry) + return nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *FsObjectDropbox) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %s", err) + return time.Now() + } + return o.modTime +} + +// Sets the modification time of the local fs object +func (o *FsObjectDropbox) SetModTime(modTime time.Time) { + err := o.readMetaData() + if err != nil { + fs.Stats.Error() + fs.Log(o, "Failed to read metadata: %s", err) + return + } + // fs.Stats.Error() + fs.Log(o, "FIXME can't update dropbox mtime") +} + +// Is this object storable +func (o *FsObjectDropbox) Storable() bool { + return true +} + +// Open an object for read +func (o *FsObjectDropbox) Open() (in io.ReadCloser, err error) { + in, _, err = o.dropbox.db.Download(o.remotePath(), "", 0) + return +} + +// Update the already existing object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *FsObjectDropbox) Update(in io.Reader, modTime time.Time, size int64) error { + rc := &readCloser{in: in} + entry, err := o.dropbox.db.UploadByChunk(rc, uploadChunkSize, o.remotePath(), true, "") + if err != nil { + return fmt.Errorf("Upload failed: %s", err) + } + o.setMetaData(entry) + return nil +} + +// Remove an object +func (o *FsObjectDropbox) Remove() error { + _, err := o.dropbox.db.Delete(o.remotePath()) + return err +} + +// Check the interfaces are satisfied +var _ fs.Fs = &FsDropbox{} +var _ fs.Purger = &FsDropbox{} +var _ fs.Object = &FsObjectDropbox{} diff --git a/rclone.go b/rclone.go index 6d89578fd..b6a3f477f 100644 --- a/rclone.go +++ b/rclone.go @@ -17,6 +17,7 @@ import ( "github.com/ncw/rclone/fs" // Active file systems _ "github.com/ncw/rclone/drive" + _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/local" _ "github.com/ncw/rclone/s3" _ "github.com/ncw/rclone/swift" diff --git a/rclonetest/rclonetest.go b/rclonetest/rclonetest.go index 19a93fff9..8e322b365 100644 --- a/rclonetest/rclonetest.go +++ b/rclonetest/rclonetest.go @@ -18,6 +18,7 @@ import ( // Active file systems _ "github.com/ncw/rclone/drive" + _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/local" _ "github.com/ncw/rclone/s3" _ "github.com/ncw/rclone/swift"