From 9e3ea3c6acbf1f427d82914ac8cc4ef91281ea8d Mon Sep 17 00:00:00 2001 From: Felix Brucker Date: Sat, 18 Aug 2018 02:39:49 +0200 Subject: [PATCH] union: Implement union backend which reads from multiple backends --- backend/all/all.go | 1 + backend/union/union.go | 211 ++++++++++++++++++++++++++++++++++++ backend/union/union_test.go | 17 +++ docs/content/docs.md | 1 + docs/content/union.md | 144 ++++++++++++++++++++++++ fstest/test_all/test_all.go | 5 + 6 files changed, 379 insertions(+) create mode 100644 backend/union/union.go create mode 100644 backend/union/union_test.go create mode 100644 docs/content/union.md diff --git a/backend/all/all.go b/backend/all/all.go index 29c4bf4ec..d80e440c8 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -25,6 +25,7 @@ import ( _ "github.com/ncw/rclone/backend/s3" _ "github.com/ncw/rclone/backend/sftp" _ "github.com/ncw/rclone/backend/swift" + _ "github.com/ncw/rclone/backend/union" _ "github.com/ncw/rclone/backend/webdav" _ "github.com/ncw/rclone/backend/yandex" ) diff --git a/backend/union/union.go b/backend/union/union.go new file mode 100644 index 000000000..6db029074 --- /dev/null +++ b/backend/union/union.go @@ -0,0 +1,211 @@ +package union + +import ( + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configmap" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/ncw/rclone/fs/hash" +) + +// Register with Fs +func init() { + fsi := &fs.RegInfo{ + Name: "union", + Description: "Builds a stackable unification remote, which can appear to merge the contents of several remotes", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "remotes", + Help: "List of space separated remotes.\nCan be 'remotea:test/dir remoteb:', '\"remotea:test/space dir\" remoteb:', etc.\nThe last remote is used to write to.", + Required: true, + }}, + } + fs.Register(fsi) +} + +// Options defines the configuration for this backend +type Options struct { + Remotes fs.SpaceSepList `config:"remotes"` +} + +// Fs represents a remote acd server +type Fs struct { + name string // name of this remote + features *fs.Features // optional features + opt Options // options for this Fs + root string // the path we are working on + remotes []fs.Fs // slice of remotes +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("union root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Rmdir removes the root directory of the Fs object +func (f *Fs) Rmdir(dir string) error { + return f.remotes[len(f.remotes)-1].Rmdir(dir) +} + +// Hashes returns hash.HashNone to indicate remote hashing is unavailable +func (f *Fs) Hashes() hash.Set { + // This could probably be set if all remotes share the same hashing algorithm + return hash.Set(hash.None) +} + +// Mkdir makes the root directory of the Fs object +func (f *Fs) Mkdir(dir string) error { + return f.remotes[len(f.remotes)-1].Mkdir(dir) +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.remotes[len(f.remotes)-1].Put(in, src, options...) +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + set := make(map[string]fs.DirEntry) + for _, remote := range f.remotes { + var remoteEntries, err = remote.List(dir) + if err != nil { + continue + } + for _, remoteEntry := range remoteEntries { + set[remoteEntry.Remote()] = remoteEntry + } + } + + for key := range set { + entries = append(entries, set[key]) + } + return entries, nil +} + +// NewObject creates a new remote union file object based on the first Object it finds (reverse remote order) +func (f *Fs) NewObject(path string) (fs.Object, error) { + for i := range f.remotes { + var remote = f.remotes[len(f.remotes)-i-1] + var obj, err = remote.NewObject(path) + if err != nil { + continue + } + return obj, nil + } + return nil, fs.ErrorObjectNotFound +} + +// Precision is the greatest Precision of all remotes +func (f *Fs) Precision() time.Duration { + var greatestPrecision = time.Second + for _, remote := range f.remotes { + if remote.Precision() <= greatestPrecision { + continue + } + greatestPrecision = remote.Precision() + } + return greatestPrecision +} + +// NewFs constructs an Fs from the path. +// +// The returned Fs is the actual Fs, referenced by remote in the config +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + if len(opt.Remotes) == 0 { + return nil, errors.New("union can't point to an empty remote - check the value of the remotes setting") + } + if len(opt.Remotes) == 1 { + return nil, errors.New("union can't point to a single remote - check the value of the remotes setting") + } + for _, remote := range opt.Remotes { + if strings.HasPrefix(remote, name+":") { + return nil, errors.New("can't point union remote at itself - check the value of the remote setting") + } + } + + var remotes []fs.Fs + for i := range opt.Remotes { + // Last remote first so we return the correct (last) matching fs in case of fs.ErrorIsFile + var remote = opt.Remotes[len(opt.Remotes)-i-1] + _, configName, fsPath, err := fs.ParseRemote(remote) + if err != nil { + return nil, err + } + var rootString = path.Join(fsPath, filepath.ToSlash(root)) + if configName != "local" { + rootString = configName + ":" + rootString + } + myFs, err := fs.NewFs(rootString) + if err != nil { + if err == fs.ErrorIsFile { + return myFs, err + } + return nil, err + } + remotes = append(remotes, myFs) + } + + // Reverse the remotes again so they are in the order as before + for i, j := 0, len(remotes)-1; i < j; i, j = i+1, j-1 { + remotes[i], remotes[j] = remotes[j], remotes[i] + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + remotes: remotes, + } + var features = (&fs.Features{}).Fill(f) + for _, remote := range f.remotes { + features = features.Mask(remote) + } + f.features = features + + return f, nil +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} +) diff --git a/backend/union/union_test.go b/backend/union/union_test.go new file mode 100644 index 000000000..82a92c7bc --- /dev/null +++ b/backend/union/union_test.go @@ -0,0 +1,17 @@ +// Test Union filesystem interface +package union_test + +import ( + "testing" + + "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestUnion:", + NilObject: (*local.Object)(nil), + }) +} diff --git a/docs/content/docs.md b/docs/content/docs.md index a5a90646d..4d655c4a4 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -42,6 +42,7 @@ See the following for detailed instructions for * [Pcloud](/pcloud/) * [QingStor](/qingstor/) * [SFTP](/sftp/) + * [Union](/union/) * [WebDAV](/webdav/) * [Yandex Disk](/yandex/) * [The local filesystem](/local/) diff --git a/docs/content/union.md b/docs/content/union.md new file mode 100644 index 000000000..bc93478f2 --- /dev/null +++ b/docs/content/union.md @@ -0,0 +1,144 @@ +--- +title: "Union" +description: "Remote Unification" +date: "2018-08-29" +--- + + Union +----------------------------------------- + +The `union` remote provides a unification similar to UnionFS using other remotes. + +Paths may be as deep as required or a local path, +eg `remote:directory/subdirectory` or `/directory/subdirectory`. + +During the initial setup with `rclone config` you will specify the target +remotes as a space separated list. The target remotes can either be a local paths or other remotes. + +The order of the remotes is important as it defines which remotes take precedence over others if there are files with the same name in the same logical path. +The last remote is the topmost remote and replaces files with the same name from previous remotes. + +Only the last remote is used to write to and delete from, all other remotes are read-only. + +Subfolders can be used in target remote. Asume a union remote named `backup` +with the remotes `mydrive:private/backup mydrive2:/backup`. Invoking `rclone mkdir backup:desktop` +is exactly the same as invoking `rclone mkdir mydrive2:/backup/desktop`. + +There will be no special handling of paths containing `..` segments. +Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking +`rclone mkdir mydrive2:/backup/../desktop`. + +Here is an example of how to make a union called `remote` for local folders. +First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive + \ "amazon cloud drive" + 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio) + \ "s3" + 4 / Backblaze B2 + \ "b2" + 5 / Box + \ "box" + 6 / Builds a stackable unification remote, which can appear to merge the contents of several remotes + \ "union" + 7 / Cache a remote + \ "cache" + 8 / Dropbox + \ "dropbox" + 9 / Encrypt/Decrypt a remote + \ "crypt" +10 / FTP Connection + \ "ftp" +11 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" +12 / Google Drive + \ "drive" +13 / Hubic + \ "hubic" +14 / JottaCloud + \ "jottacloud" +15 / Local Disk + \ "local" +16 / Mega + \ "mega" +17 / Microsoft Azure Blob Storage + \ "azureblob" +18 / Microsoft OneDrive + \ "onedrive" +19 / OpenDrive + \ "opendrive" +20 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +21 / Pcloud + \ "pcloud" +22 / QingCloud Object Storage + \ "qingstor" +23 / SSH/SFTP Connection + \ "sftp" +24 / Webdav + \ "webdav" +25 / Yandex Disk + \ "yandex" +26 / http Connection + \ "http" +Storage> union +List of space separated remotes. +Can be 'remotea:test/dir remoteb:', '"remotea:test/space dir" remoteb:', etc. +The last remote is used to write to. +Enter a string value. Press Enter for the default (""). +remotes> +Remote config +-------------------- +[remote] +type = union +remotes = C:\dir1 C:\dir2 C:\dir3 +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +remote union + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` + +Once configured you can then use `rclone` like this, + +List directories in top level in `C:\dir1`, `C:\dir2` and `C:\dir3` + + rclone lsd remote: + +List all the files in `C:\dir1`, `C:\dir2` and `C:\dir3` + + rclone ls remote: + +Copy another local directory to the union directory called source, which will be placed into `C:\dir3` + + rclone copy C:\source remote:source + diff --git a/fstest/test_all/test_all.go b/fstest/test_all/test_all.go index cffd5e221..c74879b05 100644 --- a/fstest/test_all/test_all.go +++ b/fstest/test_all/test_all.go @@ -147,6 +147,11 @@ var ( SubDir: false, FastList: false, }, + { + Name: "TestUnion:", + SubDir: false, + FastList: false, + }, } // Flags maxTries = flag.Int("maxtries", 5, "Number of times to try each test")