forked from TrueCloudLab/rclone
Get rid of LimitedFs - FIXME needs docs on copying single files
If remote:path points to a file make NewFs return a sentinel error fs.ErrorIsFile and an Fs which points to the parent. Use this to remove the LimitedFs and just add this file to the --files-from list. This means that server side operations can be used also. Fixes #518 Fixes #545
This commit is contained in:
parent
5a3b109e25
commit
1a87b69376
26 changed files with 101 additions and 239 deletions
|
@ -220,8 +220,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
// File doesn't exist so return old f
|
// File doesn't exist so return old f
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
// return a Fs Limited to this object
|
// return an error with an fs which points to the parent
|
||||||
return fs.NewLimited(&newF, obj), nil
|
return &newF, fs.ErrorIsFile
|
||||||
}
|
}
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
3
b2/b2.go
3
b2/b2.go
|
@ -239,7 +239,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
}
|
}
|
||||||
obj := f.NewFsObject(remote)
|
obj := f.NewFsObject(remote)
|
||||||
if obj != nil {
|
if obj != nil {
|
||||||
return fs.NewLimited(f, obj), nil
|
// return an error with an fs which points to the parent
|
||||||
|
return f, fs.ErrorIsFile
|
||||||
}
|
}
|
||||||
f.root = oldRoot
|
f.root = oldRoot
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -337,13 +337,13 @@ func NewFs(name, path string) (fs.Fs, error) {
|
||||||
// No root so return old f
|
// No root so return old f
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
obj, err := newF.newFsObjectWithInfoErr(remote, nil)
|
_, err := newF.newFsObjectWithInfoErr(remote, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// File doesn't exist so return old f
|
// File doesn't exist so return old f
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
// return a Fs Limited to this object
|
// return an error with an fs which points to the parent
|
||||||
return fs.NewLimited(&newF, obj), nil
|
return &newF, fs.ErrorIsFile
|
||||||
}
|
}
|
||||||
// fmt.Printf("Root id %s", f.dirCache.RootID())
|
// fmt.Printf("Root id %s", f.dirCache.RootID())
|
||||||
return f, nil
|
return f, nil
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -172,15 +172,13 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
// See if the root is actually an object
|
// See if the root is actually an object
|
||||||
entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
|
entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
|
||||||
if err == nil && !entry.IsDir {
|
if err == nil && !entry.IsDir {
|
||||||
remote := path.Base(f.root)
|
|
||||||
newRoot := path.Dir(f.root)
|
newRoot := path.Dir(f.root)
|
||||||
if newRoot == "." {
|
if newRoot == "." {
|
||||||
newRoot = ""
|
newRoot = ""
|
||||||
}
|
}
|
||||||
f.setRoot(newRoot)
|
f.setRoot(newRoot)
|
||||||
obj := f.NewFsObject(remote)
|
// return an error with an fs which points to the parent
|
||||||
// return a Fs Limited to this object
|
return f, fs.ErrorIsFile
|
||||||
return fs.NewLimited(f, obj), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return f, nil
|
return f, nil
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
41
fs/fs.go
41
fs/fs.go
|
@ -42,6 +42,7 @@ var (
|
||||||
ErrorLevelNotSupported = errors.New("level value not supported")
|
ErrorLevelNotSupported = errors.New("level value not supported")
|
||||||
ErrorListAborted = errors.New("list aborted")
|
ErrorListAborted = errors.New("list aborted")
|
||||||
ErrorListOnlyRoot = errors.New("can only list from root")
|
ErrorListOnlyRoot = errors.New("can only list from root")
|
||||||
|
ErrorIsFile = errors.New("is a file not a directory")
|
||||||
)
|
)
|
||||||
|
|
||||||
// RegInfo provides information about a filesystem
|
// RegInfo provides information about a filesystem
|
||||||
|
@ -51,8 +52,8 @@ type RegInfo struct {
|
||||||
// Description of this fs - defaults to Name
|
// Description of this fs - defaults to Name
|
||||||
Description string
|
Description string
|
||||||
// Create a new file system. If root refers to an existing
|
// Create a new file system. If root refers to an existing
|
||||||
// object, then it should return a Fs which only returns that
|
// object, then it should return a Fs which which points to
|
||||||
// object.
|
// the parent of that object and ErrorIsFile.
|
||||||
NewFs func(name string, root string) (Fs, error)
|
NewFs func(name string, root string) (Fs, error)
|
||||||
// Function to call to help with config
|
// Function to call to help with config
|
||||||
Config func(string)
|
Config func(string)
|
||||||
|
@ -347,6 +348,26 @@ func Find(name string) (*RegInfo, error) {
|
||||||
// Pattern to match an rclone url
|
// Pattern to match an rclone url
|
||||||
var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
|
var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
|
||||||
|
|
||||||
|
// ParseRemote deconstructs a path into configName, fsPath, looking up
|
||||||
|
// the fsName in the config file (returning NotFoundInConfigFile if not found)
|
||||||
|
func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err error) {
|
||||||
|
parts := matcher.FindStringSubmatch(path)
|
||||||
|
var fsName string
|
||||||
|
fsName, configName, fsPath = "local", "local", path
|
||||||
|
if parts != nil && !isDriveLetter(parts[1]) {
|
||||||
|
configName, fsPath = parts[1], parts[2]
|
||||||
|
var err error
|
||||||
|
fsName, err = ConfigFile.GetValue(configName, "type")
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", "", ErrorNotFoundInConfigFile
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// change native directory separators to / if there are any
|
||||||
|
fsPath = filepath.ToSlash(fsPath)
|
||||||
|
fsInfo, err = Find(fsName)
|
||||||
|
return fsInfo, configName, fsPath, err
|
||||||
|
}
|
||||||
|
|
||||||
// NewFs makes a new Fs object from the path
|
// NewFs makes a new Fs object from the path
|
||||||
//
|
//
|
||||||
// The path is of the form remote:path
|
// The path is of the form remote:path
|
||||||
|
@ -357,23 +378,11 @@ var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
|
||||||
// On Windows avoid single character remote names as they can be mixed
|
// On Windows avoid single character remote names as they can be mixed
|
||||||
// up with drive letters.
|
// up with drive letters.
|
||||||
func NewFs(path string) (Fs, error) {
|
func NewFs(path string) (Fs, error) {
|
||||||
parts := matcher.FindStringSubmatch(path)
|
fsInfo, configName, fsPath, err := ParseRemote(path)
|
||||||
fsName, configName, fsPath := "local", "local", path
|
|
||||||
if parts != nil && !isDriveLetter(parts[1]) {
|
|
||||||
configName, fsPath = parts[1], parts[2]
|
|
||||||
var err error
|
|
||||||
fsName, err = ConfigFile.GetValue(configName, "type")
|
|
||||||
if err != nil {
|
|
||||||
return nil, ErrorNotFoundInConfigFile
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fs, err := Find(fsName)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// change native directory separators to / if there are any
|
return fsInfo.NewFs(configName, fsPath)
|
||||||
fsPath = filepath.ToSlash(fsPath)
|
|
||||||
return fs.NewFs(configName, fsPath)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// DebugLogger - logs to Stdout
|
// DebugLogger - logs to Stdout
|
||||||
|
|
147
fs/limited.go
147
fs/limited.go
|
@ -1,147 +0,0 @@
|
||||||
package fs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Limited defines a Fs which can only return the Objects passed in
|
|
||||||
// from the Fs passed in
|
|
||||||
type Limited struct {
|
|
||||||
objects []Object
|
|
||||||
fs Fs
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewLimited maks a limited Fs limited to the objects passed in
|
|
||||||
func NewLimited(fs Fs, objects ...Object) Fs {
|
|
||||||
f := &Limited{
|
|
||||||
objects: objects,
|
|
||||||
fs: fs,
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// Name is name of the remote (as passed into NewFs)
|
|
||||||
func (f *Limited) Name() string {
|
|
||||||
return f.fs.Name() // return name of underlying remote
|
|
||||||
}
|
|
||||||
|
|
||||||
// Root is the root of the remote (as passed into NewFs)
|
|
||||||
func (f *Limited) Root() string {
|
|
||||||
return f.fs.Root() // return root of underlying remote
|
|
||||||
}
|
|
||||||
|
|
||||||
// String returns a description of the FS
|
|
||||||
func (f *Limited) String() string {
|
|
||||||
return fmt.Sprintf("%s limited to %d objects", f.fs.String(), len(f.objects))
|
|
||||||
}
|
|
||||||
|
|
||||||
// List the Fs into a channel
|
|
||||||
func (f *Limited) List(opts ListOpts, dir string) {
|
|
||||||
defer opts.Finished()
|
|
||||||
if dir != "" {
|
|
||||||
opts.SetError(ErrorListOnlyRoot)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, obj := range f.objects {
|
|
||||||
if opts.Add(obj) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewFsObject finds the Object at remote. Returns nil if can't be found
|
|
||||||
func (f *Limited) NewFsObject(remote string) Object {
|
|
||||||
for _, obj := range f.objects {
|
|
||||||
if obj.Remote() == remote {
|
|
||||||
return obj
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put in to the remote path with the modTime given of the given size
|
|
||||||
//
|
|
||||||
// May create the object even if it returns an error - if so
|
|
||||||
// will return the object and the error, otherwise will return
|
|
||||||
// nil and the error
|
|
||||||
func (f *Limited) Put(in io.Reader, src ObjectInfo) (Object, error) {
|
|
||||||
remote := src.Remote()
|
|
||||||
obj := f.NewFsObject(remote)
|
|
||||||
if obj == nil {
|
|
||||||
return nil, errors.Errorf("can't create %q in limited fs", remote)
|
|
||||||
}
|
|
||||||
return obj, obj.Update(in, src)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mkdir make the directory (container, bucket)
|
|
||||||
func (f *Limited) Mkdir() error {
|
|
||||||
// All directories are already made - just ignore
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rmdir removes the directory (container, bucket) if empty
|
|
||||||
func (f *Limited) Rmdir() error {
|
|
||||||
// Ignore this in a limited fs
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Precision of the ModTimes in this Fs
|
|
||||||
func (f *Limited) Precision() time.Duration {
|
|
||||||
return f.fs.Precision()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hashes returns the supported hash sets.
|
|
||||||
func (f *Limited) Hashes() HashSet {
|
|
||||||
return f.fs.Hashes()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Copy src to this remote using server side copy operations.
|
|
||||||
//
|
|
||||||
// This is stored with the remote path given
|
|
||||||
//
|
|
||||||
// It returns the destination Object and a possible error
|
|
||||||
//
|
|
||||||
// Will only be called if src.Fs().Name() == f.Name()
|
|
||||||
//
|
|
||||||
// If it isn't possible then return fs.ErrorCantCopy
|
|
||||||
func (f *Limited) Copy(src Object, remote string) (Object, error) {
|
|
||||||
fCopy, ok := f.fs.(Copier)
|
|
||||||
if !ok {
|
|
||||||
return nil, ErrorCantCopy
|
|
||||||
}
|
|
||||||
return fCopy.Copy(src, remote)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Move src to this remote using server side move operations.
|
|
||||||
//
|
|
||||||
// This is stored with the remote path given
|
|
||||||
//
|
|
||||||
// It returns the destination Object and a possible error
|
|
||||||
//
|
|
||||||
// Will only be called if src.Fs().Name() == f.Name()
|
|
||||||
//
|
|
||||||
// If it isn't possible then return fs.ErrorCantMove
|
|
||||||
func (f *Limited) Move(src Object, remote string) (Object, error) {
|
|
||||||
fMove, ok := f.fs.(Mover)
|
|
||||||
if !ok {
|
|
||||||
return nil, ErrorCantMove
|
|
||||||
}
|
|
||||||
return fMove.Move(src, remote)
|
|
||||||
}
|
|
||||||
|
|
||||||
// UnWrap returns the Fs that this Fs is wrapping
|
|
||||||
func (f *Limited) UnWrap() Fs {
|
|
||||||
return f.fs
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check the interfaces are satisfied
|
|
||||||
var (
|
|
||||||
_ Fs = (*Limited)(nil)
|
|
||||||
_ Copier = (*Limited)(nil)
|
|
||||||
_ Mover = (*Limited)(nil)
|
|
||||||
_ UnWrapper = (*Limited)(nil)
|
|
||||||
)
|
|
|
@ -608,31 +608,20 @@ func TestObjectStorable(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLimitedFs tests that a LimitedFs is created
|
// TestFsIsFile tests that an error is returned along with a valid fs
|
||||||
func TestLimitedFs(t *testing.T) {
|
// which points to the parent directory.
|
||||||
|
func TestFsIsFile(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
remoteName := subRemoteName + "/" + file2.Path
|
remoteName := subRemoteName + "/" + file2.Path
|
||||||
file2Copy := file2
|
file2Copy := file2
|
||||||
file2Copy.Path = "z.txt"
|
file2Copy.Path = "z.txt"
|
||||||
fileRemote, err := fs.NewFs(remoteName)
|
fileRemote, err := fs.NewFs(remoteName)
|
||||||
if err != nil {
|
assert.Equal(t, fs.ErrorIsFile, err)
|
||||||
t.Fatalf("Failed to make remote %q: %v", remoteName, err)
|
|
||||||
}
|
|
||||||
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
|
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
|
||||||
_, ok := fileRemote.(*fs.Limited)
|
|
||||||
if !ok {
|
|
||||||
// Check to see if this wraps a Limited FS
|
|
||||||
if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap {
|
|
||||||
_, ok = unwrap.UnWrap().(*fs.Limited)
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("%v is not a fs.Limited", fileRemote)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLimitedFsNotFound tests that a LimitedFs is not created if no object
|
// TestFsIsFileNotFound tests that an error is not returned if no object is found
|
||||||
func TestLimitedFsNotFound(t *testing.T) {
|
func TestFsIsFileNotFound(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
remoteName := subRemoteName + "/not found.txt"
|
remoteName := subRemoteName + "/not found.txt"
|
||||||
fileRemote, err := fs.NewFs(remoteName)
|
fileRemote, err := fs.NewFs(remoteName)
|
||||||
|
|
|
@ -252,16 +252,14 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
// Check to see if the object exists
|
// Check to see if the object exists
|
||||||
_, err = f.svc.Objects.Get(bucket, directory).Do()
|
_, err = f.svc.Objects.Get(bucket, directory).Do()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
remote := path.Base(directory)
|
|
||||||
f.root = path.Dir(directory)
|
f.root = path.Dir(directory)
|
||||||
if f.root == "." {
|
if f.root == "." {
|
||||||
f.root = ""
|
f.root = ""
|
||||||
} else {
|
} else {
|
||||||
f.root += "/"
|
f.root += "/"
|
||||||
}
|
}
|
||||||
obj := f.NewFsObject(remote)
|
// return an error with an fs which points to the parent
|
||||||
// return a Fs Limited to this object
|
return f, fs.ErrorIsFile
|
||||||
return fs.NewLimited(f, obj), nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return f, nil
|
return f, nil
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -167,11 +167,11 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
|
|
||||||
// Make inner swift Fs from the connection
|
// Make inner swift Fs from the connection
|
||||||
swiftFs, err := swift.NewFsWithConnection(name, root, c)
|
swiftFs, err := swift.NewFsWithConnection(name, root, c)
|
||||||
if err != nil {
|
if err != nil && err != fs.ErrorIsFile {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
f.Fs = swiftFs
|
f.Fs = swiftFs
|
||||||
return f, nil
|
return f, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Purge deletes all the files and the container
|
// Purge deletes all the files and the container
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -82,8 +82,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
if obj == nil {
|
if obj == nil {
|
||||||
return nil, errors.Errorf("failed to make object for %q in %q", remote, f.root)
|
return nil, errors.Errorf("failed to make object for %q in %q", remote, f.root)
|
||||||
}
|
}
|
||||||
// return a Fs Limited to this object
|
// return an error with an fs which points to the parent
|
||||||
return fs.NewLimited(f, obj), nil
|
return f, fs.ErrorIsFile
|
||||||
}
|
}
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -211,8 +211,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
// File doesn't exist so return old f
|
// File doesn't exist so return old f
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
// return a Fs Limited to this object
|
// return an error with an fs which points to the parent
|
||||||
return fs.NewLimited(&newF, obj), nil
|
return &newF, fs.ErrorIsFile
|
||||||
}
|
}
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
27
rclone.go
27
rclone.go
|
@ -7,6 +7,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"runtime"
|
"runtime"
|
||||||
"runtime/pprof"
|
"runtime/pprof"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -368,7 +369,29 @@ func ParseCommand() (*Command, []string) {
|
||||||
return command, args
|
return command, args
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFs creates a Fs from a name
|
// NewFsSrc creates a src Fs from a name
|
||||||
|
func NewFsSrc(remote string) fs.Fs {
|
||||||
|
fsInfo, configName, fsPath, err := fs.ParseRemote(remote)
|
||||||
|
if err != nil {
|
||||||
|
fs.Stats.Error()
|
||||||
|
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
||||||
|
}
|
||||||
|
f, err := fsInfo.NewFs(configName, fsPath)
|
||||||
|
if err == fs.ErrorIsFile {
|
||||||
|
if !fs.Config.Filter.InActive() {
|
||||||
|
fs.Stats.Error()
|
||||||
|
log.Fatalf("Can't limit to single files when using filters: %v", remote)
|
||||||
|
}
|
||||||
|
// Limit transfers to this file
|
||||||
|
fs.Config.Filter.AddFile(path.Base(fsPath))
|
||||||
|
} else if err != nil {
|
||||||
|
fs.Stats.Error()
|
||||||
|
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFs creates a dst Fs from a name
|
||||||
func NewFs(remote string) fs.Fs {
|
func NewFs(remote string) fs.Fs {
|
||||||
f, err := fs.NewFs(remote)
|
f, err := fs.NewFs(remote)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -462,7 +485,7 @@ func main() {
|
||||||
// Make source and destination fs
|
// Make source and destination fs
|
||||||
var fdst, fsrc fs.Fs
|
var fdst, fsrc fs.Fs
|
||||||
if len(args) >= 1 {
|
if len(args) >= 1 {
|
||||||
fdst = NewFs(args[0])
|
fdst = NewFsSrc(args[0])
|
||||||
}
|
}
|
||||||
if len(args) >= 2 {
|
if len(args) >= 2 {
|
||||||
fsrc = fdst
|
fsrc = fdst
|
||||||
|
|
6
s3/s3.go
6
s3/s3.go
|
@ -326,16 +326,14 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
}
|
}
|
||||||
_, err = f.c.HeadObject(&req)
|
_, err = f.c.HeadObject(&req)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
remote := path.Base(directory)
|
|
||||||
f.root = path.Dir(directory)
|
f.root = path.Dir(directory)
|
||||||
if f.root == "." {
|
if f.root == "." {
|
||||||
f.root = ""
|
f.root = ""
|
||||||
} else {
|
} else {
|
||||||
f.root += "/"
|
f.root += "/"
|
||||||
}
|
}
|
||||||
obj := f.NewFsObject(remote)
|
// return an error with an fs which points to the parent
|
||||||
// return a Fs Limited to this object
|
return f, fs.ErrorIsFile
|
||||||
return fs.NewLimited(f, obj), nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// f.listMultipartUploads()
|
// f.listMultipartUploads()
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -199,16 +199,14 @@ func NewFsWithConnection(name, root string, c *swift.Connection) (fs.Fs, error)
|
||||||
// Check to see if the object exists - ignoring directory markers
|
// Check to see if the object exists - ignoring directory markers
|
||||||
info, _, err := f.c.Object(container, directory)
|
info, _, err := f.c.Object(container, directory)
|
||||||
if err == nil && info.ContentType != directoryMarkerContentType {
|
if err == nil && info.ContentType != directoryMarkerContentType {
|
||||||
remote := path.Base(directory)
|
|
||||||
f.root = path.Dir(directory)
|
f.root = path.Dir(directory)
|
||||||
if f.root == "." {
|
if f.root == "." {
|
||||||
f.root = ""
|
f.root = ""
|
||||||
} else {
|
} else {
|
||||||
f.root += "/"
|
f.root += "/"
|
||||||
}
|
}
|
||||||
obj := f.NewFsObject(remote)
|
// return an error with an fs which points to the parent
|
||||||
// return a Fs Limited to this object
|
return f, fs.ErrorIsFile
|
||||||
return fs.NewLimited(f, obj), nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return f, nil
|
return f, nil
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
|
@ -128,7 +128,6 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
|
|
||||||
f.setRoot(root)
|
f.setRoot(root)
|
||||||
|
|
||||||
//limited fs
|
|
||||||
// Check to see if the object exists and is a file
|
// Check to see if the object exists and is a file
|
||||||
//request object meta info
|
//request object meta info
|
||||||
var opt2 yandex.ResourceInfoRequestOptions
|
var opt2 yandex.ResourceInfoRequestOptions
|
||||||
|
@ -136,13 +135,9 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||||
//return err
|
//return err
|
||||||
} else {
|
} else {
|
||||||
if ResourceInfoResponse.ResourceType == "file" {
|
if ResourceInfoResponse.ResourceType == "file" {
|
||||||
//limited fs
|
|
||||||
remote := path.Base(root)
|
|
||||||
f.setRoot(path.Dir(root))
|
f.setRoot(path.Dir(root))
|
||||||
|
// return an error with an fs which points to the parent
|
||||||
obj := f.newFsObjectWithInfo(remote, ResourceInfoResponse)
|
return f, fs.ErrorIsFile
|
||||||
// return a Fs Limited to this object
|
|
||||||
return fs.NewLimited(f, obj), nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,8 +51,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
|
||||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||||
|
|
Loading…
Reference in a new issue