TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Change List interface and add ListR optional interface

Browse source 
This simplifies the implementation of remotes.  The only required
interface is now `List` which is a simple one level directory list.

Optionally remotes may implement `ListR` if they have an efficient way
of doing a recursive list.
This commit is contained in:
Nick Craig-Wood 2017-06-11 22:43:31 +01:00
parent 6fc88ff32e
commit 8a6a8b9623
37 changed files with 994 additions and 1636 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
amazonclouddrive/amazonclouddrive.go
View file

@ -3,7 +3,6 @@
package amazonclouddrive
/*
FIXME make searching for directory in id and file in id more efficient
- use the name: search parameter - remember the escaping rules
- use Folder GetNode and GetFile
@ -399,47 +398,58 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
return
}
// ListDir reads the directory specified by the job into out, returning any more jobs
func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) {
fs.Debugf(f, "Reading %q", job.Path)
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
err = f.dirCache.FindRoot(false)
if err != nil {
return nil, err
}
directoryID, err := f.dirCache.FindDir(dir, false)
if err != nil {
return nil, err
}
maxTries := fs.Config.LowLevelRetries
var iErr error
for tries := 1; tries <= maxTries; tries++ {
_, err = f.listAll(job.DirID, "", false, false, func(node *acd.Node) bool {
remote := job.Path + *node.Name
entries = nil
_, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool {
remote := path.Join(dir, *node.Name)
switch *node.Kind {
case folderKind:
if out.IncludeDirectory(remote) {
// cache the directory ID for later lookups
f.dirCache.Put(remote, *node.Id)
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
dir.When, _ = time.Parse(timeFormat, *node.ModifiedDate) // FIXME
if out.AddDir(dir) {
return true
}
if job.Depth > 0 {
jobs = append(jobs, dircache.ListDirJob{DirID: *node.Id, Path: remote + "/", Depth: job.Depth - 1})
}
}
d.When, _ = time.Parse(timeFormat, *node.ModifiedDate) // FIXME
entries = append(entries, d)
case fileKind:
o, err := f.newObjectWithInfo(remote, node)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
iErr = err
return true
}
entries = append(entries, o)
default:
// ignore ASSET etc
}
return false
})
if iErr != nil {
return nil, iErr
}
if fs.IsRetryError(err) {
fs.Debugf(f, "Directory listing error for %q: %v - low level retry %d/%d", job.Path, err, tries, maxTries)
fs.Debugf(f, "Directory listing error for %q: %v - low level retry %d/%d", dir, err, tries, maxTries)
continue
}
if err != nil {
@ -447,13 +457,7 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
break
}
fs.Debugf(f, "Finished reading %q", job.Path)
return jobs, err
}
// List walks the path returning iles and directories into out
func (f *Fs) List(out fs.ListOpts, dir string) {
f.dirCache.List(f, out, dir)
return entries, nil
}
// checkUpload checks to see if an error occurred after the file was

5
amazonclouddrive/amazonclouddrive_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

121
b2/b2.go
View file

@ -438,18 +438,14 @@ var errEndList = errors.New("end list")
// than 1000)
//
// If hidden is set then it will list the hidden (deleted) files too.
func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, fn listFn) error {
func (f *Fs) list(dir string, recurse bool, prefix string, limit int, hidden bool, fn listFn) error {
root := f.root
if dir != "" {
root += dir + "/"
}
delimiter := ""
switch level {
case 1:
if !recurse {
delimiter = "/"
case fs.MaxLevel:
default:
return fs.ErrorLevelNotSupported
}
bucketID, err := f.getBucketID()
if err != nil {
@ -497,7 +493,7 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
}
remote := file.Name[len(f.root):]
// Check for directory
isDirectory := level != 0 && strings.HasSuffix(remote, "/")
isDirectory := strings.HasSuffix(remote, "/")
if isDirectory {
remote = remote[:len(remote)-1]
}
@ -522,83 +518,120 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
return nil
}
// listFiles walks the path returning files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
// List the objects
last := ""
err := f.list(dir, out.Level(), "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
// Convert a list item into a BasicInfo
func (f *Fs) itemToDirEntry(remote string, object *api.File, isDirectory bool, last *string) (fs.BasicInfo, error) {
if isDirectory {
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
return d, nil
}
} else {
if remote == last {
if remote == *last {
remote = object.UploadTimestamp.AddVersion(remote)
} else {
last = remote
*last = remote
}
// hide objects represent deleted files which we don't list
if object.Action == "hide" {
return nil
return nil, nil
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return nil, err
}
return o, nil
}
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
last := ""
err = f.list(dir, false, "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory, &last)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
if entry != nil {
entries = append(entries, entry)
}
return nil
})
if err != nil {
out.SetError(err)
return nil, err
}
return entries, nil
}
// listBuckets returns all the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
defer out.Finished()
func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
return nil, fs.ErrorListBucketRequired
}
err := f.listBucketsToFn(func(bucket *api.Bucket) error {
dir := &fs.Dir{
err = f.listBucketsToFn(func(bucket *api.Bucket) error {
d := &fs.Dir{
Name: bucket.Name,
Bytes: -1,
Count: -1,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
entries = append(entries, d)
return nil
})
if err != nil {
out.SetError(err)
return nil, err
}
return entries, nil
}
// List walks the path returning files and directories to out
func (f *Fs) List(out fs.ListOpts, dir string) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
if f.bucket == "" {
f.listBuckets(out, dir)
} else {
f.listFiles(out, dir)
return f.listBuckets(dir)
}
return
return f.listDir(dir)
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(out fs.ListOpts, dir string) {
f.List(out, dir) // FIXME
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if f.bucket == "" {
return fs.ErrorListBucketRequired
}
list := fs.NewListRHelper(callback)
last := ""
err = f.list(dir, true, "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory, &last)
if err != nil {
return err
}
return list.Add(entry)
})
if err != nil {
return err
}
return list.Flush()
}
// listBucketFn is called from listBucketsToFn to handle a bucket
@ -816,7 +849,7 @@ func (f *Fs) purge(oldOnly bool) error {
}()
}
last := ""
checkErr(f.list("", fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error {
checkErr(f.list("", true, "", 0, true, func(remote string, object *api.File, isDirectory bool) error {
if !isDirectory {
fs.Stats.Checking(remote)
if oldOnly && last != remote {
@ -961,7 +994,7 @@ func (o *Object) readMetaData() (err error) {
maxSearched = maxVersions
}
var info *api.File
err = o.fs.list("", fs.MaxLevel, baseRemote, maxSearched, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
err = o.fs.list("", true, baseRemote, maxSearched, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
if isDirectory {
return nil
}

5
b2/b2_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

169
crypt/crypt.go
View file

@ -6,7 +6,6 @@ import (
"io"
"path"
"strings"
"sync"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
@ -143,9 +142,91 @@ func (f *Fs) String() string {
return fmt.Sprintf("Encrypted drive '%s:%s'", f.name, f.root)
}
// List the Fs into a channel
func (f *Fs) List(opts fs.ListOpts, dir string) {
f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptDirName(dir))
// Encrypt an object file name to entries.
func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
remote := obj.Remote()
decryptedRemote, err := f.cipher.DecryptFileName(remote)
if err != nil {
fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
return
}
if *cryptShowMapping {
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
}
*entries = append(*entries, f.newObject(obj))
}
// Encrypt an directory file name to entries.
func (f *Fs) addDir(entries *fs.DirEntries, dir *fs.Dir) {
remote := dir.Name
decryptedRemote, err := f.cipher.DecryptDirName(remote)
if err != nil {
fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
return
}
if *cryptShowMapping {
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
}
*entries = append(*entries, f.newDir(dir))
}
// Encrypt some directory entries
func (f *Fs) encryptEntries(entries fs.DirEntries) (newEntries fs.DirEntries, err error) {
newEntries = make(fs.DirEntries, 0, len(entries))
for _, entry := range entries {
switch x := entry.(type) {
case fs.Object:
f.add(&newEntries, x)
case *fs.Dir:
f.addDir(&newEntries, x)
default:
return nil, errors.Errorf("Unknown object type %T", entry)
}
}
return newEntries, nil
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
entries, err = f.Fs.List(f.cipher.EncryptDirName(dir))
if err != nil {
return nil, err
}
return f.encryptEntries(entries)
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
return f.Fs.Features().ListR(f.cipher.EncryptDirName(dir), func(entries fs.DirEntries) error {
newEntries, err := f.encryptEntries(entries)
if err != nil {
return err
}
return callback(newEntries)
})
}
// NewObject finds the Object at remote.
@ -512,84 +593,6 @@ func (o *ObjectInfo) Hash(hash fs.HashType) (string, error) {
return "", nil
}
// ListOpts wraps a listopts decrypting the directory listing and
// replacing the Objects
type ListOpts struct {
fs.ListOpts
f *Fs
dir string // dir we are listing
mu sync.Mutex // to protect dirs
dirs map[string]struct{} // keep track of synthetic directory objects added
}
// Make a ListOpts wrapper
func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts {
if dir != "" {
dir += "/"
}
return &ListOpts{
ListOpts: lo,
f: f,
dir: dir,
dirs: make(map[string]struct{}),
}
}
// Level gets the recursion level for this listing.
//
// Fses may ignore this, but should implement it for improved efficiency if possible.
//
// Level 1 means list just the contents of the directory
//
// Each returned item must have less than level `/`s in.
func (lo *ListOpts) Level() int {
return lo.ListOpts.Level()
}
// Add an object to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
remote := obj.Remote()
decryptedRemote, err := lo.f.cipher.DecryptFileName(remote)
if err != nil {
fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
return lo.ListOpts.IsFinished()
}
if *cryptShowMapping {
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
}
return lo.ListOpts.Add(lo.f.newObject(obj))
}
// AddDir adds a directory to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
remote := dir.Name
decryptedRemote, err := lo.f.cipher.DecryptDirName(remote)
if err != nil {
fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
return lo.ListOpts.IsFinished()
}
if *cryptShowMapping {
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
}
return lo.ListOpts.AddDir(lo.f.newDir(dir))
}
// IncludeDirectory returns whether this directory should be
// included in the listing (and recursed into or not).
func (lo *ListOpts) IncludeDirectory(remote string) bool {
decryptedRemote, err := lo.f.cipher.DecryptDirName(remote)
if err != nil {
fs.Debugf(remote, "Not including undecryptable directory name: %v", err)
return false
}
return lo.ListOpts.IncludeDirectory(decryptedRemote)
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
@ -600,7 +603,7 @@ var (
_ fs.PutUncheckeder = (*Fs)(nil)
_ fs.CleanUpper = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.ObjectInfo = (*ObjectInfo)(nil)
_ fs.Object = (*Object)(nil)
_ fs.ListOpts = (*ListOpts)(nil)
)

5
crypt/crypt2_test.go
View file

@ -27,15 +27,20 @@ func TestFsMkdir2(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir2(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty2(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty2(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty2(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound2(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile12(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError2(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile22(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile12(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile22(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile22(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot2(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot2(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir2(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir2(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel22(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel22(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile12(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject2(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and22(t *testing.T) { fstests.TestFsListFile1and2(t) }

5
crypt/crypt3_test.go
View file

@ -27,15 +27,20 @@ func TestFsMkdir3(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir3(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty3(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty3(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty3(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound3(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile13(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError3(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile23(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile13(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile23(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile23(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot3(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot3(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir3(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir3(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel23(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel23(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile13(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject3(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and23(t *testing.T) { fstests.TestFsListFile1and2(t) }

5
crypt/crypt_test.go
View file

@ -27,15 +27,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

82
dircache/list.go
View file

@ -1,82 +0,0 @@
// Listing utility functions for fses which use dircache
package dircache
import (
"sync"
"github.com/ncw/rclone/fs"
)
// ListDirJob describe a directory listing that needs to be done
type ListDirJob struct {
DirID string
Path string
Depth int
}
// ListDirer describes the interface necessary to use ListDir
type ListDirer interface {
// ListDir reads the directory specified by the job into out, returning any more jobs
ListDir(out fs.ListOpts, job ListDirJob) (jobs []ListDirJob, err error)
}
// listDir lists the directory using a recursive list from the root
//
// It does this in parallel, calling f.ListDir to do the actual reading
func listDir(f ListDirer, out fs.ListOpts, dirID string, path string) {
// Start some directory listing go routines
var wg sync.WaitGroup // sync closing of go routines
var traversing sync.WaitGroup // running directory traversals
buffer := out.Buffer()
in := make(chan ListDirJob, buffer)
for i := 0; i < buffer; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for job := range in {
jobs, err := f.ListDir(out, job)
if err != nil {
out.SetError(err)
fs.Debugf(f, "Error reading %s: %s", path, err)
} else {
traversing.Add(len(jobs))
go func() {
// Now we have traversed this directory, send these
// jobs off for traversal in the background
for _, job := range jobs {
in <- job
}
}()
}
traversing.Done()
}
}()
}
// Start the process
traversing.Add(1)
in <- ListDirJob{DirID: dirID, Path: path, Depth: out.Level() - 1}
traversing.Wait()
close(in)
wg.Wait()
}
// List walks the path returning iles and directories into out
func (dc *DirCache) List(f ListDirer, out fs.ListOpts, dir string) {
defer out.Finished()
err := dc.FindRoot(false)
if err != nil {
out.SetError(err)
return
}
id, err := dc.FindDir(dir, false)
if err != nil {
out.SetError(err)
return
}
if dir != "" {
dir += "/"
}
listDir(f, out, id, dir)
}

78
drive/drive.go
View file

@ -202,17 +202,17 @@ func parseDrivePath(path string) (root string, err error) {
return
}
// User function to process a File item from listAll
// User function to process a File item from list
//
// Should return true to finish processing
type listAllFn func(*drive.File) bool
type listFn func(*drive.File) bool
// Lists the directory required calling the user function on each item found
//
// If the user fn ever returns true then it early exits with found = true
//
// Search params: https://developers.google.com/drive/search-parameters
func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, includeTrashed bool, fn listAllFn) (found bool, err error) {
func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bool, includeTrashed bool, fn listFn) (found bool, err error) {
var query []string
if !includeTrashed {
@ -239,7 +239,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
if filesOnly {
query = append(query, fmt.Sprintf("mimeType!='%s'", driveFolderType))
}
// fmt.Printf("listAll Query = %q\n", query)
// fmt.Printf("list Query = %q\n", query)
list := f.svc.Files.List()
if len(query) > 0 {
list = list.Q(strings.Join(query, " and "))
@ -488,7 +488,7 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
// FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
// Find the leaf in pathID
found, err = f.listAll(pathID, leaf, true, false, false, func(item *drive.File) bool {
found, err = f.list(pathID, leaf, true, false, false, func(item *drive.File) bool {
if item.Title == leaf {
pathIDOut = item.Id
return true
@ -554,41 +554,49 @@ func (f *Fs) findExportFormat(filepath string, item *drive.File) (extension, lin
return "", ""
}
// ListDir reads the directory specified by the job into out, returning any more jobs
func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) {
fs.Debugf(f, "Reading %q", job.Path)
_, err = f.listAll(job.DirID, "", false, false, false, func(item *drive.File) bool {
remote := job.Path + item.Title
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
err = f.dirCache.FindRoot(false)
if err != nil {
return nil, err
}
directoryID, err := f.dirCache.FindDir(dir, false)
if err != nil {
return nil, err
}
var iErr error
_, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool {
remote := path.Join(dir, item.Title)
switch {
case *driveAuthOwnerOnly && !isAuthOwned(item):
// ignore object or directory
case item.MimeType == driveFolderType:
if out.IncludeDirectory(remote) {
// cache the directory ID for later lookups
f.dirCache.Put(remote, item.Id)
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
dir.When, _ = time.Parse(timeFormatIn, item.ModifiedDate)
if out.AddDir(dir) {
return true
}
if job.Depth > 0 {
jobs = append(jobs, dircache.ListDirJob{DirID: item.Id, Path: remote + "/", Depth: job.Depth - 1})
}
}
d.When, _ = time.Parse(timeFormatIn, item.ModifiedDate)
entries = append(entries, d)
case item.Md5Checksum != "" || item.FileSize > 0:
// If item has MD5 sum or a length it is a file stored on drive
o, err := f.newObjectWithInfo(remote, item)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
iErr = err
return true
}
entries = append(entries, o)
case len(item.ExportLinks) != 0:
// If item has export links then it is a google doc
extension, link := f.findExportFormat(remote, item)
@ -597,7 +605,7 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
} else {
o, err := f.newObjectWithInfo(remote+"."+extension, item)
if err != nil {
out.SetError(err)
iErr = err
return true
}
if !*driveSkipGdocs {
@ -605,9 +613,7 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
obj.isDocument = true
obj.url = link
obj.bytes = -1
if out.Add(o) {
return true
}
entries = append(entries, o)
} else {
fs.Debugf(f, "Skip google document: %q", remote)
}
@ -617,13 +623,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
return false
})
fs.Debugf(f, "Finished reading %q", job.Path)
return jobs, err
if err != nil {
return nil, err
}
// List walks the path returning files and directories to out
func (f *Fs) List(out fs.ListOpts, dir string) {
f.dirCache.List(f, out, dir)
if iErr != nil {
return nil, iErr
}
return entries, nil
}
// Creates a drive.File info from the parameters passed in and a half
@ -731,7 +737,7 @@ func (f *Fs) Rmdir(dir string) error {
return err
}
var trashedFiles = false
found, err := f.listAll(directoryID, "", false, false, true, func(item *drive.File) bool {
found, err := f.list(directoryID, "", false, false, true, func(item *drive.File) bool {
if item.Labels == nil || !item.Labels.Trashed {
fs.Debugf(dir, "Rmdir: contains file: %q", item.Title)
return true
@ -1145,7 +1151,7 @@ func (o *Object) readMetaData() (err error) {
return err
}
found, err := o.fs.listAll(directoryID, leaf, false, true, false, func(item *drive.File) bool {
found, err := o.fs.list(directoryID, leaf, false, true, false, func(item *drive.File) bool {
if item.Title == leaf {
o.setMetaData(item)
return true

5
drive/drive_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

62
dropbox/dropbox.go
View file

@ -5,9 +5,9 @@ package dropbox
// FIXME dropbox for business would be quite easy to add
/*
FIXME is case folding of PathDisplay going to cause a problem?
The Case folding of PathDisplay problem
From the docs
From the docs:
path_display String. The cased path to be used for display purposes
only. In rare instances the casing will not correctly match the user's
@ -17,8 +17,7 @@ casing. Changes to only the casing of paths won't be returned by
list_folder/continue. This field will be null if the file or folder is
not mounted. This field is optional.
This only becomes a problem if dropbox implements the ListR interface
which it currently doesn't.
We solve this by not implementing the ListR interface. The dropbox remote will recurse directory by directory and all will be well.
*/
import (
@ -315,8 +314,16 @@ func (f *Fs) stripRoot(path string) (string, error) {
return strip(path, f.slashRootSlash)
}
// Walk the root returning a channel of Objects
func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
root := f.slashRoot
if dir != "" {
root += "/" + dir
@ -324,12 +331,11 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
started := false
var res *files.ListFolderResult
var err error
for {
if !started {
arg := files.ListFolderArg{
Path: root,
Recursive: recursive,
Recursive: false,
}
if root == "/" {
arg.Path = "" // Specify root folder as empty string
@ -346,8 +352,7 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
err = fs.ErrorDirNotFound
}
}
out.SetError(err)
return
return nil, err
}
started = false
} else {
@ -359,8 +364,7 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
return shouldRetry(err)
})
if err != nil {
out.SetError(errors.Wrap(err, "list continue"))
return
return nil, errors.Wrap(err, "list continue")
}
}
for _, entry := range res.Entries {
@ -384,56 +388,36 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
if folderInfo != nil {
name, err := f.stripRoot(entryPath + "/")
if err != nil {
out.SetError(err)
return
return nil, err
}
name = strings.Trim(name, "/")
if name != "" && name != dir {
dir := &fs.Dir{
d := &fs.Dir{
Name: name,
When: time.Now(),
//When: folderInfo.ClientMtime,
//Bytes: folderInfo.Bytes,
//Count: -1,
}
if out.AddDir(dir) {
return
}
entries = append(entries, d)
}
} else if fileInfo != nil {
path, err := f.stripRoot(entryPath)
if err != nil {
out.SetError(err)
return
return nil, err
}
o, err := f.newObjectWithInfo(path, fileInfo)
if err != nil {
out.SetError(err)
return
}
if out.Add(o) {
return
return nil, err
}
entries = append(entries, o)
}
}
if !res.HasMore {
break
}
}
}
// List walks the path returning a channel of Objects
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
level := out.Level()
switch level {
case 1:
f.list(out, dir, false)
case fs.MaxLevel:
f.list(out, dir, true)
default:
out.SetError(fs.ErrorLevelNotSupported)
}
return entries, nil
}
// A read closer which doesn't close the input

5
dropbox/dropbox_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

87
fs/fs.go
View file

@ -41,7 +41,7 @@ var (
ErrorObjectNotFound = errors.New("object not found")
ErrorLevelNotSupported = errors.New("level value not supported")
ErrorListAborted = errors.New("list aborted")
ErrorListOnlyRoot = errors.New("can only list from root")
ErrorListBucketRequired = errors.New("bucket or container name is needed in remote")
ErrorIsFile = errors.New("is a file not a directory")
ErrorNotAFile = errors.New("is a not a regular file")
ErrorNotDeleting = errors.New("not deleting files as there were IO errors")
@ -103,17 +103,16 @@ func Register(info *RegInfo) {
// ListFser is the interface for listing a remote Fs
type ListFser interface {
// List the objects and directories of the Fs starting from dir
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
//
// Fses must support recursion levels of fs.MaxLevel and 1.
// They may return ErrorLevelNotSupported otherwise.
List(out ListOpts, dir string)
// This should return ErrDirNotFound if the directory isn't
// found.
List(dir string) (entries DirEntries, err error)
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
@ -220,6 +219,15 @@ type MimeTyper interface {
MimeType() string
}
// ListRCallback defines a callback function for ListR to use
//
// It is called for each tranche of entries read from the listing and
// if it returns an error, the listing stops.
type ListRCallback func(entries DirEntries) error
// ListRFn is defines the call used to recursively list a directory
type ListRFn func(dir string, callback ListRCallback) error
// Features describe the optional features of the Fs
type Features struct {
// Feature flags
@ -302,12 +310,17 @@ type Features struct {
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
ListR func(out ListOpts, dir string)
ListR ListRFn
}
// Fill fills in the function pointers in the Features struct from the
@ -506,54 +519,22 @@ type ListRer interface {
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
ListR(out ListOpts, dir string)
ListR(dir string, callback ListRCallback) error
}
// ObjectsChan is a channel of Objects
type ObjectsChan chan Object
// ListOpts describes the interface used for Fs.List operations
type ListOpts interface {
// Add an object to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
Add(obj Object) (abort bool)
// Add a directory to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
AddDir(dir *Dir) (abort bool)
// IncludeDirectory returns whether this directory should be
// included in the listing (and recursed into or not).
IncludeDirectory(remote string) bool
// SetError will set an error state, and will cause the listing to
// be aborted.
// Multiple goroutines can set the error state concurrently,
// but only the first will be returned to the caller.
SetError(err error)
// Level returns the level it should recurse to. Fses may
// ignore this in which case the listing will be less
// efficient.
Level() int
// Buffer returns the channel depth in use
Buffer() int
// Finished should be called when listing is finished
Finished()
// IsFinished returns whether Finished or SetError have been called
IsFinished() bool
}
// Objects is a slice of Object~s
type Objects []Object

314
fs/lister.go
View file

@ -1,314 +0,0 @@
// This file implements the Lister object
package fs
import "sync"
// listerResult is returned by the lister methods
type listerResult struct {
Obj Object
Dir *Dir
Err error
}
// Lister objects are used for controlling listing of Fs objects
type Lister struct {
mu sync.RWMutex
buffer int
abort bool
results chan listerResult
closeOnce sync.Once
level int
filter *Filter
err error
}
// NewLister creates a Lister object.
//
// The default channel buffer size will be Config.Checkers unless
// overridden with SetBuffer. The default level will be infinite.
func NewLister() *Lister {
o := &Lister{}
return o.SetLevel(-1).SetBuffer(Config.Checkers)
}
// Finds and lists the files passed in
//
// Note we ignore the dir and just return all the files in the list
func (o *Lister) listFiles(f ListFser, dir string, files FilesMap) {
buffer := o.Buffer()
jobs := make(chan string, buffer)
var wg sync.WaitGroup
// Start some listing go routines so we find those name in parallel
wg.Add(buffer)
for i := 0; i < buffer; i++ {
go func() {
defer wg.Done()
for remote := range jobs {
obj, err := f.NewObject(remote)
if err == ErrorObjectNotFound {
// silently ignore files that aren't found in the files list
} else if err != nil {
o.SetError(err)
} else {
o.Add(obj)
}
}
}()
}
// Pump the names in
for name := range files {
jobs <- name
if o.IsFinished() {
break
}
}
close(jobs)
wg.Wait()
// Signal that this listing is over
o.Finished()
}
// Start starts a go routine listing the Fs passed in. It returns the
// same Lister that was passed in for convenience.
func (o *Lister) Start(f ListFser, dir string) *Lister {
o.results = make(chan listerResult, o.buffer)
if o.filter != nil && o.filter.Files() != nil {
go o.listFiles(f, dir, o.filter.Files())
} else {
go f.List(o, dir)
}
return o
}
// SetLevel sets the level to recurse to. It returns same Lister that
// was passed in for convenience. If Level is < 0 then it sets it to
// infinite. Must be called before Start().
func (o *Lister) SetLevel(level int) *Lister {
if level < 0 {
o.level = MaxLevel
} else {
o.level = level
}
return o
}
// SetFilter sets the Filter that is in use. It defaults to no
// filtering. Must be called before Start().
func (o *Lister) SetFilter(filter *Filter) *Lister {
o.filter = filter
return o
}
// Level gets the recursion level for this listing.
//
// Fses may ignore this, but should implement it for improved efficiency if possible.
//
// Level 1 means list just the contents of the directory
//
// Each returned item must have less than level `/`s in.
func (o *Lister) Level() int {
return o.level
}
// SetBuffer sets the channel buffer size in use. Must be called
// before Start().
func (o *Lister) SetBuffer(buffer int) *Lister {
if buffer < 1 {
buffer = 1
}
o.buffer = buffer
return o
}
// Buffer gets the channel buffer size in use
func (o *Lister) Buffer() int {
return o.buffer
}
// Add an object to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (o *Lister) Add(obj Object) (abort bool) {
o.mu.RLock()
defer o.mu.RUnlock()
if o.abort {
return true
}
o.results <- listerResult{Obj: obj}
return false
}
// AddDir will a directory to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (o *Lister) AddDir(dir *Dir) (abort bool) {
o.mu.RLock()
defer o.mu.RUnlock()
if o.abort {
return true
}
o.results <- listerResult{Dir: dir}
return false
}
// Error returns a globally application error that's been set on the Lister
// object.
func (o *Lister) Error() error {
o.mu.RLock()
defer o.mu.RUnlock()
return o.err
}
// IncludeDirectory returns whether this directory should be
// included in the listing (and recursed into or not).
func (o *Lister) IncludeDirectory(remote string) bool {
if o.filter == nil {
return true
}
return o.filter.IncludeDirectory(remote)
}
// finished closes the results channel and sets abort - must be called
// with o.mu held.
func (o *Lister) finished() {
o.closeOnce.Do(func() {
close(o.results)
o.abort = true
})
}
// SetError will set an error state, and will cause the listing to
// be aborted.
// Multiple goroutines can set the error state concurrently,
// but only the first will be returned to the caller.
func (o *Lister) SetError(err error) {
o.mu.Lock()
if err != nil && !o.abort {
o.err = err
o.results <- listerResult{Err: err}
o.finished()
}
o.mu.Unlock()
}
// Finished should be called when listing is finished
func (o *Lister) Finished() {
o.mu.Lock()
o.finished()
o.mu.Unlock()
}
// IsFinished returns whether the directory listing is finished or not
func (o *Lister) IsFinished() bool {
o.mu.RLock()
defer o.mu.RUnlock()
return o.abort
}
// Get an object from the listing.
// Will return either an object or a directory, never both.
// Will return (nil, nil, nil) when all objects have been returned.
func (o *Lister) Get() (Object, *Dir, error) {
select {
case r := <-o.results:
return r.Obj, r.Dir, r.Err
}
}
// GetAll gets all the objects and dirs from the listing.
func (o *Lister) GetAll() (objs []Object, dirs []*Dir, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, nil, err
case obj != nil:
objs = append(objs, obj)
case dir != nil:
dirs = append(dirs, dir)
default:
return objs, dirs, nil
}
}
}
// GetObject will return an object from the listing.
// It will skip over any directories.
// Will return (nil, nil) when all objects have been returned.
func (o *Lister) GetObject() (Object, error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
return obj, nil
case dir != nil:
// ignore
default:
return nil, nil
}
}
}
// GetObjects will return a slice of object from the listing.
// It will skip over any directories.
func (o *Lister) GetObjects() (objs []Object, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
objs = append(objs, obj)
case dir != nil:
// ignore
default:
return objs, nil
}
}
}
// GetDir will return a directory from the listing.
// It will skip over any objects.
// Will return (nil, nil) when all objects have been returned.
func (o *Lister) GetDir() (*Dir, error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
// ignore
case dir != nil:
return dir, nil
default:
return nil, nil
}
}
}
// GetDirs will return a slice of directories from the listing.
// It will skip over any objects.
func (o *Lister) GetDirs() (dirs []*Dir, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
// ignore
case dir != nil:
dirs = append(dirs, dir)
default:
return dirs, nil
}
}
}
// Check interface
var _ ListOpts = (*Lister)(nil)

384
fs/lister_test.go
View file

@ -1,384 +0,0 @@
package fs
import (
"io"
"sort"
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestListerNew(t *testing.T) {
o := NewLister()
assert.Equal(t, Config.Checkers, o.buffer)
assert.Equal(t, false, o.abort)
assert.Equal(t, MaxLevel, o.level)
}
var errNotImpl = errors.New("not implemented")
type mockObject string
func (o mockObject) String() string { return string(o) }
func (o mockObject) Fs() Info { return nil }
func (o mockObject) Remote() string { return string(o) }
func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl }
func (o mockObject) ModTime() (t time.Time) { return t }
func (o mockObject) Size() int64 { return 0 }
func (o mockObject) Storable() bool { return true }
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl }
func (o mockObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) error {
return errNotImpl
}
func (o mockObject) Remove() error { return errNotImpl }
type mockFs struct {
listFn func(o ListOpts, dir string)
}
func (f *mockFs) List(o ListOpts, dir string) {
defer o.Finished()
f.listFn(o, dir)
}
func (f *mockFs) NewObject(remote string) (Object, error) {
return mockObject(remote), nil
}
func TestListerStart(t *testing.T) {
f := &mockFs{}
ranList := false
f.listFn = func(o ListOpts, dir string) {
ranList = true
}
o := NewLister().Start(f, "")
objs, dirs, err := o.GetAll()
require.Nil(t, err)
assert.Len(t, objs, 0)
assert.Len(t, dirs, 0)
assert.Equal(t, true, ranList)
}
func TestListerStartWithFiles(t *testing.T) {
f := &mockFs{}
ranList := false
f.listFn = func(o ListOpts, dir string) {
ranList = true
}
filter, err := NewFilter()
require.NoError(t, err)
wantNames := []string{"potato", "sausage", "rutabaga", "carrot", "lettuce"}
sort.Strings(wantNames)
for _, name := range wantNames {
err = filter.AddFile(name)
require.NoError(t, err)
}
o := NewLister().SetFilter(filter).Start(f, "")
objs, dirs, err := o.GetAll()
require.Nil(t, err)
assert.Len(t, dirs, 0)
assert.Equal(t, false, ranList)
var gotNames []string
for _, obj := range objs {
gotNames = append(gotNames, obj.Remote())
}
sort.Strings(gotNames)
assert.Equal(t, wantNames, gotNames)
}
func TestListerSetLevel(t *testing.T) {
o := NewLister()
o.SetLevel(1)
assert.Equal(t, 1, o.level)
o.SetLevel(0)
assert.Equal(t, 0, o.level)
o.SetLevel(-1)
assert.Equal(t, MaxLevel, o.level)
}
func TestListerSetFilter(t *testing.T) {
filter := &Filter{}
o := NewLister().SetFilter(filter)
assert.Equal(t, filter, o.filter)
}
func TestListerLevel(t *testing.T) {
o := NewLister()
assert.Equal(t, MaxLevel, o.Level())
o.SetLevel(123)
assert.Equal(t, 123, o.Level())
}
func TestListerSetBuffer(t *testing.T) {
o := NewLister()
o.SetBuffer(2)
assert.Equal(t, 2, o.buffer)
o.SetBuffer(1)
assert.Equal(t, 1, o.buffer)
o.SetBuffer(0)
assert.Equal(t, 1, o.buffer)
o.SetBuffer(-1)
assert.Equal(t, 1, o.buffer)
}
func TestListerBuffer(t *testing.T) {
o := NewLister()
assert.Equal(t, Config.Checkers, o.Buffer())
o.SetBuffer(123)
assert.Equal(t, 123, o.Buffer())
}
func TestListerAdd(t *testing.T) {
f := &mockFs{}
objs := []Object{
mockObject("1"),
mockObject("2"),
}
f.listFn = func(o ListOpts, dir string) {
for _, obj := range objs {
assert.Equal(t, false, o.Add(obj))
}
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
require.Nil(t, err)
assert.Equal(t, objs, gotObjs)
assert.Len(t, gotDirs, 0)
}
func TestListerAddDir(t *testing.T) {
f := &mockFs{}
dirs := []*Dir{
&Dir{Name: "1"},
&Dir{Name: "2"},
}
f.listFn = func(o ListOpts, dir string) {
for _, dir := range dirs {
assert.Equal(t, false, o.AddDir(dir))
}
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
require.Nil(t, err)
assert.Len(t, gotObjs, 0)
assert.Equal(t, dirs, gotDirs)
}
func TestListerIncludeDirectory(t *testing.T) {
o := NewLister()
assert.Equal(t, true, o.IncludeDirectory("whatever"))
filter, err := NewFilter()
require.Nil(t, err)
require.NotNil(t, filter)
require.Nil(t, filter.AddRule("!"))
require.Nil(t, filter.AddRule("+ potato/*"))
require.Nil(t, filter.AddRule("- *"))
o.SetFilter(filter)
assert.Equal(t, false, o.IncludeDirectory("floop"))
assert.Equal(t, true, o.IncludeDirectory("potato"))
assert.Equal(t, false, o.IncludeDirectory("potato/sausage"))
}
func TestListerSetError(t *testing.T) {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.Add(mockObject("1")))
o.SetError(errNotImpl)
assert.Equal(t, true, o.Add(mockObject("2")))
o.SetError(errors.New("not signalled"))
assert.Equal(t, true, o.AddDir(&Dir{Name: "2"}))
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, gotObjs)
assert.Nil(t, gotDirs)
}
func TestListerIsFinished(t *testing.T) {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.IsFinished())
o.Finished()
assert.Equal(t, true, o.IsFinished())
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
assert.Nil(t, err)
assert.Len(t, gotObjs, 0)
assert.Len(t, gotDirs, 0)
}
func testListerGet(t *testing.T) *Lister {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.Add(mockObject("1")))
assert.Equal(t, false, o.AddDir(&Dir{Name: "2"}))
}
return NewLister().Start(f, "")
}
func TestListerGet(t *testing.T) {
o := testListerGet(t)
obj, dir, err := o.Get()
assert.Nil(t, err)
assert.Equal(t, obj.Remote(), "1")
assert.Nil(t, dir)
obj, dir, err = o.Get()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Equal(t, dir.Name, "2")
obj, dir, err = o.Get()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObject(t *testing.T) {
o := testListerGet(t)
obj, err := o.GetObject()
assert.Nil(t, err)
assert.Equal(t, obj.Remote(), "1")
obj, err = o.GetObject()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDir(t *testing.T) {
o := testListerGet(t)
dir, err := o.GetDir()
assert.Nil(t, err)
assert.Equal(t, dir.Name, "2")
dir, err = o.GetDir()
assert.Nil(t, err)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func testListerGetError(t *testing.T) *Lister {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
o.SetError(errNotImpl)
}
return NewLister().Start(f, "")
}
func TestListerGetError(t *testing.T) {
o := testListerGetError(t)
obj, dir, err := o.Get()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, obj)
assert.Nil(t, dir)
obj, dir, err = o.Get()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObjectError(t *testing.T) {
o := testListerGetError(t)
obj, err := o.GetObject()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, obj)
obj, err = o.GetObject()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDirError(t *testing.T) {
o := testListerGetError(t)
dir, err := o.GetDir()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, dir)
dir, err = o.GetDir()
assert.Nil(t, err)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func testListerGetAll(t *testing.T) (*Lister, []Object, []*Dir) {
objs := []Object{
mockObject("1f"),
mockObject("2f"),
mockObject("3f"),
}
dirs := []*Dir{
&Dir{Name: "1d"},
&Dir{Name: "2d"},
}
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.Add(objs[0]))
assert.Equal(t, false, o.Add(objs[1]))
assert.Equal(t, false, o.AddDir(dirs[0]))
assert.Equal(t, false, o.Add(objs[2]))
assert.Equal(t, false, o.AddDir(dirs[1]))
}
return NewLister().Start(f, ""), objs, dirs
}
func TestListerGetAll(t *testing.T) {
o, objs, dirs := testListerGetAll(t)
gotObjs, gotDirs, err := o.GetAll()
assert.Nil(t, err)
assert.Equal(t, objs, gotObjs)
assert.Equal(t, dirs, gotDirs)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObjects(t *testing.T) {
o, objs, _ := testListerGetAll(t)
gotObjs, err := o.GetObjects()
assert.Nil(t, err)
assert.Equal(t, objs, gotObjs)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDirs(t *testing.T) {
o, _, dirs := testListerGetAll(t)
gotDirs, err := o.GetDirs()
assert.Nil(t, err)
assert.Equal(t, dirs, gotDirs)
assert.Equal(t, true, o.IsFinished())
}
func testListerGetAllError(t *testing.T) *Lister {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
o.SetError(errNotImpl)
}
return NewLister().Start(f, "")
}
func TestListerGetAllError(t *testing.T) {
o := testListerGetAllError(t)
gotObjs, gotDirs, err := o.GetAll()
assert.Equal(t, errNotImpl, err)
assert.Len(t, gotObjs, 0)
assert.Len(t, gotDirs, 0)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObjectsError(t *testing.T) {
o := testListerGetAllError(t)
gotObjs, err := o.GetObjects()
assert.Equal(t, errNotImpl, err)
assert.Len(t, gotObjs, 0)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDirsError(t *testing.T) {
o := testListerGetAllError(t)
gotDirs, err := o.GetDirs()
assert.Equal(t, errNotImpl, err)
assert.Len(t, gotDirs, 0)
assert.Equal(t, true, o.IsFinished())
}

46
fs/operations.go
View file

@ -595,41 +595,41 @@ func (ds DirEntries) ForDirError(fn func(dir *Dir) error) error {
// dir is the start directory, "" for root
//
// If includeAll is specified all files will be added, otherwise only
// files passing the filter will be added.
// files and directories passing the filter will be added.
//
// Files will be returned in sorted order
func ListDirSorted(fs Fs, includeAll bool, dir string) (entries DirEntries, err error) {
list := NewLister().SetLevel(1)
// Get unfiltered entries from the fs
entries, err = fs.List(dir)
if err != nil {
return nil, err
}
// filter the entries if required
if !includeAll {
list.SetFilter(Config.Filter)
}
list.Start(fs, dir)
for {
o, dir, err := list.Get()
if err != nil {
return nil, err
} else if o != nil {
newEntries := make(DirEntries, 0, len(entries))
for _, entry := range entries {
switch x := entry.(type) {
case Object:
// Make sure we don't delete excluded files if not required
if includeAll || Config.Filter.IncludeObject(o) {
entries = append(entries, o)
if Config.Filter.IncludeObject(x) {
newEntries = append(newEntries, entry)
} else {
Debugf(o, "Excluded from sync (and deletion)")
Debugf(x, "Excluded from sync (and deletion)")
}
} else if dir != nil {
if includeAll || Config.Filter.IncludeDirectory(dir.Remote()) {
entries = append(entries, dir)
case *Dir:
if Config.Filter.IncludeDirectory(x.Remote()) {
newEntries = append(newEntries, entry)
} else {
Debugf(dir, "Excluded from sync (and deletion)")
Debugf(x, "Excluded from sync (and deletion)")
}
} else {
// finishd since err, o, dir == nil
break
default:
return nil, errors.Errorf("unknown object type %T", entry)
}
}
err = list.Error()
if err != nil {
return nil, err
entries = newEntries
}
// sort the directory entries by Remote
sort.Sort(entries)
return entries, nil

109
fs/walk.go
View file

@ -71,6 +71,10 @@ func WalkN(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error
// It implements Walk using recursive directory listing if
// available, or returns ErrorCantListR if not.
func WalkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
listR := f.Features().ListR
if listR == nil {
return ErrorCantListR
}
return walkR(f, path, includeAll, maxLevel, fn, listR)
}
@ -254,57 +258,12 @@ func (dt DirTree) String() string {
return out.String()
}
type listRCallback func(entries DirEntries) error
type listRFunc func(f Fs, dir string, callback listRCallback) error
// FIXME Pretend ListR function
func listR(f Fs, dir string, callback listRCallback) (err error) {
listR := f.Features().ListR
if listR == nil {
return ErrorCantListR
}
const maxEntries = 100
entries := make(DirEntries, 0, maxEntries)
list := NewLister()
list.Start(f, dir)
for {
o, dir, err := list.Get()
if err != nil {
return err
} else if o != nil {
entries = append(entries, o)
} else if dir != nil {
entries = append(entries, dir)
} else {
// finishd since err, o, dir == nil
break
}
if len(entries) >= maxEntries {
err = callback(entries)
if err != nil {
return err
}
entries = entries[:0]
}
}
err = list.Error()
if err != nil {
return err
}
if len(entries) > 0 {
err = callback(entries)
if err != nil {
return err
}
}
return nil
}
func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listRFn listRFunc) (DirTree, error) {
func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) {
dirs := make(DirTree)
err := listRFn(f, path, func(entries DirEntries) error {
var mu sync.Mutex
err := listR(path, func(entries DirEntries) error {
mu.Lock()
defer mu.Unlock()
for _, entry := range entries {
slashes := strings.Count(entry.Remote(), "/")
switch x := entry.(type) {
@ -352,13 +311,19 @@ func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listRFn list
return dirs, nil
}
// NewDirTree returns a DirTree filled with the directory listing using the parameters supplied
// NewDirTree returns a DirTree filled with the directory listing
// using the parameters supplied. This will return ErrorCantListR for
// remotes which don't support ListR.
func NewDirTree(f Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
listR := f.Features().ListR
if listR == nil {
return nil, ErrorCantListR
}
return walkRDirTree(f, path, includeAll, maxLevel, listR)
}
func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listRFn listRFunc) error {
dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listRFn)
func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listR ListRFn) error {
dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR)
if err != nil {
return err
}
@ -410,3 +375,41 @@ func WalkGetAll(f Fs, path string, includeAll bool, maxLevel int) (objs []Object
})
return
}
// ListRHelper is used in the implementation of ListR to accumulate DirEntries
type ListRHelper struct {
callback ListRCallback
entries DirEntries
}
// NewListRHelper should be called from ListR with the callback passed in
func NewListRHelper(callback ListRCallback) *ListRHelper {
return &ListRHelper{
callback: callback,
}
}
// send sends the stored entries to the callback if there are >= max
// entries.
func (lh *ListRHelper) send(max int) (err error) {
if len(lh.entries) >= max {
err = lh.callback(lh.entries)
lh.entries = lh.entries[:0]
}
return err
}
// Add an entry to the stored entries and send them if there are more
// than a certain amount
func (lh *ListRHelper) Add(entry BasicInfo) error {
if entry == nil {
return nil
}
lh.entries = append(lh.entries, entry)
return lh.send(100)
}
// Flush the stored entries (if any) sending them to the callback
func (lh *ListRHelper) Flush() error {
return lh.send(1)
}

28
fs/walk_test.go
View file

@ -2,8 +2,10 @@ package fs
import (
"fmt"
"io"
"sync"
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
@ -34,6 +36,24 @@ type (
}
)
var errNotImpl = errors.New("not implemented")
type mockObject string
func (o mockObject) String() string { return string(o) }
func (o mockObject) Fs() Info { return nil }
func (o mockObject) Remote() string { return string(o) }
func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl }
func (o mockObject) ModTime() (t time.Time) { return t }
func (o mockObject) Size() int64 { return 0 }
func (o mockObject) Storable() bool { return true }
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl }
func (o mockObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) error {
return errNotImpl
}
func (o mockObject) Remove() error { return errNotImpl }
func newListDirs(t *testing.T, f Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs {
return &listDirs{
t: t,
@ -82,11 +102,9 @@ func (ls *listDirs) ListDir(f Fs, includeAll bool, dir string) (entries DirEntri
}
// ListR returns the expected listing for the directory using ListR
func (ls *listDirs) ListR(f Fs, dir string, callback listRCallback) (err error) {
func (ls *listDirs) ListR(dir string, callback ListRCallback) (err error) {
ls.mu.Lock()
defer ls.mu.Unlock()
assert.Equal(ls.t, ls.fs, f)
//assert.Equal(ls.t, ls.includeAll, includeAll)
var errorReturn error
for dirPath, result := range ls.results {
@ -392,8 +410,8 @@ func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
// a very simple listRcallback function
func makeListRCallback(entries DirEntries, err error) listRFunc {
return func(f Fs, dir string, callback listRCallback) error {
func makeListRCallback(entries DirEntries, err error) ListRFn {
return func(dir string, callback ListRCallback) error {
if err == nil {
err = callback(entries)
}

45
fstest/fstests/fstests.go
View file

@ -99,6 +99,20 @@ func skipIfNotOk(t *testing.T) {
}
}
// Skip if remote is not ListR capable, otherwise set the useListR
// flag, returning a function to restore its value
func skipIfNotListR(t *testing.T) func() {
skipIfNotOk(t)
if remote.Features().ListR == nil {
t.Skip("FS has no ListR interface")
}
previous := fs.Config.UseListR
fs.Config.UseListR = true
return func() {
fs.Config.UseListR = previous
}
}
// TestFsString tests the String method
func TestFsString(t *testing.T) {
skipIfNotOk(t)
@ -187,6 +201,12 @@ func TestFsListDirEmpty(t *testing.T) {
assert.Equal(t, []string{}, dirsToNames(dirs))
}
// TestFsListRDirEmpty tests listing the directories from an empty directory using ListR
func TestFsListRDirEmpty(t *testing.T) {
defer skipIfNotListR(t)()
TestFsListDirEmpty(t)
}
// TestFsNewObjectNotFound tests not finding a object
func TestFsNewObjectNotFound(t *testing.T) {
skipIfNotOk(t)
@ -340,6 +360,13 @@ func TestFsListDirFile2(t *testing.T) {
}
}
// TestFsListRDirFile2 tests the files are correctly uploaded by doing
// Depth 1 directory listings using ListR
func TestFsListRDirFile2(t *testing.T) {
defer skipIfNotListR(t)()
TestFsListDirFile2(t)
}
// TestFsListDirRoot tests that DirList works in the root
func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t)
@ -350,6 +377,12 @@ func TestFsListDirRoot(t *testing.T) {
assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
}
// TestFsListRDirRoot tests that DirList works in the root using ListR
func TestFsListRDirRoot(t *testing.T) {
defer skipIfNotListR(t)()
TestFsListDirRoot(t)
}
// TestFsListSubdir tests List works for a subdirectory
func TestFsListSubdir(t *testing.T) {
skipIfNotOk(t)
@ -372,6 +405,12 @@ func TestFsListSubdir(t *testing.T) {
require.Len(t, dirs, 0)
}
// TestFsListRSubdir tests List works for a subdirectory using ListR
func TestFsListRSubdir(t *testing.T) {
defer skipIfNotListR(t)()
TestFsListSubdir(t)
}
// TestFsListLevel2 tests List works for 2 levels
func TestFsListLevel2(t *testing.T) {
skipIfNotOk(t)
@ -384,6 +423,12 @@ func TestFsListLevel2(t *testing.T) {
assert.Equal(t, []string{`hello_ sausage`, `hello_ sausage/êé`}, dirsToNames(dirs))
}
// TestFsListRLevel2 tests List works for 2 levels using ListR
func TestFsListRLevel2(t *testing.T) {
defer skipIfNotListR(t)()
TestFsListLevel2(t)
}
// TestFsListFile1 tests file present
func TestFsListFile1(t *testing.T) {
skipIfNotOk(t)

47
ftp/ftp.go
View file

@ -296,18 +296,25 @@ func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
return nil, fs.ErrorObjectNotFound
}
func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
c, err := f.getFtpConnection()
if err != nil {
out.SetError(errors.Wrap(err, "list"))
return
return nil, errors.Wrap(err, "list")
}
files, err := c.List(path.Join(f.root, dir))
f.putFtpConnection(&c, err)
if err != nil {
out.SetError(translateErrorDir(err))
return
return nil, translateErrorDir(err)
}
for i := range files {
object := files[i]
@ -317,20 +324,13 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
if object.Name == "." || object.Name == ".." {
continue
}
if out.IncludeDirectory(newremote) {
d := &fs.Dir{
Name: newremote,
When: object.Time,
Bytes: 0,
Count: -1,
}
if curlevel < out.Level() {
f.list(out, path.Join(dir, object.Name), curlevel+1)
}
if out.AddDir(d) {
return
}
}
entries = append(entries, d)
default:
o := &Object{
fs: f,
@ -342,27 +342,10 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
ModTime: object.Time,
}
o.info = info
if out.Add(o) {
return
entries = append(entries, o)
}
}
}
}
// List the objects and directories of the Fs starting from dir
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
//
// Fses must support recursion levels of fs.MaxLevel and 1.
// They may return ErrorLevelNotSupported otherwise.
func (f *Fs) List(out fs.ListOpts, dir string) {
// defer fs.Trace(dir, "")("")
f.list(out, dir, 1)
out.Finished()
return entries, nil
}
// Hashes are not supported

5
ftp/ftp_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

129
googlecloudstorage/googlecloudstorage.go
View file

@ -308,27 +308,28 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
//
// dir is the starting directory, "" for root
//
// If directories is set it only sends directories
func (f *Fs) list(dir string, level int, fn listFn) error {
// Set recurse to read sub directories
func (f *Fs) list(dir string, recurse bool, fn listFn) error {
root := f.root
rootLength := len(root)
if dir != "" {
root += dir + "/"
}
list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks)
switch level {
case 1:
if !recurse {
list = list.Delimiter("/")
case fs.MaxLevel:
default:
return fs.ErrorLevelNotSupported
}
for {
objects, err := list.Do()
if err != nil {
if gErr, ok := err.(*googleapi.Error); ok {
if gErr.Code == http.StatusNotFound {
err = fs.ErrorDirNotFound
}
}
return err
}
if level == 1 {
if !recurse {
var object storage.Object
for _, prefix := range objects.Prefixes {
if !strings.HasSuffix(prefix, "/") {
@ -359,94 +360,120 @@ func (f *Fs) list(dir string, level int, fn listFn) error {
return nil
}
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.bucket == "" {
out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
return
}
// List the objects
err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error {
// Convert a list item into a BasicInfo
func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory bool) (fs.BasicInfo, error) {
if isDirectory {
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: int64(object.Size),
Count: 0,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
return d, nil
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return nil, err
}
return o, nil
}
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
// List the objects
err = f.list(dir, false, func(remote string, object *storage.Object, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
if entry != nil {
entries = append(entries, entry)
}
return nil
})
if err != nil {
if gErr, ok := err.(*googleapi.Error); ok {
if gErr.Code == http.StatusNotFound {
err = fs.ErrorDirNotFound
}
}
out.SetError(err)
return nil, err
}
return entries, err
}
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
defer out.Finished()
// listBuckets lists the buckets
func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
return nil, fs.ErrorListBucketRequired
}
if f.projectNumber == "" {
out.SetError(errors.New("can't list buckets without project number"))
return
return nil, errors.New("can't list buckets without project number")
}
listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
for {
buckets, err := listBuckets.Do()
if err != nil {
out.SetError(err)
return
return nil, err
}
for _, bucket := range buckets.Items {
dir := &fs.Dir{
d := &fs.Dir{
Name: bucket.Name,
Bytes: 0,
Count: 0,
}
if out.AddDir(dir) {
return
}
entries = append(entries, d)
}
if buckets.NextPageToken == "" {
break
}
listBuckets.PageToken(buckets.NextPageToken)
}
return entries, nil
}
// List lists the path to out
func (f *Fs) List(out fs.ListOpts, dir string) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
if f.bucket == "" {
f.listBuckets(out, dir)
} else {
f.listFiles(out, dir)
return f.listBuckets(dir)
}
return
return f.listDir(dir)
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(out fs.ListOpts, dir string) {
f.List(out, dir) // FIXME
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if f.bucket == "" {
return fs.ErrorListBucketRequired
}
list := fs.NewListRHelper(callback)
err = f.list(dir, true, func(remote string, object *storage.Object, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory)
if err != nil {
return err
}
return list.Add(entry)
})
if err != nil {
return err
}
return list.Flush()
}
// Put the object into the bucket

5
googlecloudstorage/googlecloudstorage_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

5
hubic/hubic_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

118
local/local.go
View file

@ -181,26 +181,32 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, "", nil)
}
// listArgs is the arguments that a new list takes
type listArgs struct {
remote string
dirpath string
level int
}
// list traverses the directory passed in, listing to out.
// it returns a boolean whether it is finished or not.
func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (subdirs []listArgs) {
fd, err := os.Open(dirpath)
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
dir = f.dirNames.Load(dir)
fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
remote := f.cleanRemote(dir)
_, err = os.Stat(fsDirPath)
if err != nil {
out.SetError(errors.Wrapf(err, "failed to open directory %q", dirpath))
return nil
return nil, fs.ErrorDirNotFound
}
fd, err := os.Open(fsDirPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to open directory %q", dir)
}
defer func() {
err := fd.Close()
if err != nil {
out.SetError(errors.Wrapf(err, "failed to close directory %q:", dirpath))
cerr := fd.Close()
if cerr != nil && err == nil {
err = errors.Wrapf(cerr, "failed to close directory %q:", dir)
}
}()
@ -210,106 +216,46 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su
break
}
if err != nil {
out.SetError(errors.Wrapf(err, "failed to read directory %q", dirpath))
return nil
return nil, errors.Wrapf(err, "failed to read directory %q", dir)
}
for _, fi := range fis {
name := fi.Name()
mode := fi.Mode()
newRemote := path.Join(remote, name)
newPath := filepath.Join(dirpath, name)
newPath := filepath.Join(fsDirPath, name)
// Follow symlinks if required
if *followSymlinks && (mode&os.ModeSymlink) != 0 {
fi, err = os.Stat(newPath)
if err != nil {
out.SetError(err)
return nil
return nil, err
}
mode = fi.Mode()
}
if fi.IsDir() {
// Ignore directories which are symlinks. These are junction points under windows which
// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
if (mode&os.ModeSymlink) == 0 && out.IncludeDirectory(newRemote) && f.dev == readDevice(fi) {
dir := &fs.Dir{
if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi) {
d := &fs.Dir{
Name: f.dirNames.Save(newRemote, f.cleanRemote(newRemote)),
When: fi.ModTime(),
Bytes: 0,
Count: 0,
}
if out.AddDir(dir) {
return nil
}
if level > 0 {
subdirs = append(subdirs, listArgs{remote: newRemote, dirpath: newPath, level: level - 1})
}
entries = append(entries, d)
}
} else {
fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
if err != nil {
out.SetError(err)
return nil
return nil, err
}
if fso.Storable() && out.Add(fso) {
return nil
if fso.Storable() {
entries = append(entries, fso)
}
}
}
}
return subdirs
}
// List the path into out
//
// Ignores everything which isn't Storable, eg links etc
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
dir = f.dirNames.Load(dir)
root := f.cleanPath(filepath.Join(f.root, dir))
dir = f.cleanRemote(dir)
_, err := os.Stat(root)
if err != nil {
out.SetError(fs.ErrorDirNotFound)
return
}
in := make(chan listArgs, out.Buffer())
var wg sync.WaitGroup // sync closing of go routines
var traversing sync.WaitGroup // running directory traversals
// Start the process
traversing.Add(1)
in <- listArgs{remote: dir, dirpath: root, level: out.Level() - 1}
for i := 0; i < fs.Config.Checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for job := range in {
if out.IsFinished() {
continue
}
newJobs := f.list(out, job.remote, job.dirpath, job.level)
// Now we have traversed this directory, send
// these ones off for traversal
if len(newJobs) != 0 {
traversing.Add(len(newJobs))
go func() {
for _, newJob := range newJobs {
in <- newJob
}
}()
}
traversing.Done()
}
}()
}
// Wait for traversal to finish
traversing.Wait()
close(in)
wg.Wait()
return entries, nil
}
// cleanRemote makes string a valid UTF-8 string for remote strings.

5
local/local_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

57
onedrive/onedrive.go
View file

@ -399,50 +399,57 @@ OUTER:
return
}
// ListDir reads the directory specified by the job into out, returning any more jobs
func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) {
fs.Debugf(f, "Reading %q", job.Path)
_, err = f.listAll(job.DirID, false, false, func(info *api.Item) bool {
remote := job.Path + info.Name
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
err = f.dirCache.FindRoot(false)
if err != nil {
return nil, err
}
directoryID, err := f.dirCache.FindDir(dir, false)
if err != nil {
return nil, err
}
var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool {
remote := path.Join(dir, info.Name)
if info.Folder != nil {
if out.IncludeDirectory(remote) {
// cache the directory ID for later lookups
f.dirCache.Put(remote, info.ID)
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
When: time.Time(info.LastModifiedDateTime),
}
if info.Folder != nil {
dir.Count = info.Folder.ChildCount
}
if out.AddDir(dir) {
return true
}
if job.Depth > 0 {
jobs = append(jobs, dircache.ListDirJob{DirID: info.ID, Path: remote + "/", Depth: job.Depth - 1})
}
d.Count = info.Folder.ChildCount
}
entries = append(entries, d)
} else {
o, err := f.newObjectWithInfo(remote, info)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
iErr = err
return true
}
entries = append(entries, o)
}
return false
})
fs.Debugf(f, "Finished reading %q", job.Path)
return jobs, err
if err != nil {
return nil, err
}
// List walks the path returning files and directories into out
func (f *Fs) List(out fs.ListOpts, dir string) {
f.dirCache.List(f, out, dir)
if iErr != nil {
return nil, iErr
}
return entries, nil
}
// Creates from the parameters passed in a half finished Object which

5
onedrive/onedrive_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

125
s3/s3.go
View file

@ -470,20 +470,16 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error
//
// dir is the starting directory, "" for root
//
// Level is the level of the recursion
func (f *Fs) list(dir string, level int, fn listFn) error {
// Set recurse to read sub directories
func (f *Fs) list(dir string, recurse bool, fn listFn) error {
root := f.root
if dir != "" {
root += dir + "/"
}
maxKeys := int64(listChunkSize)
delimiter := ""
switch level {
case 1:
if !recurse {
delimiter = "/"
case fs.MaxLevel:
default:
return fs.ErrorLevelNotSupported
}
var marker *string
for {
@ -497,10 +493,15 @@ func (f *Fs) list(dir string, level int, fn listFn) error {
}
resp, err := f.c.ListObjects(&req)
if err != nil {
if awsErr, ok := err.(awserr.RequestFailure); ok {
if awsErr.StatusCode() == http.StatusNotFound {
err = fs.ErrorDirNotFound
}
}
return err
}
rootLength := len(f.root)
if level == 1 {
if !recurse {
for _, commonPrefix := range resp.CommonPrefixes {
if commonPrefix.Prefix == nil {
fs.Logf(f, "Nil common prefix received")
@ -546,90 +547,116 @@ func (f *Fs) list(dir string, level int, fn listFn) error {
return nil
}
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.bucket == "" {
// Return no objects at top level list
out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
return
}
// List the objects and directories
err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error {
// Convert a list item into a BasicInfo
func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool) (fs.BasicInfo, error) {
if isDirectory {
size := int64(0)
if object.Size != nil {
size = *object.Size
}
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: size,
Count: 0,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
return d, nil
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return nil, err
}
return o, nil
}
// listDir lists files and directories to out
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
// List the objects and directories
err = f.list(dir, false, func(remote string, object *s3.Object, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
if entry != nil {
entries = append(entries, entry)
}
return nil
})
if err != nil {
if awsErr, ok := err.(awserr.RequestFailure); ok {
if awsErr.StatusCode() == http.StatusNotFound {
err = fs.ErrorDirNotFound
}
}
out.SetError(err)
return nil, err
}
return entries, nil
}
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
defer out.Finished()
func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
return nil, fs.ErrorListBucketRequired
}
req := s3.ListBucketsInput{}
resp, err := f.c.ListBuckets(&req)
if err != nil {
out.SetError(err)
return
return nil, err
}
for _, bucket := range resp.Buckets {
dir := &fs.Dir{
d := &fs.Dir{
Name: aws.StringValue(bucket.Name),
When: aws.TimeValue(bucket.CreationDate),
Bytes: -1,
Count: -1,
}
if out.AddDir(dir) {
break
}
entries = append(entries, d)
}
return entries, nil
}
// List lists files and directories to out
func (f *Fs) List(out fs.ListOpts, dir string) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
if f.bucket == "" {
f.listBuckets(out, dir)
} else {
f.listFiles(out, dir)
return f.listBuckets(dir)
}
return
return f.listDir(dir)
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(out fs.ListOpts, dir string) {
f.List(out, dir) // FIXME
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if f.bucket == "" {
return fs.ErrorListBucketRequired
}
list := fs.NewListRHelper(callback)
err = f.list(dir, true, func(remote string, object *s3.Object, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory)
if err != nil {
return err
}
return list.Add(entry)
})
if err != nil {
return err
}
return list.Flush()
}
// Put the Object into the bucket

5
s3/s3_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

73
sftp/sftp.go
View file

@ -8,7 +8,6 @@ import (
"io"
"os"
"path"
"sync"
"time"
"github.com/ncw/rclone/fs"
@ -219,74 +218,52 @@ func (f *Fs) dirExists(dir string) (bool, error) {
return true, nil
}
func (f *Fs) list(out fs.ListOpts, dir string, level int, wg *sync.WaitGroup, tokens chan struct{}) {
defer wg.Done()
// take a token
<-tokens
// return it when done
defer func() {
tokens <- struct{}{}
}()
sftpDir := path.Join(f.root, dir)
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
root := path.Join(f.root, dir)
ok, err := f.dirExists(root)
if err != nil {
return nil, errors.Wrap(err, "List failed")
}
if !ok {
return nil, fs.ErrorDirNotFound
}
sftpDir := root
if sftpDir == "" {
sftpDir = "."
}
infos, err := f.sftpClient.ReadDir(sftpDir)
if err != nil {
err = errors.Wrapf(err, "error listing %q", dir)
fs.Errorf(f, "Listing failed: %v", err)
out.SetError(err)
return
return nil, errors.Wrapf(err, "error listing %q", dir)
}
for _, info := range infos {
remote := path.Join(dir, info.Name())
if info.IsDir() {
if out.IncludeDirectory(remote) {
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
When: info.ModTime(),
Bytes: -1,
Count: -1,
}
out.AddDir(dir)
if level < out.Level() {
wg.Add(1)
go f.list(out, remote, level+1, wg, tokens)
}
}
entries = append(entries, d)
} else {
file := &Object{
o := &Object{
fs: f,
remote: remote,
info: info,
}
out.Add(file)
entries = append(entries, o)
}
}
}
// List the files and directories starting at <dir>
func (f *Fs) List(out fs.ListOpts, dir string) {
root := path.Join(f.root, dir)
ok, err := f.dirExists(root)
if err != nil {
out.SetError(errors.Wrap(err, "List failed"))
return
}
if !ok {
out.SetError(fs.ErrorDirNotFound)
return
}
// tokens to control the concurrency
tokens := make(chan struct{}, fs.Config.Checkers)
for i := 0; i < fs.Config.Checkers; i++ {
tokens <- struct{}{}
}
wg := new(sync.WaitGroup)
wg.Add(1)
f.list(out, dir, 1, wg, tokens)
wg.Wait()
out.Finished()
return entries, nil
}
// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime()>

5
sftp/sftp_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

124
swift/swift.go
View file

@ -273,8 +273,8 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error
// listContainerRoot lists the objects into the function supplied from
// the container and root supplied
//
// Level is the level of the recursion
func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn listFn) error {
// Set recurse to read sub directories
func (f *Fs) listContainerRoot(container, root string, dir string, recurse bool, fn listFn) error {
prefix := root
if dir != "" {
prefix += dir + "/"
@ -284,12 +284,8 @@ func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn
Prefix: prefix,
Limit: listChunks,
}
switch level {
case 1:
if !recurse {
opts.Delimiter = '/'
case fs.MaxLevel:
default:
return fs.ErrorLevelNotSupported
}
rootLength := len(root)
return f.c.ObjectsWalk(container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) {
@ -298,7 +294,7 @@ func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn
for i := range objects {
object := &objects[i]
isDirectory := false
if level == 1 {
if !recurse {
if strings.HasSuffix(object.Name, "/") {
isDirectory = true
object.Name = object.Name[:len(object.Name)-1]
@ -319,29 +315,18 @@ func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn
})
}
// list the objects into the function supplied
func (f *Fs) list(dir string, level int, fn listFn) error {
return f.listContainerRoot(f.container, f.root, dir, level, fn)
}
type addEntryFn func(fs.BasicInfo) error
// listFiles walks the path returning a channel of Objects
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.container == "" {
out.SetError(errors.New("can't list objects at root - choose a container using lsd"))
return
}
// List the objects
err := f.list(dir, out.Level(), func(remote string, object *swift.Object, isDirectory bool) error {
// list the objects into the function supplied
func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error {
return f.listContainerRoot(f.container, f.root, dir, recurse, func(remote string, object *swift.Object, isDirectory bool) (err error) {
if isDirectory {
dir := &fs.Dir{
d := &fs.Dir{
Name: remote,
Bytes: object.Bytes,
Count: 0,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
err = fn(d)
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
@ -349,59 +334,96 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
}
// Storable does a full metadata read on 0 size objects which might be dynamic large objects
if o.Storable() {
if out.Add(o) {
return fs.ErrorListAborted
err = fn(o)
}
}
return err
})
}
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if f.container == "" {
return nil, fs.ErrorListBucketRequired
}
// List the objects
err = f.list(dir, false, func(entry fs.BasicInfo) error {
entries = append(entries, entry)
return nil
})
if err != nil {
if err == swift.ContainerNotFound {
err = fs.ErrorDirNotFound
}
out.SetError(err)
return nil, err
}
return entries, nil
}
// listContainers lists the containers
func (f *Fs) listContainers(out fs.ListOpts, dir string) {
defer out.Finished()
func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) {
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
return nil, fs.ErrorListBucketRequired
}
containers, err := f.c.ContainersAll(nil)
if err != nil {
out.SetError(err)
return
return nil, errors.Wrap(err, "container listing failed")
}
for _, container := range containers {
dir := &fs.Dir{
d := &fs.Dir{
Name: container.Name,
Bytes: container.Bytes,
Count: container.Count,
}
if out.AddDir(dir) {
break
}
entries = append(entries, d)
}
return entries, nil
}
// List walks the path returning files and directories to out
func (f *Fs) List(out fs.ListOpts, dir string) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
if f.container == "" {
f.listContainers(out, dir)
} else {
f.listFiles(out, dir)
return f.listContainers(dir)
}
return
return f.listDir(dir)
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(out fs.ListOpts, dir string) {
f.List(out, dir) // FIXME
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if f.container == "" {
return errors.New("container needed for recursive list")
}
list := fs.NewListRHelper(callback)
err = f.list(dir, true, func(entry fs.BasicInfo) error {
return list.Add(entry)
})
if err != nil {
return err
}
return list.Flush()
}
// Put the object into the container
@ -471,12 +493,8 @@ func (f *Fs) Purge() error {
go func() {
delErr <- fs.DeleteFiles(toBeDeleted)
}()
err := f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
if !isDirectory {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
err := f.list("", true, func(entry fs.BasicInfo) error {
if o, ok := entry.(*Object); ok {
toBeDeleted <- o
}
return nil
@ -679,7 +697,7 @@ func min(x, y int64) int64 {
// if except is passed in then segments with that prefix won't be deleted
func (o *Object) removeSegments(except string) error {
segmentsRoot := o.fs.root + o.remote + "/"
err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", true, func(remote string, object *swift.Object, isDirectory bool) error {
if isDirectory {
return nil
}

5
swift/swift_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

146
yandex/yandex.go
View file

@ -166,11 +166,43 @@ func (f *Fs) setRoot(root string) {
f.diskRoot = diskRoot
}
// listFn is called from list and listContainerRoot to handle an object.
type listFn func(remote string, item *yandex.ResourceInfoResponse, isDirectory bool) error
// Convert a list item into a BasicInfo
func (f *Fs) itemToDirEntry(remote string, object *yandex.ResourceInfoResponse) (fs.BasicInfo, error) {
switch object.ResourceType {
case "dir":
t, err := time.Parse(time.RFC3339Nano, object.Modified)
if err != nil {
return nil, errors.Wrap(err, "error parsing time in directory item")
}
d := &fs.Dir{
Name: remote,
When: t,
Bytes: int64(object.Size),
Count: -1,
}
return d, nil
case "file":
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return nil, err
}
return o, nil
default:
fs.Debugf(f, "Unknown resource type %q", object.ResourceType)
}
return nil, nil
}
// listDir lists this directory only returning objects and directories
func (f *Fs) listDir(dir string, fn listFn) (err error) {
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
//request object meta info
var opt yandex.ResourceInfoRequestOptions
root := f.diskRoot
@ -189,30 +221,22 @@ func (f *Fs) listDir(dir string, fn listFn) (err error) {
if err != nil {
yErr, ok := err.(yandex.DiskClientError)
if ok && yErr.Code == "DiskNotFoundError" {
return fs.ErrorDirNotFound
return nil, fs.ErrorDirNotFound
}
return err
return nil, err
}
itemsCount = uint32(len(ResourceInfoResponse.Embedded.Items))
if ResourceInfoResponse.ResourceType == "dir" {
//list all subdirs
for i, element := range ResourceInfoResponse.Embedded.Items {
for _, element := range ResourceInfoResponse.Embedded.Items {
remote := path.Join(dir, element.Name)
fs.Debugf(i, "%q", remote)
switch element.ResourceType {
case "dir":
err = fn(remote, &element, true)
entry, err := f.itemToDirEntry(remote, &element)
if err != nil {
return err
return nil, err
}
case "file":
err = fn(remote, &element, false)
if err != nil {
return err
}
default:
fs.Debugf(f, "Unknown resource type %q", element.ResourceType)
if entry != nil {
entries = append(entries, entry)
}
}
}
@ -224,13 +248,26 @@ func (f *Fs) listDir(dir string, fn listFn) (err error) {
break
}
}
return nil
return entries, nil
}
// list the objects into the function supplied
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// This does a flat listing of all the files in the drive
func (f *Fs) list(dir string, fn listFn) error {
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
//request files list. list is divided into pages. We send request for each page
//items per page is limited by limit
//TODO may be add config parameter for the items per page limit
@ -259,17 +296,26 @@ func (f *Fs) list(dir string, fn listFn) error {
itemsCount = uint32(len(info.Items))
//list files
entries := make(fs.DirEntries, 0, len(info.Items))
for _, item := range info.Items {
// filter file list and get only files we need
if strings.HasPrefix(item.Path, prefix) {
//trim root folder from filename
var name = strings.TrimPrefix(item.Path, f.diskRoot)
err = fn(name, &item, false)
entry, err := f.itemToDirEntry(name, &item)
if err != nil {
return err
}
if entry != nil {
entries = append(entries, entry)
}
}
}
// send the listing
err = callback(entries)
if err != nil {
return err
}
//offset for the next page of items
offset += itemsCount
@ -281,57 +327,6 @@ func (f *Fs) list(dir string, fn listFn) error {
return nil
}
// List walks the path returning a channel of Objects
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
listItem := func(remote string, object *yandex.ResourceInfoResponse, isDirectory bool) error {
if isDirectory {
t, err := time.Parse(time.RFC3339Nano, object.Modified)
if err != nil {
return err
}
dir := &fs.Dir{
Name: remote,
When: t,
Bytes: int64(object.Size),
Count: -1,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
}
var err error
switch out.Level() {
case 1:
err = f.listDir(dir, listItem)
case fs.MaxLevel:
err = f.list(dir, listItem)
default:
out.SetError(fs.ErrorLevelNotSupported)
}
if err != nil {
out.SetError(err)
}
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(out fs.ListOpts, dir string) {
f.List(out, dir) // FIXME
}
// NewObject finds the Object at remote. If it can't be found it
// returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
@ -657,6 +652,7 @@ var (
_ fs.Purger = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
//_ fs.Copier = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
)

5
yandex/yandex_test.go
View file

@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }