jottacloud: add --fast-list support - fixes #2532

This commit is contained in:
albertony 2018-09-06 15:13:38 +02:00 committed by Nick Craig-Wood
parent a1f935e815
commit 3fccce625c
4 changed files with 112 additions and 4 deletions

View file

@ -25,6 +25,7 @@ import (
"github.com/ncw/rclone/fs/fserrors"
"github.com/ncw/rclone/fs/fshttp"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/fs/walk"
"github.com/ncw/rclone/lib/pacer"
"github.com/ncw/rclone/lib/rest"
"github.com/pkg/errors"
@ -227,9 +228,14 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
// filePathRaw returns an unescaped file path (f.root, file)
func (f *Fs) filePathRaw(file string) string {
return path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file)))
}
// filePath returns a escaped file path (f.root, file)
func (f *Fs) filePath(file string) string {
return rest.URLPathEscape(path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file))))
return rest.URLPathEscape(f.filePathRaw(file))
}
// filePath returns a escaped file path (f.root, remote)
@ -425,6 +431,98 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
return entries, nil
}
// listFileDirFn is called from listFileDir to handle an object.
type listFileDirFn func(fs.DirEntry) error
// List the objects and directories into entries, from a
// special kind of JottaFolder representing a FileDirLis
func (f *Fs) listFileDir(rootPath string, root *api.JottaFolder, fn listFileDirFn) error {
rootLen := len(rootPath)
for i := range root.Folders {
folder := &root.Folders[i]
if folder.Deleted {
return nil
}
folderPath := path.Join(folder.Path, folder.Name)
var remoteDir string
subLen := len(folderPath) - rootLen
if subLen > 0 {
remoteDir = restoreReservedChars(folderPath[rootLen+1:])
d := fs.NewDir(remoteDir, time.Time(folder.ModifiedAt))
err := fn(d)
if err != nil {
return err
}
}
for i := range folder.Files {
file := &folder.Files[i]
if file.Deleted || file.State != "COMPLETED" {
continue
}
remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name))
o, err := f.newObjectWithInfo(remoteFile, file)
if err != nil {
return err
}
err = fn(o)
if err != nil {
return err
}
}
}
return nil
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
opts := rest.Opts{
Method: "GET",
Path: f.filePath(dir),
Parameters: url.Values{},
}
opts.Parameters.Set("mode", "list")
var resp *http.Response
var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result)
return shouldRetry(resp, err)
})
if err != nil {
if apiErr, ok := err.(*api.Error); ok {
// does not exist
if apiErr.StatusCode == http.StatusNotFound {
return fs.ErrorDirNotFound
}
}
return errors.Wrap(err, "couldn't list files")
}
rootPath := "/" + f.filePathRaw(dir)
list := walk.NewListRHelper(callback)
err = f.listFileDir(rootPath, &result, func(entry fs.DirEntry) error {
return list.Add(entry)
})
if err != nil {
return err
}
return list.Flush()
}
// Creates from the parameters passed in a half finished Object which
// must have setMetaData called on it
//
@ -666,7 +764,7 @@ func (f *Fs) About() (*fs.Usage, error) {
}
usage := &fs.Usage{
Used: fs.NewUsageValue(info.Usage),
Used: fs.NewUsageValue(info.Usage),
}
if info.Capacity > 0 {
usage.Total = fs.NewUsageValue(info.Capacity)
@ -932,6 +1030,7 @@ var (
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = (*Object)(nil)

View file

@ -81,6 +81,15 @@ To copy a local directory to an Jottacloud directory called backup
rclone copy /home/source remote:backup
### --fast-list ###
This remote supports `--fast-list` which allows you to use fewer
transactions in exchange for more memory. See the [rclone
docs](/docs/#fast-list) for more details.
Note that the implementation in Jottacloud always uses only a single
API request to get the entire list, so for large folders this could
lead to long wait time before the first results are shown.
### Modified time and hashes ###

View file

@ -138,7 +138,7 @@ operations more efficient.
| Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
| HTTP | No | No | No | No | No | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
| Jottacloud | Yes | Yes | Yes | Yes | No | No | No | No | No |
| Jottacloud | Yes | Yes | Yes | Yes | No | Yes | No | No | No |
| Mega | Yes | No | Yes | Yes | No | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Microsoft OneDrive | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |

View file

@ -75,7 +75,7 @@ var (
{
Name: "TestJottacloud:",
SubDir: false,
FastList: false,
FastList: true,
},
{
Name: "TestOneDrive:",