From fd51f24906ac247d41cdf687d0183767e3b375ed Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 2 Oct 2019 11:23:10 +0100 Subject: [PATCH] putio: use lib/encoder And in the process - fix a bug with + and & in file name - fix NewObject returning directories as files --- backend/putio/fs.go | 16 ++++++++-------- backend/putio/object.go | 8 +++++++- backend/putio/putio.go | 14 ++++++++++++++ docs/content/putio.md | 12 ++++++++++++ fs/encodings/encodings.go | 10 ++++++++++ fs/encodings/encodings_noencode.go | 3 +++ 6 files changed, 54 insertions(+), 9 deletions(-) diff --git a/backend/putio/fs.go b/backend/putio/fs.go index 4139a4746..8de5eff5a 100644 --- a/backend/putio/fs.go +++ b/backend/putio/fs.go @@ -145,7 +145,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, var entry putio.File err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID) - entry, err = f.client.Files.CreateFolder(ctx, leaf, parentID) + entry, err = f.client.Files.CreateFolder(ctx, enc.FromStandardName(leaf), parentID) return shouldRetry(err) }) return itoa(entry.ID), err @@ -172,11 +172,11 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin return } for _, child := range children { - if child.Name == leaf { + if enc.ToStandardName(child.Name) == leaf { found = true pathIDOut = itoa(child.ID) if !child.IsDir() { - err = fs.ErrorNotAFile + err = fs.ErrorIsFile } return } @@ -214,7 +214,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return } for _, child := range children { - remote := path.Join(dir, child.Name) + remote := path.Join(dir, enc.ToStandardName(child.Name)) // fs.Debugf(f, "child: %s", remote) if child.IsDir() { f.dirCache.Put(remote, itoa(child.ID)) @@ -292,7 +292,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext req.Header.Set("tus-resumable", "1.0.0") req.Header.Set("upload-length", strconv.FormatInt(size, 10)) - b64name := base64.StdEncoding.EncodeToString([]byte(name)) + b64name := base64.StdEncoding.EncodeToString([]byte(enc.FromStandardName(name))) b64true := base64.StdEncoding.EncodeToString([]byte("true")) b64parentID := base64.StdEncoding.EncodeToString([]byte(parentID)) b64modifiedAt := base64.StdEncoding.EncodeToString([]byte(modTime.Format(time.RFC3339))) @@ -505,7 +505,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec params := url.Values{} params.Set("file_id", strconv.FormatInt(srcObj.file.ID, 10)) params.Set("parent_id", directoryID) - params.Set("name", leaf) + params.Set("name", enc.FromStandardName(leaf)) req, err := f.client.NewRequest(ctx, "POST", "/v2/files/copy", strings.NewReader(params.Encode())) if err != nil { return false, err @@ -544,7 +544,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec params := url.Values{} params.Set("file_id", strconv.FormatInt(srcObj.file.ID, 10)) params.Set("parent_id", directoryID) - params.Set("name", leaf) + params.Set("name", enc.FromStandardName(leaf)) req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode())) if err != nil { return false, err @@ -633,7 +633,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string params := url.Values{} params.Set("file_id", srcID) params.Set("parent_id", dstDirectoryID) - params.Set("name", leaf) + params.Set("name", enc.FromStandardName(leaf)) req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode())) if err != nil { return false, err diff --git a/backend/putio/object.go b/backend/putio/object.go index 550557b55..0ab042e8c 100644 --- a/backend/putio/object.go +++ b/backend/putio/object.go @@ -137,7 +137,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) { } err = o.fs.pacer.Call(func() (bool, error) { // fs.Debugf(o, "requesting child. directoryID: %s, name: %s", directoryID, leaf) - req, err := o.fs.client.NewRequest(ctx, "GET", "/v2/files/"+directoryID+"/child?name="+url.PathEscape(leaf), nil) + req, err := o.fs.client.NewRequest(ctx, "GET", "/v2/files/"+directoryID+"/child?name="+url.QueryEscape(enc.FromStandardName(leaf)), nil) if err != nil { return false, err } @@ -147,6 +147,12 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) { } return shouldRetry(err) }) + if err != nil { + return nil, err + } + if resp.File.IsDir() { + return nil, fs.ErrorNotAFile + } return &resp.File, err } diff --git a/backend/putio/putio.go b/backend/putio/putio.go index e56bae2bb..6fa23447b 100644 --- a/backend/putio/putio.go +++ b/backend/putio/putio.go @@ -8,11 +8,25 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/oauthutil" "golang.org/x/oauth2" ) +/* +// TestPutio +stringNeedsEscaping = []rune{ + '/', '\x00' +} +maxFileLength = 255 +canWriteUnnormalized = true +canReadUnnormalized = true +canReadRenormalized = true +canStream = false +*/ +const enc = encodings.Putio + // Constants const ( rcloneClientID = "4131" diff --git a/docs/content/putio.md b/docs/content/putio.md index dc357ac96..cc4761189 100644 --- a/docs/content/putio.md +++ b/docs/content/putio.md @@ -96,5 +96,17 @@ To copy a local directory to a put.io directory called backup rclone copy /home/source remote:backup +#### Restricted filename characters + +In addition to the [default restricted characters set](/overview/#restricted-characters) +the following characters are also replaced: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| \ | 0x5C | ＼ | + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in JSON strings. + diff --git a/fs/encodings/encodings.go b/fs/encodings/encodings.go index 3ee5f5882..d28c70c03 100644 --- a/fs/encodings/encodings.go +++ b/fs/encodings/encodings.go @@ -238,6 +238,16 @@ const Pcloud = encoder.MultiEncoder( encoder.EncodeBackSlash | encoder.EncodeInvalidUtf8) +// Putio is the encoding used by the putio backend +// +// Note that \ is renamed to - +// +// Encode invalid UTF-8 bytes as json doesn't handle them properly. +const Putio = encoder.MultiEncoder( + uint(Display) | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8) + // Fichier is the encoding used by the fichier backend // // Characters that need escaping diff --git a/fs/encodings/encodings_noencode.go b/fs/encodings/encodings_noencode.go index b283d8232..6f29e5966 100644 --- a/fs/encodings/encodings_noencode.go +++ b/fs/encodings/encodings_noencode.go @@ -29,8 +29,11 @@ const ( OneDrive = Base OpenDrive = Base Pcloud = Base + PremiumizeMe = Base + Putio = Base QingStor = Base S3 = Base + Sharefile = Base Swift = Base )