From dac20093c57127851f63606bc61f63213d997034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Nov 2018 13:14:19 +0100 Subject: [PATCH] onedrive: use lib/encoder --- backend/onedrive/onedrive.go | 47 +++++++++++------ backend/onedrive/replace.go | 91 -------------------------------- backend/onedrive/replace_test.go | 30 ----------- docs/content/onedrive.md | 37 +++++++++++++ 4 files changed, 67 insertions(+), 138 deletions(-) delete mode 100644 backend/onedrive/replace.go delete mode 100644 backend/onedrive/replace_test.go diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 45c98a435..c23af9b21 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -15,8 +15,6 @@ import ( "strings" "time" - "github.com/rclone/rclone/lib/atexit" - "github.com/pkg/errors" "github.com/rclone/rclone/backend/onedrive/api" "github.com/rclone/rclone/fs" @@ -24,8 +22,10 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/oauthutil" "github.com/rclone/rclone/lib/pacer" @@ -34,6 +34,8 @@ import ( "golang.org/x/oauth2" ) +const enc = encodings.OneDrive + const ( rcloneClientID = "b15665d9-eda6-4092-8539-0eec376afd59" rcloneEncryptedClientSecret = "_JUdzh3LnKNqSPcf4Wu5fgMFIQOI8glZu_akYgR8yf6egowNBg-R" @@ -345,7 +347,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) { // "shared with me" folders in OneDrive Personal (See #2536, #2778) // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480 func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) { - opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath)))) + opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(enc.FromStandardPath(relPath)))) err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) @@ -368,7 +370,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It } else { opts = rest.Opts{ Method: "GET", - Path: "/root:/" + rest.URLPathEscape(replaceReservedChars(path)), + Path: "/root:/" + rest.URLPathEscape(enc.FromStandardPath(path)), } } err = f.pacer.Call(func() (bool, error) { @@ -616,7 +618,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e var info *api.Item opts := newOptsCall(dirID, "POST", "/children") mkdir := api.CreateItemRequest{ - Name: replaceReservedChars(leaf), + Name: enc.FromStandardName(leaf), ConflictBehavior: "fail", } err = f.pacer.Call(func() (bool, error) { @@ -676,7 +678,7 @@ OUTER: if item.Deleted != nil { continue } - item.Name = restoreReservedChars(item.GetName()) + item.Name = enc.ToStandardName(item.GetName()) if fn(item) { found = true break OUTER @@ -913,8 +915,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, return nil, err } - srcPath := srcObj.fs.rootSlash() + srcObj.remote - dstPath := f.rootSlash() + remote + srcPath := srcObj.rootPath() + dstPath := f.rootPath(remote) if strings.ToLower(srcPath) == strings.ToLower(dstPath) { return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) } @@ -932,7 +934,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, id, dstDriveID, _ := parseNormalizedID(directoryID) - replacedLeaf := replaceReservedChars(leaf) + replacedLeaf := enc.FromStandardName(leaf) copyReq := api.CopyItemRequest{ Name: &replacedLeaf, ParentReference: api.ItemReference{ @@ -1016,7 +1018,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, opts := newOptsCall(srcObj.id, "PATCH", "") move := api.MoveItemRequest{ - Name: replaceReservedChars(leaf), + Name: enc.FromStandardName(leaf), ParentReference: &api.ItemReference{ DriveID: dstDriveID, ID: id, @@ -1131,7 +1133,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string // Do the move opts := newOptsCall(srcID, "PATCH", "") move := api.MoveItemRequest{ - Name: replaceReservedChars(leaf), + Name: enc.FromStandardName(leaf), ParentReference: &api.ItemReference{ DriveID: dstDriveID, ID: parsedDstDirID, @@ -1197,7 +1199,7 @@ func (f *Fs) Hashes() hash.Set { // PublicLink returns a link for downloading without accout. func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) { - info, _, err := f.readMetaDataForPath(ctx, f.srvPath(remote)) + info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote)) if err != nil { return "", err } @@ -1241,9 +1243,19 @@ func (o *Object) Remote() string { return o.remote } +// rootPath returns a path for use in server given a remote +func (f *Fs) rootPath(remote string) string { + return f.rootSlash() + remote +} + +// rootPath returns a path for use in local functions +func (o *Object) rootPath() string { + return o.fs.rootPath(o.remote) +} + // srvPath returns a path for use in server given a remote func (f *Fs) srvPath(remote string) string { - return replaceReservedChars(f.rootSlash() + remote) + return enc.FromStandardPath(f.rootSlash() + remote) } // srvPath returns a path for use in server @@ -1320,7 +1332,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { if o.hasMetaData { return nil } - info, _, err := o.fs.readMetaDataForPath(ctx, o.srvPath()) + info, _, err := o.fs.readMetaDataForPath(ctx, o.rootPath()) if err != nil { if apiErr, ok := err.(*api.Error); ok { if apiErr.ErrorInfo.Code == "itemNotFound" { @@ -1355,7 +1367,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, opts = rest.Opts{ Method: "PATCH", RootURL: rootURL, - Path: "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)), + Path: "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(enc.FromStandardName(leaf))), } } else { opts = rest.Opts{ @@ -1429,7 +1441,8 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re opts = rest.Opts{ Method: "POST", RootURL: rootURL, - Path: "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(replaceReservedChars(leaf)) + ":/createUploadSession", + Path: fmt.Sprintf("/%s/items/%s:/%s:/createUploadSession", + drive, id, rest.URLPathEscape(enc.FromStandardName(leaf))), } } else { opts = rest.Opts{ @@ -1581,7 +1594,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, opts = rest.Opts{ Method: "PUT", RootURL: rootURL, - Path: "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf) + ":/content", + Path: "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(enc.FromStandardName(leaf)) + ":/content", ContentLength: &size, Body: in, } diff --git a/backend/onedrive/replace.go b/backend/onedrive/replace.go deleted file mode 100644 index 1d38d56df..000000000 --- a/backend/onedrive/replace.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Translate file names for one drive - -OneDrive reserved characters - -The following characters are OneDrive reserved characters, and can't -be used in OneDrive folder and file names. - - onedrive-reserved = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" - onedrive-business-reserved - = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" / "#" / "%" - -Note: Folder names can't end with a period (.). - -Note: OneDrive for Business file or folder names cannot begin with a -tilde ('~'). - -*/ - -package onedrive - -import ( - "regexp" - "strings" -) - -// charMap holds replacements for characters -// -// Onedrive has a restricted set of characters compared to other cloud -// storage systems, so we to map these to the FULLWIDTH unicode -// equivalents -// -// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS -var ( - charMap = map[rune]rune{ - '\\': '＼', // FULLWIDTH REVERSE SOLIDUS - '*': '＊', // FULLWIDTH ASTERISK - '<': '＜', // FULLWIDTH LESS-THAN SIGN - '>': '＞', // FULLWIDTH GREATER-THAN SIGN - '?': '？', // FULLWIDTH QUESTION MARK - ':': '：', // FULLWIDTH COLON - '|': '｜', // FULLWIDTH VERTICAL LINE - '#': '＃', // FULLWIDTH NUMBER SIGN - '%': '％', // FULLWIDTH PERCENT SIGN - '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved - '.': '．', // FULLWIDTH FULL STOP - '~': '～', // FULLWIDTH TILDE - ' ': '␠', // SYMBOL FOR SPACE - } - invCharMap map[rune]rune - fixEndingInPeriod = regexp.MustCompile(`\.(/|$)`) - fixStartingWithTilde = regexp.MustCompile(`(/|^)~`) - fixStartingWithSpace = regexp.MustCompile(`(/|^) `) -) - -func init() { - // Create inverse charMap - invCharMap = make(map[rune]rune, len(charMap)) - for k, v := range charMap { - invCharMap[v] = k - } -} - -// replaceReservedChars takes a path and substitutes any reserved -// characters in it -func replaceReservedChars(in string) string { - // Folder names can't end with a period '.' - in = fixEndingInPeriod.ReplaceAllString(in, string(charMap['.'])+"$1") - // OneDrive for Business file or folder names cannot begin with a tilde '~' - in = fixStartingWithTilde.ReplaceAllString(in, "$1"+string(charMap['~'])) - // Apparently file names can't start with space either - in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) - // Replace reserved characters - return strings.Map(func(c rune) rune { - if replacement, ok := charMap[c]; ok && c != '.' && c != '~' && c != ' ' { - return replacement - } - return c - }, in) -} - -// restoreReservedChars takes a path and undoes any substitutions -// made by replaceReservedChars -func restoreReservedChars(in string) string { - return strings.Map(func(c rune) rune { - if replacement, ok := invCharMap[c]; ok { - return replacement - } - return c - }, in) -} diff --git a/backend/onedrive/replace_test.go b/backend/onedrive/replace_test.go deleted file mode 100644 index bac8a590e..000000000 --- a/backend/onedrive/replace_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package onedrive - -import "testing" - -func TestReplace(t *testing.T) { - for _, test := range []struct { - in string - out string - }{ - {"", ""}, - {"abc 123", "abc 123"}, - {`\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~`}, - {`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~/＼＊＜＞？：｜＃％＂.~`}, - {" leading space", "␠leading space"}, - {"~leading tilde", "～leading tilde"}, - {"trailing dot.", "trailing dot．"}, - {" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"}, - {"~leading tilde/~leading tilde/~leading tilde", "～leading tilde/～leading tilde/～leading tilde"}, - {"trailing dot./trailing dot./trailing dot.", "trailing dot．/trailing dot．/trailing dot．"}, - } { - got := replaceReservedChars(test.in) - if got != test.out { - t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) - } - got2 := restoreReservedChars(got) - if got2 != test.in { - t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) - } - } -} diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 242be0f42..55c72cafd 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -148,6 +148,43 @@ Sharepoint Server support For all types of OneDrive you can use the `--checksum` flag. +#### Restricted filename characters + +In addition to the [default restricted characters set](/overview/#restricted-characters) +the following characters are also replaced: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| " | 0x22 | ＂ | +| * | 0x2A | ＊ | +| : | 0x3A | ： | +| < | 0x3C | ＜ | +| > | 0x3E | ＞ | +| ? | 0x3F | ？ | +| \ | 0x5C | ＼ | +| \| | 0x7C | ｜ | +| # | 0x23 | ＃ | +| % | 0x25 | ％ | + +File names can also not end with the following characters. +These only get replaced if they are last character in the name: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| SP | 0x20 | ␠ | +| . | 0x2E | ． | + +File names can also not begin with the following characters. +These only get replaced if they are first character in the name: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| SP | 0x20 | ␠ | +| ~ | 0x7E | ～ | + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in JSON strings. + ### Deleting files ### Any files you delete with rclone will end up in the trash. Microsoft