From 4b9fdb8475e17ef79ec04d0cc8394ad9e4b0ffc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Nov 2018 13:14:30 +0100 Subject: [PATCH] opendrive: use lib/encoder --- backend/opendrive/opendrive.go | 28 +++++++---- backend/opendrive/replace.go | 78 ------------------------------- backend/opendrive/replace_test.go | 28 ----------- docs/content/opendrive.md | 30 ++++++++++++ 4 files changed, 50 insertions(+), 114 deletions(-) delete mode 100644 backend/opendrive/replace.go delete mode 100644 backend/opendrive/replace_test.go diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index e1fd43113..df1597362 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -16,6 +16,7 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -25,6 +26,8 @@ import ( "github.com/rclone/rclone/lib/rest" ) +const enc = encodings.OpenDrive + const ( defaultEndpoint = "https://dev.opendrive.com/api/v1" minSleep = 10 * time.Millisecond @@ -585,7 +588,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, fs: f, remote: remote, } - return o, leaf, directoryID, nil + return o, enc.FromStandardName(leaf), directoryID, nil } // readMetaDataForPath reads the metadata from the path @@ -636,7 +639,11 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . var resp *http.Response response := createFileResponse{} err := o.fs.pacer.Call(func() (bool, error) { - createFileData := createFile{SessionID: o.fs.session.SessionID, FolderID: directoryID, Name: replaceReservedChars(leaf)} + createFileData := createFile{ + SessionID: o.fs.session.SessionID, + FolderID: directoryID, + Name: leaf, + } opts := rest.Opts{ Method: "POST", Path: "/upload/create_file.json", @@ -683,7 +690,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err = f.pacer.Call(func() (bool, error) { createDirData := createFolder{ SessionID: f.session.SessionID, - FolderName: replaceReservedChars(leaf), + FolderName: enc.FromStandardName(leaf), FolderSubParent: pathID, FolderIsPublic: 0, FolderPublicUpl: 0, @@ -729,8 +736,8 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin return "", false, errors.Wrap(err, "failed to get folder list") } + leaf = enc.FromStandardName(leaf) for _, folder := range folderList.Folders { - folder.Name = restoreReservedChars(folder.Name) // fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID) if leaf == folder.Name { @@ -777,7 +784,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e } for _, folder := range folderList.Folders { - folder.Name = restoreReservedChars(folder.Name) + folder.Name = enc.ToStandardName(folder.Name) // fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID) remote := path.Join(dir, folder.Name) // cache the directory ID for later lookups @@ -788,7 +795,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e } for _, file := range folderList.Files { - file.Name = restoreReservedChars(file.Name) + file.Name = enc.ToStandardName(file.Name) // fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID) remote := path.Join(dir, file.Name) o, err := f.newObjectWithInfo(ctx, remote, &file) @@ -851,7 +858,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { NoResponse: true, Path: "/file/filesettings.json", } - update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)} + update := modTimeFile{ + SessionID: o.fs.session.SessionID, + FileID: o.id, + FileModificationTime: strconv.FormatInt(modTime.Unix(), 10), + } err := o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil) return o.fs.shouldRetry(resp, err) @@ -1038,7 +1049,8 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { err = o.fs.pacer.Call(func() (bool, error) { opts := rest.Opts{ Method: "GET", - Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)), + Path: fmt.Sprintf("/folder/itembyname.json/%s/%s?name=%s", + o.fs.session.SessionID, directoryID, url.QueryEscape(enc.FromStandardName(leaf))), } resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList) return o.fs.shouldRetry(resp, err) diff --git a/backend/opendrive/replace.go b/backend/opendrive/replace.go deleted file mode 100644 index 15dcef5b3..000000000 --- a/backend/opendrive/replace.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Translate file names for OpenDrive - -OpenDrive reserved characters - -The following characters are OpenDrive reserved characters, and can't -be used in OpenDrive folder and file names. - -\ / : * ? " < > | - -OpenDrive files and folders can't have leading or trailing spaces also. - -*/ - -package opendrive - -import ( - "regexp" - "strings" -) - -// charMap holds replacements for characters -// -// OpenDrive has a restricted set of characters compared to other cloud -// storage systems, so we to map these to the FULLWIDTH unicode -// equivalents -// -// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS -var ( - charMap = map[rune]rune{ - '\\': '＼', // FULLWIDTH REVERSE SOLIDUS - ':': '：', // FULLWIDTH COLON - '*': '＊', // FULLWIDTH ASTERISK - '?': '？', // FULLWIDTH QUESTION MARK - '"': '＂', // FULLWIDTH QUOTATION MARK - '<': '＜', // FULLWIDTH LESS-THAN SIGN - '>': '＞', // FULLWIDTH GREATER-THAN SIGN - '|': '｜', // FULLWIDTH VERTICAL LINE - ' ': '␠', // SYMBOL FOR SPACE - } - fixStartingWithSpace = regexp.MustCompile(`(/|^) `) - fixEndingWithSpace = regexp.MustCompile(` (/|$)`) - invCharMap map[rune]rune -) - -func init() { - // Create inverse charMap - invCharMap = make(map[rune]rune, len(charMap)) - for k, v := range charMap { - invCharMap[v] = k - } -} - -// replaceReservedChars takes a path and substitutes any reserved -// characters in it -func replaceReservedChars(in string) string { - // Filenames can't start with space - in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) - // Filenames can't end with space - in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1") - return strings.Map(func(c rune) rune { - if replacement, ok := charMap[c]; ok && c != ' ' { - return replacement - } - return c - }, in) -} - -// restoreReservedChars takes a path and undoes any substitutions -// made by replaceReservedChars -func restoreReservedChars(in string) string { - return strings.Map(func(c rune) rune { - if replacement, ok := invCharMap[c]; ok { - return replacement - } - return c - }, in) -} diff --git a/backend/opendrive/replace_test.go b/backend/opendrive/replace_test.go deleted file mode 100644 index 1c4978f6d..000000000 --- a/backend/opendrive/replace_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package opendrive - -import "testing" - -func TestReplace(t *testing.T) { - for _, test := range []struct { - in string - out string - }{ - {"", ""}, - {"abc 123", "abc 123"}, - {`\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~`}, - {`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~/＼＊＜＞？：｜#%＂.~`}, - {" leading space", "␠leading space"}, - {" path/ leading spaces", "␠path/␠ leading spaces"}, - {"trailing space ", "trailing space␠"}, - {"trailing spaces /path ", "trailing spaces ␠/path␠"}, - } { - got := replaceReservedChars(test.in) - if got != test.out { - t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) - } - got2 := restoreReservedChars(got) - if got2 != test.in { - t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) - } - } -} diff --git a/docs/content/opendrive.md b/docs/content/opendrive.md index 152e83fd6..ca1105c05 100644 --- a/docs/content/opendrive.md +++ b/docs/content/opendrive.md @@ -69,6 +69,36 @@ OpenDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not. +#### Restricted filename characters + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| NUL | 0x00 | ␀ | +| / | 0x2F | ／ | +| " | 0x22 | ＂ | +| * | 0x2A | ＊ | +| : | 0x3A | ： | +| < | 0x3C | ＜ | +| > | 0x3E | ＞ | +| ? | 0x3F | ？ | +| \ | 0x5C | ＼ | +| \| | 0x7C | ｜ | + +File names can also not begin or end with the following characters. +These only get replaced if they are the first or last character in the name: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| SP | 0x20 | ␠ | +| HT | 0x09 | ␉ | +| LF | 0x0A | ␊ | +| VT | 0x0B | ␋ | +| CR | 0x0D | ␍ | + + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in JSON strings. + ### Standard Options