From 4837bc354682283b4f351c80f8e157f6bd980400 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabian=20M=C3=B6ller?= <fabianm88@gmail.com>
Date: Fri, 2 Nov 2018 13:13:47 +0100
Subject: [PATCH] jottacloud: use lib/encoder
---
backend/jottacloud/jottacloud.go | 19 ++++----
backend/jottacloud/replace.go | 77 ------------------------------
backend/jottacloud/replace_test.go | 28 -----------
docs/content/jottacloud.md | 18 +++++++
fs/encodings/encodings.go | 3 ++
5 files changed, 32 insertions(+), 113 deletions(-)
delete mode 100644 backend/jottacloud/replace.go
delete mode 100644 backend/jottacloud/replace_test.go
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 0d365230b..36e18ebfa 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -26,6 +26,7 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
+ "github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@@ -36,6 +37,8 @@ import (
"golang.org/x/oauth2"
)
+const enc = encodings.JottaCloud
+
// Globals
const (
minSleep = 10 * time.Millisecond
@@ -460,7 +463,7 @@ func urlPathEscape(in string) string {
// filePathRaw returns an unescaped file path (f.root, file)
func (f *Fs) filePathRaw(file string) string {
- return path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file)))
+ return path.Join(f.endpointURL, enc.FromStandardPath(path.Join(f.root, file)))
}
// filePath returns a escaped file path (f.root, file)
@@ -673,7 +676,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if item.Deleted {
continue
}
- remote := path.Join(dir, restoreReservedChars(item.Name))
+ remote := path.Join(dir, enc.ToStandardName(item.Name))
d := fs.NewDir(remote, time.Time(item.ModifiedAt))
entries = append(entries, d)
}
@@ -683,7 +686,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if item.Deleted || item.State != "COMPLETED" {
continue
}
- remote := path.Join(dir, restoreReservedChars(item.Name))
+ remote := path.Join(dir, enc.ToStandardName(item.Name))
o, err := f.newObjectWithInfo(ctx, remote, item)
if err != nil {
continue
@@ -708,7 +711,7 @@ func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolde
if folder.Deleted {
return nil
}
- folderPath := restoreReservedChars(path.Join(folder.Path, folder.Name))
+ folderPath := enc.ToStandardPath(path.Join(folder.Path, folder.Name))
folderPathLength := len(folderPath)
var remoteDir string
if folderPathLength > pathPrefixLength {
@@ -726,7 +729,7 @@ func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolde
if file.Deleted || file.State != "COMPLETED" {
continue
}
- remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name))
+ remoteFile := path.Join(remoteDir, enc.ToStandardName(file.Name))
o, err := f.newObjectWithInfo(ctx, remoteFile, file)
if err != nil {
return err
@@ -897,7 +900,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
Parameters: url.Values{},
}
- opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, dest))))
+ opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, enc.FromStandardPath(path.Join(f.root, dest))))
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
@@ -1004,7 +1007,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return fs.ErrorDirExists
}
- _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote)
+ _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, enc.FromStandardPath(srcPath))+"/", dstRemote)
if err != nil {
return errors.Wrap(err, "couldn't move directory")
@@ -1295,7 +1298,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Created: fileDate,
Modified: fileDate,
Md5: md5String,
- Path: path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
+ Path: path.Join(o.fs.opt.Mountpoint, enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
}
// send it
diff --git a/backend/jottacloud/replace.go b/backend/jottacloud/replace.go
deleted file mode 100644
index 698726036..000000000
--- a/backend/jottacloud/replace.go
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
-Translate file names for JottaCloud adapted from OneDrive
-
-
-The following characters are JottaCloud reserved characters, and can't
-be used in JottaCloud folder and file names.
-
- jottacloud = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"
-
-
-*/
-
-package jottacloud
-
-import (
- "regexp"
- "strings"
-)
-
-// charMap holds replacements for characters
-//
-// Onedrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
- charMap = map[rune]rune{
- '\\': '＼', // FULLWIDTH REVERSE SOLIDUS
- '*': '＊', // FULLWIDTH ASTERISK
- '<': '＜', // FULLWIDTH LESS-THAN SIGN
- '>': '＞', // FULLWIDTH GREATER-THAN SIGN
- '?': '？', // FULLWIDTH QUESTION MARK
- ':': '：', // FULLWIDTH COLON
- ';': '；', // FULLWIDTH SEMICOLON
- '|': '｜', // FULLWIDTH VERTICAL LINE
- '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
- ' ': '␠', // SYMBOL FOR SPACE
- }
- invCharMap map[rune]rune
- fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
- fixEndingWithSpace = regexp.MustCompile(` (/|$)`)
-)
-
-func init() {
- // Create inverse charMap
- invCharMap = make(map[rune]rune, len(charMap))
- for k, v := range charMap {
- invCharMap[v] = k
- }
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
- // Filenames can't start with space
- in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
- // Filenames can't end with space
- in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
- return strings.Map(func(c rune) rune {
- if replacement, ok := charMap[c]; ok && c != ' ' {
- return replacement
- }
- return c
- }, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
- return strings.Map(func(c rune) rune {
- if replacement, ok := invCharMap[c]; ok {
- return replacement
- }
- return c
- }, in)
-}
diff --git a/backend/jottacloud/replace_test.go b/backend/jottacloud/replace_test.go
deleted file mode 100644
index b1f2979b8..000000000
--- a/backend/jottacloud/replace_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package jottacloud
-
-import "testing"
-
-func TestReplace(t *testing.T) {
- for _, test := range []struct {
- in string
- out string
- }{
- {"", ""},
- {"abc 123", "abc 123"},
- {`\*<>?:;|"`, `＼＊＜＞？：；｜＂`},
- {`\*<>?:;|"\*<>?:;|"`, `＼＊＜＞？：；｜＂＼＊＜＞？：；｜＂`},
- {" leading space", "␠leading space"},
- {"trailing space ", "trailing space␠"},
- {" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
- {"trailing space /trailing space /trailing space ", "trailing space␠/trailing space␠/trailing space␠"},
- } {
- got := replaceReservedChars(test.in)
- if got != test.out {
- t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
- }
- got2 := restoreReservedChars(got)
- if got2 != test.in {
- t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
- }
- }
-}
diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md
index 4a5df99ec..e6c7a1f2e 100644
--- a/docs/content/jottacloud.md
+++ b/docs/content/jottacloud.md
@@ -129,6 +129,24 @@ temporarily on disk (wherever the `TMPDIR` environment variable points
to) before it is uploaded. Small files will be cached in memory - see
the `--jottacloud-md5-memory-limit` flag.
+#### Restricted filename characters
+
+In addition to the [default restricted characters set](/overview/#restricted-characters)
+the following characters are also replaced:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| " | 0x22 | ＂ |
+| * | 0x2A | ＊ |
+| : | 0x3A | ： |
+| < | 0x3C | ＜ |
+| > | 0x3E | ＞ |
+| ? | 0x3F | ？ |
+| \| | 0x7C | ｜ |
+
+Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
+as they can't be used in XML strings.
+
### Deleting files ###
By default rclone will send all files to the trash when deleting files.
diff --git a/fs/encodings/encodings.go b/fs/encodings/encodings.go
index fefac77d4..f230c285b 100644
--- a/fs/encodings/encodings.go
+++ b/fs/encodings/encodings.go
@@ -110,8 +110,11 @@ const GoogleCloudStorage = encoder.MultiEncoder(
// JottaCloud is the encoding used by the jottacloud backend
//
// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
+//
+// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
const JottaCloud = encoder.MultiEncoder(
uint(Display) |
+ encoder.EncodeWin | // :?"*<>|
encoder.EncodeInvalidUtf8)
// Koofr is the encoding used by the koofr backend