From 334f19c97440975e07f5f427f26ac9a324fa65e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Nov 2018 13:12:09 +0100 Subject: [PATCH] info: improve allowed character testing --- cmd/info/info.go | 123 +++++++++++++++++++++++++++++++++++--------- cmd/info/process.sh | 40 ++++++++++++++ cmd/info/test.cmd | 3 ++ cmd/info/test.sh | 43 ++++++++++++++++ 4 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 cmd/info/process.sh create mode 100644 cmd/info/test.cmd create mode 100755 cmd/info/test.sh diff --git a/cmd/info/info.go b/cmd/info/info.go index 02e6ad3eb..b4aad3444 100644 --- a/cmd/info/info.go +++ b/cmd/info/info.go @@ -21,11 +21,22 @@ import ( "github.com/spf13/cobra" ) +type position int + +const ( + positionMiddle position = 1 << iota + positionLeft + positionRight + positionNone position = 0 + positionAll position = positionRight<<1 - 1 +) + var ( checkNormalization bool checkControl bool checkLength bool checkStreaming bool + positionList = []position{positionMiddle, positionLeft, positionRight} ) func init() { @@ -59,7 +70,7 @@ a bit of go code for each one. type results struct { f fs.Fs mu sync.Mutex - charNeedsEscaping map[rune]bool + stringNeedsEscaping map[string]position maxFileLength int canWriteUnnormalized bool canReadUnnormalized bool @@ -69,8 +80,8 @@ type results struct { func newResults(f fs.Fs) *results { return &results{ - f: f, - charNeedsEscaping: make(map[rune]bool), + f: f, + stringNeedsEscaping: make(map[string]position), } } @@ -79,13 +90,13 @@ func (r *results) Print() { fmt.Printf("// %s\n", r.f.Name()) if checkControl { escape := []string{} - for c, needsEscape := range r.charNeedsEscaping { - if needsEscape { + for c, needsEscape := range r.stringNeedsEscaping { + if needsEscape != positionNone { escape = append(escape, fmt.Sprintf("0x%02X", c)) } } sort.Strings(escape) - fmt.Printf("charNeedsEscaping = []byte{\n") + fmt.Printf("stringNeedsEscaping = []byte{\n") fmt.Printf("\t%s\n", strings.Join(escape, ", ")) fmt.Printf("}\n") } @@ -130,20 +141,45 @@ func (r *results) checkUTF8Normalization() { } } -// check we can write file with the rune passed in -func (r *results) checkChar(c rune) { - fs.Infof(r.f, "Writing file 0x%02X", c) - path := fmt.Sprintf("0x%02X-%c-", c, c) - _, err := r.writeFile(path) - escape := false - if err != nil { - fs.Infof(r.f, "Couldn't write file 0x%02X", c) - escape = true - } else { - fs.Infof(r.f, "OK writing file 0x%02X", c) +func (r *results) checkStringPositions(s string) { + fs.Infof(r.f, "Writing position file 0x%0X", s) + positionError := positionNone + + for _, pos := range positionList { + path := "" + switch pos { + case positionMiddle: + path = fmt.Sprintf("position-middle-%0X-%s-", s, s) + case positionLeft: + path = fmt.Sprintf("%s-position-left-%0X", s, s) + case positionRight: + path = fmt.Sprintf("position-right-%0X-%s", s, s) + default: + panic("invalid position: " + pos.String()) + } + _, writeErr := r.writeFile(path) + if writeErr != nil { + fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr) + } else { + fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s) + } + obj, getErr := r.f.NewObject(path) + if getErr != nil { + fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr) + } else { + if obj.Size() != 50 { + fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size()) + } else { + fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s) + } + } + if writeErr != nil || getErr != nil { + positionError += pos + } } + r.mu.Lock() - r.charNeedsEscaping[c] = escape + r.stringNeedsEscaping[s] = positionError r.mu.Unlock() } @@ -157,19 +193,28 @@ func (r *results) checkControls() { } var wg sync.WaitGroup for i := rune(0); i < 128; i++ { + s := string(i) if i == 0 || i == '/' { // We're not even going to check NULL or / - r.charNeedsEscaping[i] = true + r.stringNeedsEscaping[s] = positionAll continue } wg.Add(1) - c := i - go func() { + go func(s string) { defer wg.Done() token := <-tokens - r.checkChar(c) + r.checkStringPositions(s) tokens <- token - }() + }(s) + } + for _, s := range []string{"＼", "\xBF", "\xFE"} { + wg.Add(1) + go func(s string) { + defer wg.Done() + token := <-tokens + r.checkStringPositions(s) + tokens <- token + }(s) } wg.Wait() fs.Infof(r.f, "Done trying to create control character file names") @@ -268,3 +313,35 @@ func readInfo(f fs.Fs) error { r.Print() return nil } + +func (e position) String() string { + switch e { + case positionNone: + return "none" + case positionAll: + return "all" + } + var buf bytes.Buffer + if e&positionMiddle != 0 { + buf.WriteString("middle") + e &= ^positionMiddle + } + if e&positionLeft != 0 { + if buf.Len() != 0 { + buf.WriteRune(',') + } + buf.WriteString("left") + e &= ^positionLeft + } + if e&positionRight != 0 { + if buf.Len() != 0 { + buf.WriteRune(',') + } + buf.WriteString("right") + e &= ^positionRight + } + if e != positionNone { + panic("invalid position") + } + return buf.String() +} diff --git a/cmd/info/process.sh b/cmd/info/process.sh new file mode 100644 index 000000000..605e55e47 --- /dev/null +++ b/cmd/info/process.sh @@ -0,0 +1,40 @@ +set -euo pipefail + +for f in info-*.log; do + for pos in middle left right; do + egrep -oe " Writing $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.write_$pos + egrep -oe " Getting $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.get_$pos + done + { + echo "${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}" + echo "Write\tWrite\tWrite\tGet\tGet\tGet" + echo "Mid\tLeft\tRight\tMid\tLeft\tRight" + paste $f.write_{middle,left,right} $f.get_{middle,left,right} + } > $f.csv +done + +for f in info-*.list; do + for pos in middle left right; do + cat $f | perl -lne 'print $1 if /^\s+[0-9]+\s+(.*)/' | grep -a "position-$pos-" | sort > $f.$pos + done + { + echo "${${f%.list}#info-}\t${${f%.list}#info-}\t${${f%.list}#info-}" + echo "List\tList\tList" + echo "Mid\tLeft\tRight" + for e in 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F BF EFBCBC FE; do + echo -n $(perl -lne 'print "'$e'-$1" if /^position-middle-'$e'-(.*)-/' $f.middle | tr -d "\t\r" | grep -a . || echo Miss) + echo -n "\t" + echo -n $(perl -lne 'print "'$e'-$1" if /^(.*)-position-left-'$e'/' $f.left | tr -d "\t\r" | grep -a . || echo Miss) + echo -n "\t" + echo $(perl -lne 'print "'$e'-$1" if /^position-right-'$e'-(.*)/' $f.right | tr -d "\t\r" | grep -a . || echo Miss) + # echo -n $(grep -a "position-middle-$e-" $f.middle | tr -d "\t\r" || echo Miss)"\t" + # echo -n $(grep -a "position-left-$e" $f.left | tr -d "\t\r" || echo Miss)"\t" + # echo $(grep -a "position-right-$e-" $f.right | tr -d "\t\r" || echo Miss) + done + } > $f.csv +done + +for f in info-*.list; do + paste ${f%.list}.log.csv $f.csv > ${f%.list}.full.csv +done +paste *.full.csv > info-complete.csv diff --git a/cmd/info/test.cmd b/cmd/info/test.cmd new file mode 100644 index 000000000..9569fe63e --- /dev/null +++ b/cmd/info/test.cmd @@ -0,0 +1,3 @@ +rclone.exe purge info +rclone.exe info -vv info > info-LocalWindows.log 2>&1 +rclone.exe ls -vv info > info-LocalWindows.list 2>&1 diff --git a/cmd/info/test.sh b/cmd/info/test.sh new file mode 100755 index 000000000..3c27a4cf2 --- /dev/null +++ b/cmd/info/test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env zsh +# +# example usage: +# $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh --list | \ +# parallel -P20 $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh + +export PATH=$GOPATH/src/github.com/ncw/rclone:$PATH + +typeset -A allRemotes + allRemotes=( + TestAmazonCloudDrive '--low-level-retries=2 --checkers=5' + TestB2 '' + TestBox '' + TestDrive '--tpslimit=5' + TestCrypt '' + TestDropbox '--checkers=1' + TestJottacloud '' + TestMega '' + TestOneDrive '' + TestOpenDrive '--low-level-retries=2 --checkers=5' + TestPcloud '--low-level-retries=2 --timeout=15s' + TestS3 '' + Local '' +) + +set -euo pipefail + +if [[ $# -eq 0 ]]; then + set -- ${(k)allRemotes[@]} +elif [[ $1 = --list ]]; then + printf '%s\n' ${(k)allRemotes[@]} + exit 0 +fi + +for remote; do + dir=$remote:infotest + if [[ $remote = Local ]]; then + dir=infotest + fi + rclone purge $dir || : + rclone info -vv $dir ${=allRemotes[$remote]} &> info-$remote.log + rclone ls -vv $dir &> info-$remote.list +done \ No newline at end of file