From 0b9d7fec0c03848d27f39a6dbff45c95575b11dd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 14 Feb 2019 08:45:03 +0000 Subject: [PATCH] lsf: add 'e' format to show encrypted names and 'o' for original IDs This brings it up to par with lsjson. This commit also reworks the framework to use ListJSON internally which removes duplicated code and makes testing easier. --- cmd/lsf/lsf.go | 43 ++++++++------- fs/operations/operations.go | 76 ++++++++++++++++----------- fs/operations/operations_test.go | 90 ++++++++++++++++++++++---------- 3 files changed, 130 insertions(+), 79 deletions(-) diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index 3dffe5ce3..c2f07c8c8 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -10,7 +10,6 @@ import ( "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/hash" "github.com/ncw/rclone/fs/operations" - "github.com/ncw/rclone/fs/walk" "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -67,8 +66,10 @@ output: s - size t - modification time h - hash - i - ID of object if known + i - ID of object + o - Original ID of underlying object m - MimeType of object if known + e - encrypted name So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last. @@ -161,6 +162,10 @@ func Lsf(fsrc fs.Fs, out io.Writer) error { list.SetCSV(csv) list.SetDirSlash(dirSlash) list.SetAbsolute(absolute) + var opt = operations.ListJSONOpt{ + NoModTime: true, + Recurse: recurse, + } for _, char := range format { switch char { @@ -168,38 +173,38 @@ func Lsf(fsrc fs.Fs, out io.Writer) error { list.AddPath() case 't': list.AddModTime() + opt.NoModTime = false case 's': list.AddSize() case 'h': list.AddHash(hashType) + opt.ShowHash = true case 'i': list.AddID() case 'm': list.AddMimeType() + case 'e': + list.AddEncrypted() + opt.ShowEncrypted = true + case 'o': + list.AddOrigID() + opt.ShowOrigIDs = true default: return errors.Errorf("Unknown format character %q", char) } } - return walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(path string, entries fs.DirEntries, err error) error { - if err != nil { - fs.CountError(err) - fs.Errorf(path, "error listing: %v", err) - return nil - } - for _, entry := range entries { - _, isDir := entry.(fs.Directory) - if isDir { - if filesOnly { - continue - } - } else { - if dirsOnly { - continue - } + return operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error { + if item.IsDir { + if filesOnly { + return nil + } + } else { + if dirsOnly { + return nil } - _, _ = fmt.Fprintln(out, list.Format(entry)) } + _, _ = fmt.Fprintln(out, list.Format(item)) return nil }) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 417ad58d3..a90732cc3 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1479,8 +1479,7 @@ type ListFormat struct { separator string dirSlash bool absolute bool - output []func() string - entry fs.DirEntry + output []func(entry *ListJSONItem) string csv *csv.Writer buf bytes.Buffer } @@ -1516,76 +1515,91 @@ func (l *ListFormat) SetCSV(useCSV bool) { } // SetOutput sets functions used to create files information -func (l *ListFormat) SetOutput(output []func() string) { +func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) { l.output = output } // AddModTime adds file's Mod Time to output func (l *ListFormat) AddModTime() { - l.AppendOutput(func() string { return l.entry.ModTime().Local().Format("2006-01-02 15:04:05") }) + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ModTime.When.Local().Format("2006-01-02 15:04:05") + }) } // AddSize adds file's size to output func (l *ListFormat) AddSize() { - l.AppendOutput(func() string { - return strconv.FormatInt(l.entry.Size(), 10) + l.AppendOutput(func(entry *ListJSONItem) string { + return strconv.FormatInt(entry.Size, 10) }) } +// normalisePath makes sure the path has the correct slashes for the current mode +func (l *ListFormat) normalisePath(entry *ListJSONItem, remote string) string { + if l.absolute && !strings.HasPrefix(remote, "/") { + remote = "/" + remote + } + if entry.IsDir && l.dirSlash { + remote += "/" + } + return remote +} + // AddPath adds path to file to output func (l *ListFormat) AddPath() { - l.AppendOutput(func() string { - remote := l.entry.Remote() - if l.absolute && !strings.HasPrefix(remote, "/") { - remote = "/" + remote - } - _, isDir := l.entry.(fs.Directory) - if isDir && l.dirSlash { - remote += "/" - } - return remote + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Path) + }) +} + +// AddEncrypted adds the encrypted path to file to output +func (l *ListFormat) AddEncrypted() { + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Encrypted) }) } // AddHash adds the hash of the type given to the output func (l *ListFormat) AddHash(ht hash.Type) { - l.AppendOutput(func() string { - o, ok := l.entry.(fs.Object) - if !ok { + hashName := ht.String() + l.AppendOutput(func(entry *ListJSONItem) string { + if entry.IsDir { return "" } - return hashSum(ht, o) + return entry.Hashes[hashName] }) } // AddID adds file's ID to the output if known func (l *ListFormat) AddID() { - l.AppendOutput(func() string { - if do, ok := l.entry.(fs.IDer); ok { - return do.ID() - } - return "" + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ID + }) +} + +// AddOrigID adds file's Original ID to the output if known +func (l *ListFormat) AddOrigID() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.OrigID }) } // AddMimeType adds file's MimeType to the output if known func (l *ListFormat) AddMimeType() { - l.AppendOutput(func() string { - return fs.MimeTypeDirEntry(l.entry) + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.MimeType }) } // AppendOutput adds string generated by specific function to printed output -func (l *ListFormat) AppendOutput(functionToAppend func() string) { +func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string) { l.output = append(l.output, functionToAppend) } // Format prints information about the DirEntry in the format defined -func (l *ListFormat) Format(entry fs.DirEntry) (result string) { - l.entry = entry +func (l *ListFormat) Format(entry *ListJSONItem) (result string) { var out []string for _, fun := range l.output { - out = append(out, fun()) + out = append(out, fun(entry)) } if l.csv != nil { l.buf.Reset() diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index c219187b6..1ee69ee6a 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -39,7 +39,6 @@ import ( "github.com/ncw/rclone/fs/accounting" "github.com/ncw/rclone/fs/filter" "github.com/ncw/rclone/fs/hash" - "github.com/ncw/rclone/fs/list" "github.com/ncw/rclone/fs/operations" "github.com/ncw/rclone/fstest" "github.com/stretchr/testify/assert" @@ -873,61 +872,90 @@ func TestCheckEqualReaders(t *testing.T) { } func TestListFormat(t *testing.T) { - r := fstest.NewRun(t) - defer r.Finalise() - file1 := r.WriteObject("a", "a", t1) - file2 := r.WriteObject("subdir/b", "b", t1) + item0 := &operations.ListJSONItem{ + Path: "a", + Name: "a", + Encrypted: "encryptedFileName", + Size: 1, + MimeType: "application/octet-stream", + ModTime: operations.Timestamp{ + When: t1, + Format: "2006-01-02T15:04:05.000000000Z07:00"}, + IsDir: false, + Hashes: map[string]string{ + "MD5": "0cc175b9c0f1b6a831c399e269772661", + "SHA-1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", + "DropboxHash": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8", + "QuickXorHash": "6100000000000000000000000100000000000000"}, + ID: "fileID", + OrigID: "fileOrigID", + } - fstest.CheckItems(t, r.Fremote, file1, file2) + item1 := &operations.ListJSONItem{ + Path: "subdir", + Name: "subdir", + Encrypted: "encryptedDirName", + Size: -1, + MimeType: "inode/directory", + ModTime: operations.Timestamp{ + When: t2, + Format: "2006-01-02T15:04:05.000000000Z07:00"}, + IsDir: true, + Hashes: map[string]string(nil), + ID: "dirID", + OrigID: "dirOrigID", + } - items, _ := list.DirSorted(r.Fremote, true, "") var list operations.ListFormat list.AddPath() list.SetDirSlash(false) - assert.Equal(t, "subdir", list.Format(items[1])) + assert.Equal(t, "subdir", list.Format(item1)) list.SetDirSlash(true) - assert.Equal(t, "subdir/", list.Format(items[1])) + assert.Equal(t, "subdir/", list.Format(item1)) list.SetOutput(nil) - assert.Equal(t, "", list.Format(items[1])) + assert.Equal(t, "", list.Format(item1)) - list.AppendOutput(func() string { return "a" }) - list.AppendOutput(func() string { return "b" }) - assert.Equal(t, "ab", list.Format(items[1])) + list.AppendOutput(func(item *operations.ListJSONItem) string { return "a" }) + list.AppendOutput(func(item *operations.ListJSONItem) string { return "b" }) + assert.Equal(t, "ab", list.Format(item1)) list.SetSeparator(":::") - assert.Equal(t, "a:::b", list.Format(items[1])) + assert.Equal(t, "a:::b", list.Format(item1)) list.SetOutput(nil) list.AddModTime() - assert.Equal(t, items[0].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[0])) + assert.Equal(t, t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) list.SetOutput(nil) + list.SetSeparator("|") list.AddID() - _ = list.Format(items[0]) // Can't really check anything - at least it didn't panic! + list.AddOrigID() + assert.Equal(t, "fileID|fileOrigID", list.Format(item0)) + assert.Equal(t, "dirID|dirOrigID", list.Format(item1)) list.SetOutput(nil) list.AddMimeType() - assert.Contains(t, list.Format(items[0]), "/") - assert.Equal(t, "inode/directory", list.Format(items[1])) + assert.Contains(t, list.Format(item0), "/") + assert.Equal(t, "inode/directory", list.Format(item1)) list.SetOutput(nil) list.AddPath() list.SetAbsolute(true) - assert.Equal(t, "/a", list.Format(items[0])) + assert.Equal(t, "/a", list.Format(item0)) list.SetAbsolute(false) - assert.Equal(t, "a", list.Format(items[0])) + assert.Equal(t, "a", list.Format(item0)) list.SetOutput(nil) list.AddSize() - assert.Equal(t, "1", list.Format(items[0])) + assert.Equal(t, "1", list.Format(item0)) list.AddPath() list.AddModTime() list.SetDirSlash(true) list.SetSeparator("__SEP__") - assert.Equal(t, "1__SEP__a__SEP__"+items[0].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[0])) - assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"__SEP__subdir/__SEP__"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1])) + assert.Equal(t, "1__SEP__a__SEP__"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + assert.Equal(t, "-1__SEP__subdir/__SEP__"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) for _, test := range []struct { ht hash.Type @@ -939,10 +967,7 @@ func TestListFormat(t *testing.T) { } { list.SetOutput(nil) list.AddHash(test.ht) - got := list.Format(items[0]) - if got != "UNSUPPORTED" && got != "" { - assert.Equal(t, test.want, got) - } + assert.Equal(t, test.want, list.Format(item0)) } list.SetOutput(nil) @@ -952,8 +977,15 @@ func TestListFormat(t *testing.T) { list.AddPath() list.AddModTime() list.SetDirSlash(true) - assert.Equal(t, "1|a|"+items[0].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[0])) - assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"|subdir/|"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1])) + assert.Equal(t, "1|a|"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + assert.Equal(t, "-1|subdir/|"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) + + list.SetOutput(nil) + list.SetSeparator("|") + list.AddPath() + list.AddEncrypted() + assert.Equal(t, "a|encryptedFileName", list.Format(item0)) + assert.Equal(t, "subdir/|encryptedDirName/", list.Format(item1)) }