diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index 3dffe5ce3..c2f07c8c8 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -10,7 +10,6 @@ import ( "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/hash" "github.com/ncw/rclone/fs/operations" - "github.com/ncw/rclone/fs/walk" "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -67,8 +66,10 @@ output: s - size t - modification time h - hash - i - ID of object if known + i - ID of object + o - Original ID of underlying object m - MimeType of object if known + e - encrypted name So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last. @@ -161,6 +162,10 @@ func Lsf(fsrc fs.Fs, out io.Writer) error { list.SetCSV(csv) list.SetDirSlash(dirSlash) list.SetAbsolute(absolute) + var opt = operations.ListJSONOpt{ + NoModTime: true, + Recurse: recurse, + } for _, char := range format { switch char { @@ -168,38 +173,38 @@ func Lsf(fsrc fs.Fs, out io.Writer) error { list.AddPath() case 't': list.AddModTime() + opt.NoModTime = false case 's': list.AddSize() case 'h': list.AddHash(hashType) + opt.ShowHash = true case 'i': list.AddID() case 'm': list.AddMimeType() + case 'e': + list.AddEncrypted() + opt.ShowEncrypted = true + case 'o': + list.AddOrigID() + opt.ShowOrigIDs = true default: return errors.Errorf("Unknown format character %q", char) } } - return walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(path string, entries fs.DirEntries, err error) error { - if err != nil { - fs.CountError(err) - fs.Errorf(path, "error listing: %v", err) - return nil - } - for _, entry := range entries { - _, isDir := entry.(fs.Directory) - if isDir { - if filesOnly { - continue - } - } else { - if dirsOnly { - continue - } + return operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error { + if item.IsDir { + if filesOnly { + return nil + } + } else { + if dirsOnly { + return nil } - _, _ = fmt.Fprintln(out, list.Format(entry)) } + _, _ = fmt.Fprintln(out, list.Format(item)) return nil }) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 417ad58d3..a90732cc3 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1479,8 +1479,7 @@ type ListFormat struct { separator string dirSlash bool absolute bool - output []func() string - entry fs.DirEntry + output []func(entry *ListJSONItem) string csv *csv.Writer buf bytes.Buffer } @@ -1516,76 +1515,91 @@ func (l *ListFormat) SetCSV(useCSV bool) { } // SetOutput sets functions used to create files information -func (l *ListFormat) SetOutput(output []func() string) { +func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) { l.output = output } // AddModTime adds file's Mod Time to output func (l *ListFormat) AddModTime() { - l.AppendOutput(func() string { return l.entry.ModTime().Local().Format("2006-01-02 15:04:05") }) + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ModTime.When.Local().Format("2006-01-02 15:04:05") + }) } // AddSize adds file's size to output func (l *ListFormat) AddSize() { - l.AppendOutput(func() string { - return strconv.FormatInt(l.entry.Size(), 10) + l.AppendOutput(func(entry *ListJSONItem) string { + return strconv.FormatInt(entry.Size, 10) }) } +// normalisePath makes sure the path has the correct slashes for the current mode +func (l *ListFormat) normalisePath(entry *ListJSONItem, remote string) string { + if l.absolute && !strings.HasPrefix(remote, "/") { + remote = "/" + remote + } + if entry.IsDir && l.dirSlash { + remote += "/" + } + return remote +} + // AddPath adds path to file to output func (l *ListFormat) AddPath() { - l.AppendOutput(func() string { - remote := l.entry.Remote() - if l.absolute && !strings.HasPrefix(remote, "/") { - remote = "/" + remote - } - _, isDir := l.entry.(fs.Directory) - if isDir && l.dirSlash { - remote += "/" - } - return remote + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Path) + }) +} + +// AddEncrypted adds the encrypted path to file to output +func (l *ListFormat) AddEncrypted() { + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Encrypted) }) } // AddHash adds the hash of the type given to the output func (l *ListFormat) AddHash(ht hash.Type) { - l.AppendOutput(func() string { - o, ok := l.entry.(fs.Object) - if !ok { + hashName := ht.String() + l.AppendOutput(func(entry *ListJSONItem) string { + if entry.IsDir { return "" } - return hashSum(ht, o) + return entry.Hashes[hashName] }) } // AddID adds file's ID to the output if known func (l *ListFormat) AddID() { - l.AppendOutput(func() string { - if do, ok := l.entry.(fs.IDer); ok { - return do.ID() - } - return "" + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ID + }) +} + +// AddOrigID adds file's Original ID to the output if known +func (l *ListFormat) AddOrigID() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.OrigID }) } // AddMimeType adds file's MimeType to the output if known func (l *ListFormat) AddMimeType() { - l.AppendOutput(func() string { - return fs.MimeTypeDirEntry(l.entry) + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.MimeType }) } // AppendOutput adds string generated by specific function to printed output -func (l *ListFormat) AppendOutput(functionToAppend func() string) { +func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string) { l.output = append(l.output, functionToAppend) } // Format prints information about the DirEntry in the format defined -func (l *ListFormat) Format(entry fs.DirEntry) (result string) { - l.entry = entry +func (l *ListFormat) Format(entry *ListJSONItem) (result string) { var out []string for _, fun := range l.output { - out = append(out, fun()) + out = append(out, fun(entry)) } if l.csv != nil { l.buf.Reset() diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index c219187b6..1ee69ee6a 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -39,7 +39,6 @@ import ( "github.com/ncw/rclone/fs/accounting" "github.com/ncw/rclone/fs/filter" "github.com/ncw/rclone/fs/hash" - "github.com/ncw/rclone/fs/list" "github.com/ncw/rclone/fs/operations" "github.com/ncw/rclone/fstest" "github.com/stretchr/testify/assert" @@ -873,61 +872,90 @@ func TestCheckEqualReaders(t *testing.T) { } func TestListFormat(t *testing.T) { - r := fstest.NewRun(t) - defer r.Finalise() - file1 := r.WriteObject("a", "a", t1) - file2 := r.WriteObject("subdir/b", "b", t1) + item0 := &operations.ListJSONItem{ + Path: "a", + Name: "a", + Encrypted: "encryptedFileName", + Size: 1, + MimeType: "application/octet-stream", + ModTime: operations.Timestamp{ + When: t1, + Format: "2006-01-02T15:04:05.000000000Z07:00"}, + IsDir: false, + Hashes: map[string]string{ + "MD5": "0cc175b9c0f1b6a831c399e269772661", + "SHA-1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", + "DropboxHash": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8", + "QuickXorHash": "6100000000000000000000000100000000000000"}, + ID: "fileID", + OrigID: "fileOrigID", + } - fstest.CheckItems(t, r.Fremote, file1, file2) + item1 := &operations.ListJSONItem{ + Path: "subdir", + Name: "subdir", + Encrypted: "encryptedDirName", + Size: -1, + MimeType: "inode/directory", + ModTime: operations.Timestamp{ + When: t2, + Format: "2006-01-02T15:04:05.000000000Z07:00"}, + IsDir: true, + Hashes: map[string]string(nil), + ID: "dirID", + OrigID: "dirOrigID", + } - items, _ := list.DirSorted(r.Fremote, true, "") var list operations.ListFormat list.AddPath() list.SetDirSlash(false) - assert.Equal(t, "subdir", list.Format(items[1])) + assert.Equal(t, "subdir", list.Format(item1)) list.SetDirSlash(true) - assert.Equal(t, "subdir/", list.Format(items[1])) + assert.Equal(t, "subdir/", list.Format(item1)) list.SetOutput(nil) - assert.Equal(t, "", list.Format(items[1])) + assert.Equal(t, "", list.Format(item1)) - list.AppendOutput(func() string { return "a" }) - list.AppendOutput(func() string { return "b" }) - assert.Equal(t, "ab", list.Format(items[1])) + list.AppendOutput(func(item *operations.ListJSONItem) string { return "a" }) + list.AppendOutput(func(item *operations.ListJSONItem) string { return "b" }) + assert.Equal(t, "ab", list.Format(item1)) list.SetSeparator(":::") - assert.Equal(t, "a:::b", list.Format(items[1])) + assert.Equal(t, "a:::b", list.Format(item1)) list.SetOutput(nil) list.AddModTime() - assert.Equal(t, items[0].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[0])) + assert.Equal(t, t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) list.SetOutput(nil) + list.SetSeparator("|") list.AddID() - _ = list.Format(items[0]) // Can't really check anything - at least it didn't panic! + list.AddOrigID() + assert.Equal(t, "fileID|fileOrigID", list.Format(item0)) + assert.Equal(t, "dirID|dirOrigID", list.Format(item1)) list.SetOutput(nil) list.AddMimeType() - assert.Contains(t, list.Format(items[0]), "/") - assert.Equal(t, "inode/directory", list.Format(items[1])) + assert.Contains(t, list.Format(item0), "/") + assert.Equal(t, "inode/directory", list.Format(item1)) list.SetOutput(nil) list.AddPath() list.SetAbsolute(true) - assert.Equal(t, "/a", list.Format(items[0])) + assert.Equal(t, "/a", list.Format(item0)) list.SetAbsolute(false) - assert.Equal(t, "a", list.Format(items[0])) + assert.Equal(t, "a", list.Format(item0)) list.SetOutput(nil) list.AddSize() - assert.Equal(t, "1", list.Format(items[0])) + assert.Equal(t, "1", list.Format(item0)) list.AddPath() list.AddModTime() list.SetDirSlash(true) list.SetSeparator("__SEP__") - assert.Equal(t, "1__SEP__a__SEP__"+items[0].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[0])) - assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"__SEP__subdir/__SEP__"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1])) + assert.Equal(t, "1__SEP__a__SEP__"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + assert.Equal(t, "-1__SEP__subdir/__SEP__"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) for _, test := range []struct { ht hash.Type @@ -939,10 +967,7 @@ func TestListFormat(t *testing.T) { } { list.SetOutput(nil) list.AddHash(test.ht) - got := list.Format(items[0]) - if got != "UNSUPPORTED" && got != "" { - assert.Equal(t, test.want, got) - } + assert.Equal(t, test.want, list.Format(item0)) } list.SetOutput(nil) @@ -952,8 +977,15 @@ func TestListFormat(t *testing.T) { list.AddPath() list.AddModTime() list.SetDirSlash(true) - assert.Equal(t, "1|a|"+items[0].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[0])) - assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"|subdir/|"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1])) + assert.Equal(t, "1|a|"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + assert.Equal(t, "-1|subdir/|"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) + + list.SetOutput(nil) + list.SetSeparator("|") + list.AddPath() + list.AddEncrypted() + assert.Equal(t, "a|encryptedFileName", list.Format(item0)) + assert.Equal(t, "subdir/|encryptedDirName/", list.Format(item1)) }