From 66929416d40f3ef61fe67a3b3adc4e5fb1495cb0 Mon Sep 17 00:00:00 2001 From: nielash Date: Thu, 7 Dec 2023 19:29:55 -0500 Subject: [PATCH] lsf: add --time-format flag Before this change, lsf's time format was hard-coded to "2006-01-02 15:04:05", regardless of the Fs's precision. After this change, a new optional --time-format flag is added to allow customizing the format (the default is unchanged). Examples: rclone lsf remote:path --format pt --time-format 'Jan 2, 2006 at 3:04pm (MST)' rclone lsf remote:path --format pt --time-format '2006-01-02 15:04:05.000000000' rclone lsf remote:path --format pt --time-format '2006-01-02T15:04:05.999999999Z07:00' rclone lsf remote:path --format pt --time-format RFC3339 rclone lsf remote:path --format pt --time-format DateOnly rclone lsf remote:path --format pt --time-format max --time-format max will automatically truncate '2006-01-02 15:04:05.000000000' to the maximum precision supported by the remote. --- cmd/lsf/lsf.go | 38 +++++++++++---- cmd/lsf/lsf_test.go | 80 ++++++++++++++++++++++++++++++++ fs/operations/operations.go | 74 ++++++++++++++++++++++++++++- fs/operations/operations_test.go | 6 +-- 4 files changed, 183 insertions(+), 15 deletions(-) diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index 22e4de754..38598c18e 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -17,21 +17,23 @@ import ( ) var ( - format string - separator string - dirSlash bool - recurse bool - hashType = hash.MD5 - filesOnly bool - dirsOnly bool - csv bool - absolute bool + format string + timeFormat string + separator string + dirSlash bool + recurse bool + hashType = hash.MD5 + filesOnly bool + dirsOnly bool + csv bool + absolute bool ) func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.StringVarP(cmdFlags, &format, "format", "F", "p", "Output format - see help for details", "") + flags.StringVarP(cmdFlags, &timeFormat, "time-format", "t", "", "Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)", "") flags.StringVarP(cmdFlags, &separator, "separator", "s", ";", "Separator for the items in the format", "") flags.BoolVarP(cmdFlags, &dirSlash, "dir-slash", "d", true, "Append a slash to directory names", "") flags.FVarP(cmdFlags, &hashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "") @@ -141,6 +143,19 @@ those only (without traversing the whole directory structure): rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files rclone copy --files-from-raw new_files /path/to/local remote:path +The default time format is ` + "`'2006-01-02 15:04:05'`" + `. +[Other formats](https://pkg.go.dev/time#pkg-constants) can be specified with the ` + "`--time-format`" + ` flag. +Examples: + + rclone lsf remote:path --format pt --time-format 'Jan 2, 2006 at 3:04pm (MST)' + rclone lsf remote:path --format pt --time-format '2006-01-02 15:04:05.000000000' + rclone lsf remote:path --format pt --time-format '2006-01-02T15:04:05.999999999Z07:00' + rclone lsf remote:path --format pt --time-format RFC3339 + rclone lsf remote:path --format pt --time-format DateOnly + rclone lsf remote:path --format pt --time-format max +` + "`--time-format max`" + ` will automatically truncate ` + "'`2006-01-02 15:04:05.000000000`'" + ` +to the maximum precision supported by the remote. + ` + lshelp.Help, Annotations: map[string]string{ "versionIntroduced": "v1.40", @@ -183,7 +198,10 @@ func Lsf(ctx context.Context, fsrc fs.Fs, out io.Writer) error { case 'p': list.AddPath() case 't': - list.AddModTime() + if timeFormat == "max" { + timeFormat = operations.FormatForLSFPrecision(fsrc.Precision()) + } + list.AddModTime(timeFormat) opt.NoModTime = false case 's': list.AddSize() diff --git a/cmd/lsf/lsf_test.go b/cmd/lsf/lsf_test.go index 72c0853b0..886acbcf0 100644 --- a/cmd/lsf/lsf_test.go +++ b/cmd/lsf/lsf_test.go @@ -8,6 +8,7 @@ import ( _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/list" + "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -223,3 +224,82 @@ subdir/file3_+_111_+_`+expectedOutput[6]+` recurse = false dirSlash = false } + +func TestTimeFormat(t *testing.T) { + fstest.Initialise() + f, err := fs.NewFs(context.Background(), "testfiles") + require.NoError(t, err) + format = "pst" + separator = "_+_" + recurse = true + dirSlash = true + timeFormat = "Jan 2, 2006 at 3:04pm (MST)" + + buf := new(bytes.Buffer) + err = Lsf(context.Background(), f, buf) + require.NoError(t, err) + + items, _ := list.DirSorted(context.Background(), f, true, "") + itemsInSubdir, _ := list.DirSorted(context.Background(), f, true, "subdir") + var expectedOutput []string + for _, item := range items { + expectedOutput = append(expectedOutput, item.ModTime(context.Background()).Format(timeFormat)) + } + for _, item := range itemsInSubdir { + expectedOutput = append(expectedOutput, item.ModTime(context.Background()).Format(timeFormat)) + } + + assert.Equal(t, `file1_+_0_+_`+expectedOutput[0]+` +file2_+_321_+_`+expectedOutput[1]+` +file3_+_1234_+_`+expectedOutput[2]+` +subdir/_+_-1_+_`+expectedOutput[3]+` +subdir/file1_+_0_+_`+expectedOutput[4]+` +subdir/file2_+_1_+_`+expectedOutput[5]+` +subdir/file3_+_111_+_`+expectedOutput[6]+` +`, buf.String()) + + format = "" + separator = "" + recurse = false + dirSlash = false +} + +func TestTimeFormatMax(t *testing.T) { + fstest.Initialise() + f, err := fs.NewFs(context.Background(), "testfiles") + require.NoError(t, err) + format = "pst" + separator = "_+_" + recurse = true + dirSlash = true + timeFormat = "max" + precision := operations.FormatForLSFPrecision(f.Precision()) + + buf := new(bytes.Buffer) + err = Lsf(context.Background(), f, buf) + require.NoError(t, err) + + items, _ := list.DirSorted(context.Background(), f, true, "") + itemsInSubdir, _ := list.DirSorted(context.Background(), f, true, "subdir") + var expectedOutput []string + for _, item := range items { + expectedOutput = append(expectedOutput, item.ModTime(context.Background()).Format(precision)) + } + for _, item := range itemsInSubdir { + expectedOutput = append(expectedOutput, item.ModTime(context.Background()).Format(precision)) + } + + assert.Equal(t, `file1_+_0_+_`+expectedOutput[0]+` +file2_+_321_+_`+expectedOutput[1]+` +file3_+_1234_+_`+expectedOutput[2]+` +subdir/_+_-1_+_`+expectedOutput[3]+` +subdir/file1_+_0_+_`+expectedOutput[4]+` +subdir/file2_+_1_+_`+expectedOutput[5]+` +subdir/file3_+_111_+_`+expectedOutput[6]+` +`, buf.String()) + + format = "" + separator = "" + recurse = false + dirSlash = false +} diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 62d03ab37..d426c673c 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1916,9 +1916,54 @@ func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) { } // AddModTime adds file's Mod Time to output -func (l *ListFormat) AddModTime() { +func (l *ListFormat) AddModTime(timeFormat string) { + switch timeFormat { + case "": + timeFormat = "2006-01-02 15:04:05" + case "Layout": + timeFormat = time.Layout + case "ANSIC": + timeFormat = time.ANSIC + case "UnixDate": + timeFormat = time.UnixDate + case "RubyDate": + timeFormat = time.RubyDate + case "RFC822": + timeFormat = time.RFC822 + case "RFC822Z": + timeFormat = time.RFC822Z + case "RFC850": + timeFormat = time.RFC850 + case "RFC1123": + timeFormat = time.RFC1123 + case "RFC1123Z": + timeFormat = time.RFC1123Z + case "RFC3339": + timeFormat = time.RFC3339 + case "RFC3339Nano": + timeFormat = time.RFC3339Nano + case "Kitchen": + timeFormat = time.Kitchen + case "Stamp": + timeFormat = time.Stamp + case "StampMilli": + timeFormat = time.StampMilli + case "StampMicro": + timeFormat = time.StampMicro + case "StampNano": + timeFormat = time.StampNano + case "DateTime": + // timeFormat = time.DateTime // missing in go1.19 + timeFormat = "2006-01-02 15:04:05" + case "DateOnly": + // timeFormat = time.DateOnly // missing in go1.19 + timeFormat = "2006-01-02" + case "TimeOnly": + // timeFormat = time.TimeOnly // missing in go1.19 + timeFormat = "15:04:05" + } l.AppendOutput(func(entry *ListJSONItem) string { - return entry.ModTime.When.Local().Format("2006-01-02 15:04:05") + return entry.ModTime.When.Local().Format(timeFormat) }) } @@ -2030,6 +2075,31 @@ func (l *ListFormat) Format(entry *ListJSONItem) (result string) { return result } +// FormatForLSFPrecision Returns a time format for the given precision +func FormatForLSFPrecision(precision time.Duration) string { + switch { + case precision <= time.Nanosecond: + return "2006-01-02 15:04:05.000000000" + case precision <= 10*time.Nanosecond: + return "2006-01-02 15:04:05.00000000" + case precision <= 100*time.Nanosecond: + return "2006-01-02 15:04:05.0000000" + case precision <= time.Microsecond: + return "2006-01-02 15:04:05.000000" + case precision <= 10*time.Microsecond: + return "2006-01-02 15:04:05.00000" + case precision <= 100*time.Microsecond: + return "2006-01-02 15:04:05.0000" + case precision <= time.Millisecond: + return "2006-01-02 15:04:05.000" + case precision <= 10*time.Millisecond: + return "2006-01-02 15:04:05.00" + case precision <= 100*time.Millisecond: + return "2006-01-02 15:04:05.0" + } + return "2006-01-02 15:04:05" +} + // DirMove renames srcRemote to dstRemote // // It does this by loading the directory tree into memory (using ListR diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 02f7b108c..19d58ea30 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1240,7 +1240,7 @@ func TestListFormat(t *testing.T) { assert.Equal(t, "a:::b", list.Format(item1)) list.SetOutput(nil) - list.AddModTime() + list.AddModTime("") assert.Equal(t, t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) list.SetOutput(nil) @@ -1272,7 +1272,7 @@ func TestListFormat(t *testing.T) { assert.Equal(t, "1", list.Format(item0)) list.AddPath() - list.AddModTime() + list.AddModTime("") list.SetDirSlash(true) list.SetSeparator("__SEP__") assert.Equal(t, "1__SEP__a__SEP__"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) @@ -1295,7 +1295,7 @@ func TestListFormat(t *testing.T) { list.SetCSV(true) list.AddSize() list.AddPath() - list.AddModTime() + list.AddModTime("") list.SetDirSlash(true) assert.Equal(t, "1|a|"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) assert.Equal(t, "-1|subdir/|"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1))