diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index da9184f30..38076541e 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -3,22 +3,137 @@ package sync import ( "context" + "io" + "os" + + mutex "sync" // renamed as "sync" already in use "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/operations/operationsflags" "github.com/rclone/rclone/fs/sync" "github.com/spf13/cobra" ) var ( createEmptySrcDirs = false + opt = operations.LoggerOpt{} + loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{} ) func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "") + operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt) + // TODO: add same flags to move and copy +} + +var lock mutex.Mutex + +func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) { + lock.Lock() + defer lock.Unlock() + + if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil { + opt.PrintDestAfter(ctx, sigil, src, dst, err) + return + } + + _, srcOk := src.(fs.Object) + _, dstOk := dst.(fs.Object) + var filename string + if !srcOk && !dstOk { + return + } else if srcOk && !dstOk { + filename = src.String() + } else { + filename = dst.String() + } + + if sigil.Writer(opt) != nil { + operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename) + } + if opt.Combined != nil { + operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename) + fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename) + } + if opt.DestAfter != nil { + opt.PrintDestAfter(ctx, sigil, src, dst, err) + } +} + +// GetSyncLoggerOpt gets the options corresponding to the logger flags +func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) { + closers := []io.Closer{} + + opt.LoggerFn = syncLoggerFn + if opt.TimeFormat == "max" { + opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision()) + } + opt.SetListFormat(ctx, command.Flags()) + opt.NewListJSON(ctx, fdst, "") + + open := func(name string, pout *io.Writer) error { + if name == "" { + return nil + } + if name == "-" { + *pout = os.Stdout + return nil + } + out, err := os.Create(name) + if err != nil { + return err + } + *pout = out + closers = append(closers, out) + return nil + } + + if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil { + return opt, nil, err + } + if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil { + return opt, nil, err + } + if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil { + return opt, nil, err + } + if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil { + return opt, nil, err + } + if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil { + return opt, nil, err + } + if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil { + return opt, nil, err + } + if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil { + return opt, nil, err + } + + close := func() { + for _, closer := range closers { + err := closer.Close() + if err != nil { + fs.Errorf(nil, "Failed to close report output: %v", err) + } + } + } + + return opt, close, nil +} + +func anyNotBlank(s ...string) bool { + for _, x := range s { + if x != "" { + return true + } + } + return false } var commandDefinition = &cobra.Command{ @@ -59,6 +174,44 @@ destination that is inside the source directory. **Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info. + +## Logger Flags + +The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` + + "`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or +stdout if it is ` + "`-`" + `) supplied. What they write is described in the +help below. For example ` + "`--differ`" + ` will write all paths which are +present on both the source and destination but different. + +The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all +file paths with a symbol and then a space and then the path to tell +you what happened to it. These are reminiscent of diff files. + +- ` + "`= path`" + ` means path was found in source and destination and was identical +- ` + "`- path`" + ` means path was missing on the source, so only in the destination +- ` + "`+ path`" + ` means path was missing on the destination, so only in the source +- ` + "`* path`" + ` means path was present in source and destination but different. +- ` + "`! path`" + ` means there was an error reading or hashing the source or dest. + +The ` + "`--dest-after`" + ` flag writes a list file using the same format flags +as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options +for hash, modtime, etc.](/commands/rclone_lsf/#synopsis)) +Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical +-- it should output an accurate list of what will be on the destination +after the sync. + +Note that these logger flags have a few limitations, and certain scenarios +are not currently supported: + +- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + ` +- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + ` +- server-side moves of an entire dir at once +- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable) +- Possibly some unusual error scenarios + +Note also that each file is logged during the sync, as opposed to after, so it +is most useful as a predictor of what SHOULD happen to each file +(which may or may not match what actually DID.) `, Annotations: map[string]string{ "groups": "Sync,Copy,Filter,Listing,Important", @@ -67,10 +220,22 @@ See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14 cmd.CheckArgs(2, 2, command, args) fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args) cmd.Run(true, true, command, func() error { - if srcFileName == "" { - return sync.Sync(context.Background(), fdst, fsrc, createEmptySrcDirs) + ctx := context.Background() + opt, close, err := GetSyncLoggerOpt(ctx, fdst, command) + if err != nil { + return err } - return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName) + defer close() + + if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst, + loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) { + ctx = operations.WithSyncLogger(ctx, opt) + } + + if srcFileName == "" { + return sync.Sync(ctx, fdst, fsrc, createEmptySrcDirs) + } + return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName) }) }, } diff --git a/docs/content/commands/rclone_sync.md b/docs/content/commands/rclone_sync.md index 8096e01c3..5ecbb3c03 100644 --- a/docs/content/commands/rclone_sync.md +++ b/docs/content/commands/rclone_sync.md @@ -48,6 +48,43 @@ destination that is inside the source directory. **Note**: Use the `rclone dedupe` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info. +## Logger Flags + +The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` +and `--error` flags write paths, one per line, to the file name (or +stdout if it is `-`) supplied. What they write is described in the +help below. For example `--differ` will write all paths which are +present on both the source and destination but different. + +The `--combined` flag will write a file (or stdout) which contains all +file paths with a symbol and then a space and then the path to tell +you what happened to it. These are reminiscent of diff files. + +- `= path` means path was found in source and destination and was identical +- `- path` means path was missing on the source, so only in the destination +- `+ path` means path was missing on the destination, so only in the source +- `* path` means path was present in source and destination but different. +- `! path` means there was an error reading or hashing the source or dest. + +The `--dest-after` flag writes a list file using the same format flags +as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options +for hash, modtime, etc.](/commands/rclone_lsf/#synopsis)) +Conceptually it is similar to rsync's `--itemize-changes`, but not identical +-- it should output an accurate list of what will be on the destination +after the sync. + +Note that these logger flags have a few limitations, and certain scenarios +are not currently supported: + +- `--max-duration` / `CutoffModeHard` +- `--compare-dest` / `--copy-dest` +- server-side moves of an entire dir at once +- High-level retries, because there would be duplicates (use `--retries 1` to disable) +- Possibly some unusual error scenarios + +Note also that each file is logged during the sync, as opposed to after, so it +is most useful as a predictor of what SHOULD happen to each file +(which may or may not match what actually DID.) ``` rclone sync source:path dest:path [flags] diff --git a/fs/logger/logger.go b/fs/logger/logger.go new file mode 100644 index 000000000..722b5d396 --- /dev/null +++ b/fs/logger/logger.go @@ -0,0 +1,17 @@ +// Package logger implements testing for the sync (and bisync) logger +package logger + +import ( + _ "github.com/rclone/rclone/backend/all" // import all backends + "github.com/rclone/rclone/cmd" + _ "github.com/rclone/rclone/cmd/all" // import all commands + _ "github.com/rclone/rclone/lib/plugin" // import plugins +) + +// Main enables the testscript package. See: +// https://bitfieldconsulting.com/golang/cli-testing +// https://pkg.go.dev/github.com/rogpeppe/go-internal@v1.11.0/testscript +func Main() int { + cmd.Main() + return 0 +} diff --git a/fs/logger/logger_test.go b/fs/logger/logger_test.go new file mode 100644 index 000000000..0e53563ce --- /dev/null +++ b/fs/logger/logger_test.go @@ -0,0 +1,33 @@ +package logger_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/rclone/rclone/fs/logger" + "github.com/rogpeppe/go-internal/testscript" +) + +// TestMain drives the tests +func TestMain(m *testing.M) { + // This enables the testscript package. See: + // https://bitfieldconsulting.com/golang/cli-testing + // https://pkg.go.dev/github.com/rogpeppe/go-internal@v1.11.0/testscript + os.Exit(testscript.RunMain(m, map[string]func() int{ + "rclone": logger.Main, + })) +} + +func TestLogger(t *testing.T) { + // Usage: https://bitfieldconsulting.com/golang/cli-testing + + testscript.Run(t, testscript.Params{ + Dir: "testdata/script", + Setup: func(env *testscript.Env) error { + env.Setenv("SRC", filepath.Join("$WORK", "src")) + env.Setenv("DST", filepath.Join("$WORK", "dst")) + return nil + }, + }) +} diff --git a/fs/logger/testdata/script/TestBeforeVsAfter.txtar b/fs/logger/testdata/script/TestBeforeVsAfter.txtar new file mode 100644 index 000000000..0ccadc973 --- /dev/null +++ b/fs/logger/testdata/script/TestBeforeVsAfter.txtar @@ -0,0 +1,48 @@ +# tests whether an md5sum file generated post-sync matches our pre-sync prediction + +# Filling src and dst with two different versions of rclone source!: +exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip +exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip +exec unzip $SRC/src.zip -d $SRC +exec unzip $DST/dst.zip -d $DST + + +# generating sumfiles: +exec rclone md5sum $SRC --output-file $WORK/src-before.txt +exec rclone md5sum $DST --output-file $WORK/dst-before.txt + +# running sync with output files: +exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt --dest-after $WORK/SYNCdestafter.txt --format 'hp' --separator ' ' + +# generating sumfiles: +exec rclone md5sum $SRC --output-file $WORK/src-after.txt +exec rclone md5sum $DST --output-file $WORK/dst-after.txt + +# sorting them by line and diffing: +exec sort $WORK/src-before.txt -o $WORK/src-before.txt +exec sort $WORK/dst-before.txt -o $WORK/dst-before.txt +exec sort $WORK/src-after.txt -o $WORK/src-after.txt +exec sort $WORK/dst-after.txt -o $WORK/dst-after.txt + +exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt +exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt +exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt +exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt +exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt +exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt +exec sort $WORK/SYNCdestafter.txt -o $WORK/SYNCdestafter.txt + +# diff src before vs src after: +cmp $WORK/src-before.txt $WORK/src-after.txt + +# diff dst before vs dst after: +! cmp $WORK/dst-before.txt $WORK/dst-after.txt + +# diff src before vs dst after: +cmp $WORK/src-before.txt $WORK/dst-after.txt + +# diff dst before vs src after: +! cmp $WORK/dst-before.txt $WORK/src-after.txt + +# diff md5sum dst after vs sync dest-after: +cmp $WORK/dst-after.txt $WORK/SYNCdestafter.txt \ No newline at end of file diff --git a/fs/logger/testdata/script/TestCheckVsSync.txtar b/fs/logger/testdata/script/TestCheckVsSync.txtar new file mode 100644 index 000000000..11d2803d1 --- /dev/null +++ b/fs/logger/testdata/script/TestCheckVsSync.txtar @@ -0,0 +1,48 @@ +# tests whether rclone check and rclone sync output exactly the same file lists. + +# Filling src and dst with makefiles: +exec rclone test makefiles $SRC --seed 0 +exec rclone test makefiles $DST --seed 1 +exec rclone test makefiles $SRC --seed 2 +exec rclone test makefiles $DST --seed 2 + +# running rclone check for baseline test: +# error is expected here: +! exec rclone check $SRC $DST --match $WORK/CHECKmatch.txt --combined $WORK/CHECKcombined.txt --missing-on-src $WORK/CHECKmissingonsrc.txt --missing-on-dst $WORK/CHECKmissingondst.txt --error $WORK/CHECKerr.txt --differ $WORK/CHECKdiffer.txt -q + +# running sync with output files: +exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt + +# sorting them by line and diffing: +exec sort $WORK/CHECKmatch.txt -o $WORK/CHECKmatch.txt +exec sort $WORK/CHECKcombined.txt -o $WORK/CHECKcombined.txt +exec sort $WORK/CHECKmissingonsrc.txt -o $WORK/CHECKmissingonsrc.txt +exec sort $WORK/CHECKmissingondst.txt -o $WORK/CHECKmissingondst.txt +exec sort $WORK/CHECKerr.txt -o $WORK/CHECKerr.txt +exec sort $WORK/CHECKdiffer.txt -o $WORK/CHECKdiffer.txt + +exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt +exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt +exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt +exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt +exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt +exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt + +# diff match check vs. sync: +cmp $WORK/CHECKmatch.txt $WORK/SYNCmatch.txt +# diff combined check vs. sync: +cmp $WORK/CHECKcombined.txt $WORK/SYNCcombined.txt +# diff missingonsrc check vs. sync: +cmp $WORK/CHECKmissingonsrc.txt $WORK/SYNCmissingonsrc.txt +# diff missingondst check vs. sync: +cmp $WORK/CHECKmissingondst.txt $WORK/SYNCmissingondst.txt +# diff error check vs. sync: +cmp $WORK/CHECKerr.txt $WORK/SYNCerr.txt +# diff differ check vs. sync: +cmp $WORK/CHECKdiffer.txt $WORK/SYNCdiffer.txt + +# verify accuracy +exec rclone check $SRC $DST +exec diff -rya --suppress-common-lines $SRC $DST +[!windows] exec rsync -aEvhPu $SRC/ $WORK/rsyncDst +[!windows] exec rclone check $DST $WORK/rsyncDst \ No newline at end of file diff --git a/fs/logger/testdata/script/TestRepoCompare.txtar b/fs/logger/testdata/script/TestRepoCompare.txtar new file mode 100644 index 000000000..4bbe9220a --- /dev/null +++ b/fs/logger/testdata/script/TestRepoCompare.txtar @@ -0,0 +1,52 @@ +# tests whether rclone check and rclone sync output exactly the same file lists. +# uses two different old versions of rclone source code for the src and dst +# to produce a more realistic test than makefiles +# (has lots of files with same name but different content) + +# Filling src and dst with two different versions of rclone source!: +exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip +exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip +exec unzip $SRC/src.zip -d $SRC +exec unzip $DST/dst.zip -d $DST + + +# running rclone check for baseline test: +# error is expected here: +! exec rclone check $SRC $DST --match $WORK/CHECKmatch.txt --combined $WORK/CHECKcombined.txt --missing-on-src $WORK/CHECKmissingonsrc.txt --missing-on-dst $WORK/CHECKmissingondst.txt --error $WORK/CHECKerr.txt --differ $WORK/CHECKdiffer.txt -q + +# running sync with output files: +exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt + +# sorting them by line and diffing: +exec sort $WORK/CHECKmatch.txt -o $WORK/CHECKmatch.txt +exec sort $WORK/CHECKcombined.txt -o $WORK/CHECKcombined.txt +exec sort $WORK/CHECKmissingonsrc.txt -o $WORK/CHECKmissingonsrc.txt +exec sort $WORK/CHECKmissingondst.txt -o $WORK/CHECKmissingondst.txt +exec sort $WORK/CHECKerr.txt -o $WORK/CHECKerr.txt +exec sort $WORK/CHECKdiffer.txt -o $WORK/CHECKdiffer.txt + +exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt +exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt +exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt +exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt +exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt +exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt + +# diff match check vs. sync: +cmp $WORK/CHECKmatch.txt $WORK/SYNCmatch.txt +# diff combined check vs. sync: +cmp $WORK/CHECKcombined.txt $WORK/SYNCcombined.txt +# diff missingonsrc check vs. sync: +cmp $WORK/CHECKmissingonsrc.txt $WORK/SYNCmissingonsrc.txt +# diff missingondst check vs. sync: +cmp $WORK/CHECKmissingondst.txt $WORK/SYNCmissingondst.txt +# diff error check vs. sync: +cmp $WORK/CHECKerr.txt $WORK/SYNCerr.txt +# diff differ check vs. sync: +cmp $WORK/CHECKdiffer.txt $WORK/SYNCdiffer.txt + +# verify accuracy +exec rclone check $SRC $DST +exec diff -rya --suppress-common-lines $SRC $DST +[!windows] exec rsync -aEvhPu $SRC/ $WORK/rsyncDst +[!windows] exec rclone check $DST $WORK/rsyncDst \ No newline at end of file diff --git a/fs/operations/logger.go b/fs/operations/logger.go index 9d0e213b4..e0447659d 100644 --- a/fs/operations/logger.go +++ b/fs/operations/logger.go @@ -8,7 +8,6 @@ import ( "io" "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/hash" "github.com/spf13/pflag" ) @@ -107,28 +106,6 @@ type LoggerOpt struct { Absolute bool } -// AddLoggerFlags adds the logger flags to the cmdFlags command -func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *LoggerOpt, combined, missingOnSrc, missingOnDst, match, differ, errFile, destAfter *string) { - flags.StringVarP(cmdFlags, combined, "combined", "", *combined, "Make a combined report of changes to this file", "Sync") - flags.StringVarP(cmdFlags, missingOnSrc, "missing-on-src", "", *missingOnSrc, "Report all files missing from the source to this file", "Sync") - flags.StringVarP(cmdFlags, missingOnDst, "missing-on-dst", "", *missingOnDst, "Report all files missing from the destination to this file", "Sync") - flags.StringVarP(cmdFlags, match, "match", "", *match, "Report all matching files to this file", "Sync") - flags.StringVarP(cmdFlags, differ, "differ", "", *differ, "Report all non-matching files to this file", "Sync") - flags.StringVarP(cmdFlags, errFile, "error", "", *errFile, "Report all files with errors (hashing or reading) to this file", "Sync") - flags.StringVarP(cmdFlags, destAfter, "dest-after", "", *destAfter, "Report all files that exist on the dest post-sync", "Sync") - - // lsf flags for destAfter - flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync") - flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync") - flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync") - flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync") - flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync") - flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync") - flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync") - flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync") - // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "") -} - // WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger func WithLogger(ctx context.Context, logger LoggerFn) context.Context { return context.WithValue(ctx, loggerKey, logger) diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go new file mode 100644 index 000000000..80a6d77fb --- /dev/null +++ b/fs/operations/operationsflags/operationsflags.go @@ -0,0 +1,45 @@ +// Package operationsflags defines the flags used by rclone operations. +// It is decoupled into a separate package so it can be replaced. +package operationsflags + +import ( + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/spf13/pflag" +) + +// AddLoggerFlagsOptions contains options for the Logger Flags +type AddLoggerFlagsOptions struct { + Combined string // a file with file names with leading sigils + MissingOnSrc string // files only in the destination + MissingOnDst string // files only in the source + Match string // matching files + Differ string // differing files + ErrFile string // files with errors of some kind + DestAfter string // files that exist on the destination post-sync +} + +// AddLoggerFlags adds the logger flags to the cmdFlags command +func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) { + flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.MissingOnSrc, "missing-on-src", "", flagsOpt.MissingOnSrc, "Report all files missing from the source to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.MissingOnDst, "missing-on-dst", "", flagsOpt.MissingOnDst, "Report all files missing from the destination to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.Match, "match", "", flagsOpt.Match, "Report all matching files to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.Differ, "differ", "", flagsOpt.Differ, "Report all non-matching files to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.ErrFile, "error", "", flagsOpt.ErrFile, "Report all files with errors (hashing or reading) to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync") + + // lsf flags for destAfter + flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync") + flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)", "") + flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync") + opt.HashType = hash.MD5 + flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync") + flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync") + flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync") + flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync") + // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "") +} diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index e757af77c..4438477f4 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1495,10 +1495,10 @@ func TestServerSideMoveOverlap(t *testing.T) { r.CheckRemoteItems(t, file1) // Subdir move with no filters should return ErrorCantMoveOverlapping - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) assert.EqualError(t, err, fs.ErrorOverlapping.Error()) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // Now try with a filter which should also fail with ErrorCantMoveOverlapping fi, err := filter.NewFilter(nil) @@ -1506,10 +1506,10 @@ func TestServerSideMoveOverlap(t *testing.T) { fi.Opt.MinSize = 40 ctx = filter.ReplaceConfig(ctx, fi) - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) assert.EqualError(t, err, fs.ErrorOverlapping.Error()) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) } // Test a sync with overlap @@ -1822,9 +1822,9 @@ func TestSyncCopyDest(t *testing.T) { r.CheckLocalItems(t, file1) accounting.GlobalStats().ResetCounters() - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = Sync(ctx, fdst, r.Flocal, false) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // not currently supported require.NoError(t, err) file1dst := file1 @@ -1838,9 +1838,9 @@ func TestSyncCopyDest(t *testing.T) { r.CheckLocalItems(t, file1b) accounting.GlobalStats().ResetCounters() - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = Sync(ctx, fdst, r.Flocal, false) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) require.NoError(t, err) file1bdst := file1b @@ -1859,9 +1859,9 @@ func TestSyncCopyDest(t *testing.T) { r.CheckLocalItems(t, file1c) accounting.GlobalStats().ResetCounters() - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = Sync(ctx, fdst, r.Flocal, false) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) require.NoError(t, err) file2dst := file2 @@ -1878,8 +1878,8 @@ func TestSyncCopyDest(t *testing.T) { r.CheckLocalItems(t, file1c, file5) accounting.GlobalStats().ResetCounters() - ctx = predictDstFromLogger(ctx) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // ctx = predictDstFromLogger(ctx) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) err = Sync(ctx, fdst, r.Flocal, false) require.NoError(t, err) @@ -1890,9 +1890,9 @@ func TestSyncCopyDest(t *testing.T) { // check new dest, new copy accounting.GlobalStats().ResetCounters() - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = Sync(ctx, fdst, r.Flocal, false) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) require.NoError(t, err) r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) @@ -1904,9 +1904,9 @@ func TestSyncCopyDest(t *testing.T) { r.CheckLocalItems(t, file1c, file5, file7) accounting.GlobalStats().ResetCounters() - ctx = predictDstFromLogger(ctx) + // ctx = predictDstFromLogger(ctx) err = Sync(ctx, fdst, r.Flocal, false) - testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) require.NoError(t, err) file7dst := file7 @@ -2312,17 +2312,19 @@ func predictDstFromLogger(ctx context.Context) context.Context { file := winner.Obj obj, ok := file.(fs.ObjectInfo) checksum := "" + timeFormat := "2006-01-02 15:04:05" if ok { if obj.Fs().Hashes().GetOne() == hash.MD5 { // skip if no MD5 - checksum, _ = obj.Hash(ctx, obj.Fs().Hashes().GetOne()) + checksum, _ = obj.Hash(ctx, hash.MD5) } + timeFormat = operations.FormatForLSFPrecision(obj.Fs().Precision()) } errMsg := "" if winner.Err != nil { errMsg = ";" + winner.Err.Error() } - operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), checksum, file.Size(), file.Remote(), errMsg) + operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg) } } return operations.WithSyncLogger(ctx, opt) @@ -2407,10 +2409,10 @@ func testLoggerVsLsf(ctx context.Context, Fremote fs.Fs, logger *bytes.Buffer, t newlogger.Write(logger.Bytes()) } - lsf := DstLsf(ctx, Fremote) - err := LoggerMatchesLsf(&newlogger, lsf) r := fstest.NewRun(t) if r.Flocal.Precision() == Fremote.Precision() && r.Flocal.Hashes().Contains(hash.MD5) && canTestHash { + lsf := DstLsf(ctx, Fremote) + err := LoggerMatchesLsf(&newlogger, lsf) require.NoError(t, err) } }