potyarkin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

sync: report list of synced paths to file -- see #7282

Browse source 
Allows rclone sync to accept the same output file flags as rclone check,
for the purpose of writing results to a file.
A new --dest-after option is also supported, which writes a list file using
the same ListFormat flags as lsf (including customizable options for hash,
modtime, etc.) Conceptually it is similar to rsync's --itemize-changes, but
not identical -- it should output an accurate list of what will be on the
destination after the sync.

Note that it has a few limitations, and certain scenarios
are not currently supported:

--max-duration / CutoffModeHard
--compare-dest / --copy-dest (because equal() is called multiple times for the
    same file)
server-side moves of an entire dir at once (because we never get the individual
file objects in the dir)
High-level retries, because there would be dupes
Possibly some error scenarios that didn't come up on the tests

Note also that each file is logged during the sync, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)

Only rclone sync is currently supported -- support for copy and move may be
added in the future.
This commit is contained in:
nielash 2023-11-06 02:45:51 -05:00
parent c0968a0987
commit 3a50f35df9
10 changed files with 470 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

23
fs/operations/logger.go
View file

@ -8,7 +8,6 @@ import (
"io"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
"github.com/spf13/pflag"
)
@ -107,28 +106,6 @@ type LoggerOpt struct {
Absolute bool
}
// AddLoggerFlags adds the logger flags to the cmdFlags command
func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *LoggerOpt, combined, missingOnSrc, missingOnDst, match, differ, errFile, destAfter *string) {
flags.StringVarP(cmdFlags, combined, "combined", "", *combined, "Make a combined report of changes to this file", "Sync")
flags.StringVarP(cmdFlags, missingOnSrc, "missing-on-src", "", *missingOnSrc, "Report all files missing from the source to this file", "Sync")
flags.StringVarP(cmdFlags, missingOnDst, "missing-on-dst", "", *missingOnDst, "Report all files missing from the destination to this file", "Sync")
flags.StringVarP(cmdFlags, match, "match", "", *match, "Report all matching files to this file", "Sync")
flags.StringVarP(cmdFlags, differ, "differ", "", *differ, "Report all non-matching files to this file", "Sync")
flags.StringVarP(cmdFlags, errFile, "error", "", *errFile, "Report all files with errors (hashing or reading) to this file", "Sync")
flags.StringVarP(cmdFlags, destAfter, "dest-after", "", *destAfter, "Report all files that exist on the dest post-sync", "Sync")
// lsf flags for destAfter
flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")
flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")
flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")
flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")
flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")
flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")
flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")
flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
}
// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
return context.WithValue(ctx, loggerKey, logger)