From d19ee919604026c176583a63e7eaf572de64e379 Mon Sep 17 00:00:00 2001 From: nielash Date: Mon, 18 Dec 2023 11:47:46 -0500 Subject: [PATCH] check and cryptcheck: report directory differences with --report-dirs - fixes #6440 Before this change, check and cryptcheck would not inform the user of directory differences (for example, an empty directory that exists only on the src.) After this change, a new --report-dirs flag allows alerting users of such differences. Note that directories will be reported regardless of whether they are empty. They will be included in the total error count, but counted and summarized separately, and not included in output files. --report-dirs is ignored when --checkfile is in use. --- cmd/check/check.go | 9 ++++++--- cmd/cryptcheck/cryptcheck.go | 2 ++ fs/operations/check.go | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/cmd/check/check.go b/cmd/check/check.go index f01307c18..90b6bd492 100644 --- a/cmd/check/check.go +++ b/cmd/check/check.go @@ -21,6 +21,7 @@ import ( var ( download = false oneway = false + ReportDirs = false combined = "" missingOnSrc = "" missingOnDst = "" @@ -41,6 +42,7 @@ func init() { // AddFlags adds the check flags to the cmdFlags command func AddFlags(cmdFlags *pflag.FlagSet) { flags.BoolVarP(cmdFlags, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote", "") + flags.BoolVarP(cmdFlags, &ReportDirs, "report-dirs", "", ReportDirs, "Report directory differences in addition to files", "") flags.StringVarP(cmdFlags, &combined, "combined", "", combined, "Make a combined report of changes to this file", "") flags.StringVarP(cmdFlags, &missingOnSrc, "missing-on-src", "", missingOnSrc, "Report all files missing from the source to this file", "") flags.StringVarP(cmdFlags, &missingOnDst, "missing-on-dst", "", missingOnDst, "Report all files missing from the destination to this file", "") @@ -82,9 +84,10 @@ func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err closers := []io.Closer{} opt = &operations.CheckOpt{ - Fsrc: fsrc, - Fdst: fdst, - OneWay: oneway, + Fsrc: fsrc, + Fdst: fdst, + OneWay: oneway, + ReportDirs: ReportDirs, } open := func(name string, pout *io.Writer) error { diff --git a/cmd/cryptcheck/cryptcheck.go b/cmd/cryptcheck/cryptcheck.go index 63a0e32c1..71861f542 100644 --- a/cmd/cryptcheck/cryptcheck.go +++ b/cmd/cryptcheck/cryptcheck.go @@ -80,6 +80,8 @@ func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error { } defer close() + opt.ReportDirs = check.ReportDirs + // checkIdentical checks to see if dst and src are identical // // it returns true if differences were found diff --git a/fs/operations/check.go b/fs/operations/check.go index b9fe36aab..babdb7961 100644 --- a/fs/operations/check.go +++ b/fs/operations/check.go @@ -38,6 +38,7 @@ type CheckOpt struct { Fdst, Fsrc fs.Fs // fses to check Check checkFn // function to use for checking OneWay bool // one way only? + ReportDirs bool // report dir differences in addition to files Combined io.Writer // a file with file names with leading sigils MissingOnSrc io.Writer // files only in the destination MissingOnDst io.Writer // files only in the source @@ -56,6 +57,8 @@ type checkMarch struct { noHashes atomic.Int32 srcFilesMissing atomic.Int32 dstFilesMissing atomic.Int32 + srcDirsMissing atomic.Int32 + dstDirsMissing atomic.Int32 matches atomic.Int32 opt CheckOpt } @@ -92,6 +95,13 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) { if c.opt.OneWay { return false } + if c.opt.ReportDirs { + err := fmt.Errorf("directory not in %v", c.opt.Fsrc) + fs.Errorf(dst, "%v", err) + _ = fs.CountError(err) + c.differences.Add(1) + c.srcDirsMissing.Add(1) + } return true default: panic("Bad object in DirEntries") @@ -111,6 +121,13 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) { c.report(src, c.opt.MissingOnDst, '+') case fs.Directory: // Do the same thing to the entire contents of the directory + if c.opt.ReportDirs { + err := fmt.Errorf("directory not in %v", c.opt.Fdst) + fs.Errorf(src, "%v", err) + _ = fs.CountError(err) + c.differences.Add(1) + c.dstDirsMissing.Add(1) + } return true default: panic("Bad object in DirEntries") @@ -184,8 +201,11 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b } case fs.Directory: // Do the same thing to the entire contents of the directory - _, ok := dst.(fs.Directory) + dstX, ok := dst.(fs.Directory) if ok { + if c.opt.ReportDirs { + fs.Debugf(dstX, "OK (no hash to check for directories)") + } return true } err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc) @@ -246,6 +266,14 @@ func (c *checkMarch) reportResults(ctx context.Context, err error) error { } fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity) } + if c.opt.ReportDirs { + if c.dstDirsMissing.Load() > 0 { + fs.Logf(c.opt.Fdst, "%d directories missing", c.dstDirsMissing.Load()) + } + if c.srcDirsMissing.Load() > 0 { + fs.Logf(c.opt.Fsrc, "%d directories missing", c.srcDirsMissing.Load()) + } + } fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors()) if errs := accounting.Stats(ctx).GetErrors(); errs > 0 { @@ -416,6 +444,10 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha return fmt.Errorf("%s: hash type is not supported by file system: %s", hashType, opt.Fdst) } + if options.ReportDirs { + fs.Logf(nil, "ignoring --report-dirs as --checkfile is in use.") + } + if sumFile == "" { return fmt.Errorf("not a sum file: %s", fsum) }