package hasher import ( "context" "path" "github.com/pkg/errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/lib/kv" ) // Command the backend to run a named command // // The command run is name // args may be used to read arguments from // opts may be used to read optional arguments from // // The result should be capable of being JSON encoded // If it is a string or a []string it will be shown to the user // otherwise it will be JSON encoded and shown to the user like that func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) { switch name { case "drop": return nil, f.db.Stop(true) case "dump", "fulldump": return nil, f.dbDump(ctx, name == "fulldump", "") case "import", "stickyimport": sticky := name == "stickyimport" if len(arg) != 2 { return nil, errors.New("please provide checksum type and path to sum file") } return nil, f.dbImport(ctx, arg[0], arg[1], sticky) default: return nil, fs.ErrorCommandNotFound } } var commandHelp = []fs.CommandHelp{{ Name: "drop", Short: "Drop cache", Long: `Completely drop checksum cache. Usage Example: rclone backend drop hasher: `, }, { Name: "dump", Short: "Dump the database", Long: "Dump cache records covered by the current remote", }, { Name: "fulldump", Short: "Full dump of the database", Long: "Dump all cache records in the database", }, { Name: "import", Short: "Import a SUM file", Long: `Amend hash cache from a SUM file and bind checksums to files by size/time. Usage Example: rclone backend import hasher:subdir md5 /path/to/sum.md5 `, }, { Name: "stickyimport", Short: "Perform fast import of a SUM file", Long: `Fill hash cache from a SUM file without verifying file fingerprints. Usage Example: rclone backend stickyimport hasher:subdir md5 remote:path/to/sum.md5 `, }} func (f *Fs) dbDump(ctx context.Context, full bool, root string) error { if root == "" { remoteFs, err := cache.Get(ctx, f.opt.Remote) if err != nil { return err } root = fspath.JoinRootPath(remoteFs.Root(), f.Root()) } op := &kvDump{ full: full, root: root, path: f.db.Path(), fs: f, } err := f.db.Do(false, op) if err == kv.ErrEmpty { fs.Infof(op.path, "empty") err = nil } return err } func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bool) error { var hashType hash.Type if err := hashType.Set(hashName); err != nil { return err } if hashType == hash.None { return errors.New("please provide a valid hash type") } if !f.suppHashes.Contains(hashType) { return errors.New("unsupported hash type") } if !f.keepHashes.Contains(hashType) { fs.Infof(nil, "Need not import hashes of this type") return nil } _, sumPath, err := fspath.SplitFs(sumRemote) if err != nil { return err } sumFs, err := cache.Get(ctx, sumRemote) switch err { case fs.ErrorIsFile: // ok case nil: return errors.Errorf("not a file: %s", sumRemote) default: return err } sumObj, err := sumFs.NewObject(ctx, path.Base(sumPath)) if err != nil { return errors.Wrap(err, "cannot open sum file") } hashes, err := operations.ParseSumFile(ctx, sumObj) if err != nil { return errors.Wrap(err, "failed to parse sum file") } if sticky { rootPath := f.Fs.Root() for remote, hashVal := range hashes { key := path.Join(rootPath, remote) hashSums := operations.HashSums{hashName: hashVal} if err := f.putRawHashes(ctx, key, anyFingerprint, hashSums); err != nil { fs.Errorf(nil, "%s: failed to import: %v", remote, err) } } fs.Infof(nil, "Summary: %d checksum(s) imported", len(hashes)) return nil } const longImportThreshold = 100 if len(hashes) > longImportThreshold { fs.Infof(nil, "Importing %d checksums. Please wait...", len(hashes)) } doneCount := 0 err = operations.ListFn(ctx, f, func(obj fs.Object) { remote := obj.Remote() hash := hashes[remote] hashes[remote] = "" // mark as handled o, ok := obj.(*Object) if ok && hash != "" { if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil { fs.Errorf(nil, "%s: failed to import: %v", remote, err) } accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err) doneCount++ } }) if err != nil { fs.Errorf(nil, "Import failed: %v", err) } skipCount := 0 for remote, emptyOrDone := range hashes { if emptyOrDone != "" { fs.Infof(nil, "Skip vanished object: %s", remote) skipCount++ } } fs.Infof(nil, "Summary: %d imported, %d skipped", doneCount, skipCount) return err }