hasher: add hasher backend #5587

backend/hasher/commands.go

@@ -0,0 +1,179 @@
+package hasher
+
+import (
+	"context"
+	"path"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/fspath"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/lib/kv"
+)
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "drop":
+		return nil, f.db.Stop(true)
+	case "dump", "fulldump":
+		return nil, f.dbDump(ctx, name == "fulldump", "")
+	case "import", "stickyimport":
+		sticky := name == "stickyimport"
+		if len(arg) != 2 {
+			return nil, errors.New("please provide checksum type and path to sum file")
+		}
+		return nil, f.dbImport(ctx, arg[0], arg[1], sticky)
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
+var commandHelp = []fs.CommandHelp{{
+	Name:  "drop",
+	Short: "Drop cache",
+	Long: `Completely drop checksum cache.
+Usage Example:
+    rclone backend drop hasher:
+`,
+}, {
+	Name:  "dump",
+	Short: "Dump the database",
+	Long:  "Dump cache records covered by the current remote",
+}, {
+	Name:  "fulldump",
+	Short: "Full dump of the database",
+	Long:  "Dump all cache records in the database",
+}, {
+	Name:  "import",
+	Short: "Import a SUM file",
+	Long: `Amend hash cache from a SUM file and bind checksums to files by size/time.
+Usage Example:
+    rclone backend import hasher:subdir md5 /path/to/sum.md5
+`,
+}, {
+	Name:  "stickyimport",
+	Short: "Perform fast import of a SUM file",
+	Long: `Fill hash cache from a SUM file without verifying file fingerprints.
+Usage Example:
+    rclone backend stickyimport hasher:subdir md5 remote:path/to/sum.md5
+`,
+}}
+
+func (f *Fs) dbDump(ctx context.Context, full bool, root string) error {
+	if root == "" {
+		remoteFs, err := cache.Get(ctx, f.opt.Remote)
+		if err != nil {
+			return err
+		}
+		root = fspath.JoinRootPath(remoteFs.Root(), f.Root())
+	}
+	op := &kvDump{
+		full: full,
+		root: root,
+		path: f.db.Path(),
+		fs:   f,
+	}
+	err := f.db.Do(false, op)
+	if err == kv.ErrEmpty {
+		fs.Infof(op.path, "empty")
+		err = nil
+	}
+	return err
+}
+
+func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bool) error {
+	var hashType hash.Type
+	if err := hashType.Set(hashName); err != nil {
+		return err
+	}
+	if hashType == hash.None {
+		return errors.New("please provide a valid hash type")
+	}
+	if !f.suppHashes.Contains(hashType) {
+		return errors.New("unsupported hash type")
+	}
+	if !f.keepHashes.Contains(hashType) {
+		fs.Infof(nil, "Need not import hashes of this type")
+		return nil
+	}
+
+	_, sumPath, err := fspath.SplitFs(sumRemote)
+	if err != nil {
+		return err
+	}
+	sumFs, err := cache.Get(ctx, sumRemote)
+	switch err {
+	case fs.ErrorIsFile:
+		// ok
+	case nil:
+		return errors.Errorf("not a file: %s", sumRemote)
+	default:
+		return err
+	}
+
+	sumObj, err := sumFs.NewObject(ctx, path.Base(sumPath))
+	if err != nil {
+		return errors.Wrap(err, "cannot open sum file")
+	}
+	hashes, err := operations.ParseSumFile(ctx, sumObj)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse sum file")
+	}
+
+	if sticky {
+		rootPath := f.Fs.Root()
+		for remote, hashVal := range hashes {
+			key := path.Join(rootPath, remote)
+			hashSums := operations.HashSums{hashName: hashVal}
+			if err := f.putRawHashes(ctx, key, anyFingerprint, hashSums); err != nil {
+				fs.Errorf(nil, "%s: failed to import: %v", remote, err)
+			}
+		}
+		fs.Infof(nil, "Summary: %d checksum(s) imported", len(hashes))
+		return nil
+	}
+
+	const longImportThreshold = 100
+	if len(hashes) > longImportThreshold {
+		fs.Infof(nil, "Importing %d checksums. Please wait...", len(hashes))
+	}
+
+	doneCount := 0
+	err = operations.ListFn(ctx, f, func(obj fs.Object) {
+		remote := obj.Remote()
+		hash := hashes[remote]
+		hashes[remote] = "" // mark as handled
+		o, ok := obj.(*Object)
+		if ok && hash != "" {
+			if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
+				fs.Errorf(nil, "%s: failed to import: %v", remote, err)
+			}
+			accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
+			doneCount++
+		}
+	})
+	if err != nil {
+		fs.Errorf(nil, "Import failed: %v", err)
+	}
+	skipCount := 0
+	for remote, emptyOrDone := range hashes {
+		if emptyOrDone != "" {
+			fs.Infof(nil, "Skip vanished object: %s", remote)
+			skipCount++
+		}
+	}
+	fs.Infof(nil, "Summary: %d imported, %d skipped", doneCount, skipCount)
+	return err
+}