From 7048cc3e58f7db03a2c7ac6a107297b2474454b2 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Tue, 13 Aug 2019 18:27:20 +0200 Subject: [PATCH] Add copy functionality Add a copy command to copy snapshots between repositories. It allows the user to specify a destination repository, password, password-file, password-command or key-hint to supply the necessary details to open the destination repository. You need to supply a list of snapshots to copy, snapshots which already exist in the destination repository will be skipped. Note, when using the network this becomes rather slow, as it needs to read the blocks, decrypt them using the source key, then encrypt them again using the destination key before finally writing them out to the destination repository. --- changelog/unreleased/issue-323 | 13 +++ cmd/restic/cmd_copy.go | 203 +++++++++++++++++++++++++++++++++ cmd/restic/global.go | 4 +- cmd/restic/main.go | 2 +- 4 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/issue-323 create mode 100644 cmd/restic/cmd_copy.go diff --git a/changelog/unreleased/issue-323 b/changelog/unreleased/issue-323 new file mode 100644 index 000000000..6b3b56b31 --- /dev/null +++ b/changelog/unreleased/issue-323 @@ -0,0 +1,13 @@ +Enhancement: Add command for copying snapshots between repositories + +We've added a copy command, allowing you to copy snapshots from one +repository to another. + +Note that this process will have to read (download) and write (upload) the +entire snapshot(s) due to the different encryption keys used on the source +and destination repository. Also, the transferred files are not re-chunked, +which may break deduplication between files already stored in the +destination repo and files copied there using this command. + +https://github.com/restic/restic/issues/323 +https://github.com/restic/restic/pull/2606 diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go new file mode 100644 index 000000000..30545e27b --- /dev/null +++ b/cmd/restic/cmd_copy.go @@ -0,0 +1,203 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + + "github.com/spf13/cobra" +) + +var cmdCopy = &cobra.Command{ + Use: "copy [flags] [snapshotID ...]", + Short: "Copy snapshots from one repository to another", + Long: ` +The "copy" command copies one or more snapshots from one repository to another +repository. Note that this will have to read (download) and write (upload) the +entire snapshot(s) due to the different encryption keys on the source and +destination, and that transferred files are not re-chunked, which may break +their deduplication. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runCopy(copyOptions, globalOptions, args) + }, +} + +// CopyOptions bundles all options for the copy command. +type CopyOptions struct { + Repo string + PasswordFile string + PasswordCommand string + KeyHint string + Hosts []string + Tags restic.TagLists + Paths []string +} + +var copyOptions CopyOptions + +func init() { + cmdRoot.AddCommand(cmdCopy) + + f := cmdCopy.Flags() + f.StringVarP(©Options.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), "destination repository to copy snapshots to (default: $RESTIC_REPOSITORY2)") + f.StringVarP(©Options.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "read the destination repository password from a file (default: $RESTIC_PASSWORD_FILE2)") + f.StringVarP(©Options.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the destination repository first (default: $RESTIC_KEY_HINT2)") + f.StringVarP(©Options.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "specify a shell command to obtain a password for the destination repository (default: $RESTIC_PASSWORD_COMMAND2)") + + f.StringArrayVarP(©Options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)") + f.Var(©Options.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given") + f.StringArrayVar(©Options.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given") +} + +func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { + if opts.Repo == "" { + return errors.Fatal("Please specify a destination repository location (--repo2)") + } + var err error + dstGopts := gopts + dstGopts.Repo = opts.Repo + dstGopts.PasswordFile = opts.PasswordFile + dstGopts.PasswordCommand = opts.PasswordCommand + dstGopts.KeyHint = opts.KeyHint + dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2") + if err != nil { + return err + } + dstGopts.password, err = ReadPassword(dstGopts, "enter password for destination repository: ") + if err != nil { + return err + } + + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + + srcRepo, err := OpenRepository(gopts) + if err != nil { + return err + } + + dstRepo, err := OpenRepository(dstGopts) + if err != nil { + return err + } + + srcLock, err := lockRepo(srcRepo) + defer unlockRepo(srcLock) + if err != nil { + return err + } + + dstLock, err := lockRepo(dstRepo) + defer unlockRepo(dstLock) + if err != nil { + return err + } + + debug.Log("Loading source index") + if err := srcRepo.LoadIndex(ctx); err != nil { + return err + } + + debug.Log("Loading destination index") + if err := dstRepo.LoadIndex(ctx); err != nil { + return err + } + + for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { + Verbosef("snapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef(" copy started, this may take a while...\n") + + if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree); err != nil { + return err + } + debug.Log("tree copied") + + if err = dstRepo.Flush(ctx); err != nil { + return err + } + debug.Log("flushed packs") + + err = dstRepo.SaveIndex(ctx) + if err != nil { + debug.Log("error saving index: %v", err) + return err + } + debug.Log("saved index") + + // save snapshot + sn.Parent = nil // Parent does not have relevance in the new repo. + sn.Original = nil // Original does not have relevance in the new repo. + newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + if err != nil { + return err + } + Verbosef("snapshot %s saved\n", newID.Str()) + } + return nil +} + +func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID) error { + tree, err := srcRepo.LoadTree(ctx, treeID) + if err != nil { + return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err) + } + // Do we already have this tree blob? + if !dstRepo.Index().Has(treeID, restic.TreeBlob) { + newTreeID, err := dstRepo.SaveTree(ctx, tree) + if err != nil { + return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err) + } + // Assurance only. + if newTreeID != treeID { + return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str()) + } + } + + // TODO: keep only one (big) buffer around. + // TODO: parellize this stuff, likely only needed inside a tree. + + for _, entry := range tree.Nodes { + // If it is a directory, recurse + if entry.Type == "dir" && entry.Subtree != nil { + if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree); err != nil { + return err + } + } + // Copy the blobs for this file. + for _, blobID := range entry.Content { + // Do we already have this data blob? + if dstRepo.Index().Has(blobID, restic.DataBlob) { + continue + } + debug.Log("Copying blob %s\n", blobID.Str()) + size, found := srcRepo.LookupBlobSize(blobID, restic.DataBlob) + if !found { + return fmt.Errorf("LookupBlobSize(%v) failed", blobID) + } + buf := restic.NewBlobBuffer(int(size)) + n, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf) + if err != nil { + return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err) + } + if n != len(buf) { + return fmt.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n) + } + + newBlobID, err := dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID) + if err != nil { + return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err) + } + // Assurance only. + if newBlobID != blobID { + return fmt.Errorf("SaveBlob(%v) returned unexpected id %s", blobID.Str(), newBlobID.Str()) + } + } + } + + return nil +} diff --git a/cmd/restic/global.go b/cmd/restic/global.go index d3b264b3d..bc34b5b98 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -274,7 +274,7 @@ func Exitf(exitcode int, format string, args ...interface{}) { } // resolvePassword determines the password to be used for opening the repository. -func resolvePassword(opts GlobalOptions) (string, error) { +func resolvePassword(opts GlobalOptions, envStr string) (string, error) { if opts.PasswordFile != "" && opts.PasswordCommand != "" { return "", errors.Fatalf("Password file and command are mutually exclusive options") } @@ -299,7 +299,7 @@ func resolvePassword(opts GlobalOptions) (string, error) { return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") } - if pwd := os.Getenv("RESTIC_PASSWORD"); pwd != "" { + if pwd := os.Getenv(envStr); pwd != "" { return pwd, nil } diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 2f553c9cc..2e75575b3 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -54,7 +54,7 @@ directories in an encrypted repository stored on different backends. if c.Name() == "version" { return nil } - pwd, err := resolvePassword(globalOptions) + pwd, err := resolvePassword(globalOptions, "RESTIC_PASSWORD") if err != nil { fmt.Fprintf(os.Stderr, "Resolving password failed: %v\n", err) Exit(1)