From 3468108d4c2066647bdfbd6bf9e55049ec949cee Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 19:55:58 +0100 Subject: [PATCH 01/10] Implement --tag processing to `backup` and `restore` command Add `tags` argument to `FindLatestSnapshot()` --- src/cmds/restic/cmd_backup.go | 6 +++--- src/cmds/restic/cmd_restore.go | 4 +++- src/restic/snapshot.go | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go index ee4a3e61d..d98e7e56d 100644 --- a/src/cmds/restic/cmd_backup.go +++ b/src/cmds/restic/cmd_backup.go @@ -67,12 +67,12 @@ func init() { f := cmdBackup.Flags() f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) - f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)") + f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file") f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin") - f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)") + f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)") f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually") f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)") } @@ -391,7 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { // Find last snapshot to set it as parent, if not already set if !opts.Force && parentSnapshotID == nil { - id, err := restic.FindLatestSnapshot(repo, target, opts.Hostname) + id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname) if err == nil { parentSnapshotID = &id } else if err != restic.ErrNoSnapshotFound { diff --git a/src/cmds/restic/cmd_restore.go b/src/cmds/restic/cmd_restore.go index 8a3e0757e..6a9ec953d 100644 --- a/src/cmds/restic/cmd_restore.go +++ b/src/cmds/restic/cmd_restore.go @@ -31,6 +31,7 @@ type RestoreOptions struct { Target string Host string Paths []string + Tags []string } var restoreOptions RestoreOptions @@ -44,6 +45,7 @@ func init() { flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`) + flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"") flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"") } @@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { var id restic.ID if snapshotIDString == "latest" { - id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host) + id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host) if err != nil { Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host) } diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go index ed89a60ca..343dfe411 100644 --- a/src/restic/snapshot.go +++ b/src/restic/snapshot.go @@ -177,8 +177,8 @@ func (sn *Snapshot) SamePaths(paths []string) bool { // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. var ErrNoSnapshotFound = errors.New("no snapshot found") -// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters. -func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) { +// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters. +func FindLatestSnapshot(repo Repository, targets []string, tags []string, hostname string) (ID, error) { var ( latest time.Time latestID ID @@ -190,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, if err != nil { return ID{}, errors.Errorf("Error listing snapshot: %v", err) } - if snapshot.Time.After(latest) && snapshot.HasPaths(targets) && (hostname == "" || hostname == snapshot.Hostname) { + if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) { latest = snapshot.Time latestID = snapshotID found = true From b1c8071163398c744cb02f5736f4d87f69f442b3 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 19:59:19 +0100 Subject: [PATCH 02/10] Add filtering to `mount` command --- src/cmds/restic/cmd_mount.go | 16 ++++++++++++---- src/restic/fuse/snapshot.go | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/cmds/restic/cmd_mount.go b/src/cmds/restic/cmd_mount.go index a4b8340cd..20ce5dec3 100644 --- a/src/cmds/restic/cmd_mount.go +++ b/src/cmds/restic/cmd_mount.go @@ -35,6 +35,9 @@ type MountOptions struct { OwnerRoot bool AllowRoot bool AllowOther bool + Host string + Tags []string + Paths []string } var mountOptions MountOptions @@ -42,9 +45,14 @@ var mountOptions MountOptions func init() { cmdRoot.AddCommand(cmdMount) - cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs") - cmdMount.Flags().BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory") - cmdMount.Flags().BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") + mountFlags := cmdMount.Flags() + mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs") + mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory") + mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") + + mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`) + mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`") + mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") } func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error { @@ -91,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error { Printf("Don't forget to umount after quitting!\n") root := fs.Tree{} - root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot)) + root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host)) debug.Log("serving mount at %v", mountpoint) err = fs.Serve(c, &root) diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go index 1e1092dea..2a654397b 100644 --- a/src/restic/fuse/snapshot.go +++ b/src/restic/fuse/snapshot.go @@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{}) type SnapshotsDir struct { repo restic.Repository ownerIsRoot bool + paths []string + tags []string + host string // knownSnapshots maps snapshot timestamp to the snapshot sync.RWMutex @@ -40,12 +43,15 @@ type SnapshotsDir struct { } // NewSnapshotsDir returns a new dir object for the snapshots. -func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir { +func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir { debug.Log("fuse mount initiated") return &SnapshotsDir{ repo: repo, - knownSnapshots: make(map[string]SnapshotWithId), ownerIsRoot: ownerIsRoot, + paths: paths, + tags: tags, + host: host, + knownSnapshots: make(map[string]SnapshotWithId), processed: restic.NewIDSet(), } } @@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error { return err } + // Filter snapshots we don't care for. + if (sn.host != "" && sn.host != snapshot.Hostname) || + !snapshot.HasTags(sn.tags) || + !snapshot.HasPaths(sn.paths) { + continue + } + timestamp := snapshot.Time.Format(time.RFC3339) for i := 1; ; i++ { if _, ok := sn.knownSnapshots[timestamp]; !ok { From 3eaaa0f286ff61997f4a80cc7867be92cca2761b Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:09:24 +0100 Subject: [PATCH 03/10] Correct some typo's in comments. --- src/cmds/restic/cmd_check.go | 2 +- src/cmds/restic/cmd_find.go | 2 +- src/cmds/restic/cmd_snapshots.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmds/restic/cmd_check.go b/src/cmds/restic/cmd_check.go index cd041a42c..2f0064f1a 100644 --- a/src/cmds/restic/cmd_check.go +++ b/src/cmds/restic/cmd_check.go @@ -24,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore. }, } -// CheckOptions bundle all options for the 'check' command. +// CheckOptions bundles all options for the 'check' command. type CheckOptions struct { ReadData bool CheckUnused bool diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index cb9e6f594..26ae92ad4 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -24,7 +24,7 @@ repo. `, }, } -// FindOptions bundle all options for the find command. +// FindOptions bundles all options for the find command. type FindOptions struct { Oldest string Newest string diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go index 5c7a671b9..7ff52831d 100644 --- a/src/cmds/restic/cmd_snapshots.go +++ b/src/cmds/restic/cmd_snapshots.go @@ -23,7 +23,7 @@ The "snapshots" command lists all snapshots stored in the repository. }, } -// SnapshotOptions bundle all options for the snapshots command. +// SnapshotOptions bundles all options for the snapshots command. type SnapshotOptions struct { Host string Paths []string From b4526c4e6e987a665b098fc18ced2e76077a1379 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:12:16 +0100 Subject: [PATCH 04/10] Enable the use of `context` in restic Set up a cancelble context in global options, hook it into the ctrl-C handler for proper cancel propegation. Bump up minimal requirement for Go to version 1.7 in documentation and test-build files. --- .travis.yml | 3 --- CONTRIBUTING.md | 2 +- README.md | 2 +- Vagrantfile | 2 +- doc/Manual.md | 2 +- src/cmds/restic/global.go | 9 +++++++++ src/cmds/restic/integration_helpers_test.go | 2 ++ 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2d6b6bb9..570423172 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: go sudo: false go: - - 1.6.4 - 1.7.5 - 1.8 - tip @@ -17,8 +16,6 @@ env: matrix: exclude: - - os: osx - go: 1.6.4 - os: osx go: 1.7.5 - os: osx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5529ea7f7..5ffdc40d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary: [...] $ bin/restic version restic compiled manually - compiled at unknown time with go1.6 + compiled at unknown time with go1.7 The following commands can be used to run all the tests: diff --git a/README.md b/README.md index cc90f7b6c..3fe4c0edb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ You can download the latest pre-compiled binary from the [restic release page](h Build restic ============ -Install Go/Golang (at least version 1.6), then run `go run build.go`, +Install Go/Golang (at least version 1.7), then run `go run build.go`, afterwards you'll find the binary in the current directory: $ go run build.go diff --git a/Vagrantfile b/Vagrantfile index a26aa6b61..ee872b3f4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -GO_VERSION = '1.6' +GO_VERSION = '1.7' def packages_freebsd return <<-EOF diff --git a/doc/Manual.md b/doc/Manual.md index 513ef612c..5f2a470ff 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -27,7 +27,7 @@ $ pacaur -S restic-git # Building restic -restic is written in the Go programming language and you need at least Go version 1.6. +restic is written in the Go programming language and you need at least Go version 1.7. Building restic may also work with older versions of Go, but that's not supported. See the [Getting started](https://golang.org/doc/install) guide of the Go project for instructions how to install Go. diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 02a2e9b52..4acd79069 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "io/ioutil" @@ -33,6 +34,7 @@ type GlobalOptions struct { NoLock bool JSON bool + ctx context.Context password string stdout io.Writer stderr io.Writer @@ -49,6 +51,13 @@ func init() { globalOptions.password = pw } + var cancel context.CancelFunc + globalOptions.ctx, cancel = context.WithCancel(context.Background()) + AddCleanupHandler(func() error { + cancel() + return nil + }) + f := cmdRoot.PersistentFlags() f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)") f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file") diff --git a/src/cmds/restic/integration_helpers_test.go b/src/cmds/restic/integration_helpers_test.go index c71b12067..ad6acc8a1 100644 --- a/src/cmds/restic/integration_helpers_test.go +++ b/src/cmds/restic/integration_helpers_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io/ioutil" "os" @@ -194,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) gopts := GlobalOptions{ Repo: env.repo, Quiet: true, + ctx: context.Background(), password: TestPassword, stdout: os.Stdout, stderr: os.Stderr, From 0f7b6ec5ac68be323f6b8134a59c7b6cb3420723 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:17:30 +0100 Subject: [PATCH 05/10] Adapt `key` command to context world. --- src/cmds/restic/cmd_key.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cmds/restic/cmd_key.go b/src/cmds/restic/cmd_key.go index 7e51def40..052dd5b8a 100644 --- a/src/cmds/restic/cmd_key.go +++ b/src/cmds/restic/cmd_key.go @@ -1,13 +1,13 @@ package main import ( + "context" "fmt" "restic" - - "github.com/spf13/cobra" - "restic/errors" "restic/repository" + + "github.com/spf13/cobra" ) var cmdKey = &cobra.Command{ @@ -25,15 +25,12 @@ func init() { cmdRoot.AddCommand(cmdKey) } -func listKeys(s *repository.Repository) error { +func listKeys(ctx context.Context, s *repository.Repository) error { tab := NewTable() tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created") tab.RowFormat = "%s%-10s %-10s %-10s %s" - done := make(chan struct{}) - defer close(done) - - for id := range s.List(restic.KeyFile, done) { + for id := range s.List(restic.KeyFile, ctx.Done()) { k, err := repository.LoadKey(s, id.String()) if err != nil { Warnf("LoadKey() failed: %v\n", err) @@ -124,6 +121,9 @@ func runKey(gopts GlobalOptions, args []string) error { return errors.Fatal("wrong number of arguments") } + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + repo, err := OpenRepository(gopts) if err != nil { return err @@ -137,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error { return err } - return listKeys(repo) + return listKeys(ctx, repo) case "add": lock, err := lockRepo(repo) defer unlockRepo(lock) From 11d237c2528d6996cdbefdb81383748923cec79d Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:19:12 +0100 Subject: [PATCH 06/10] New helper function `FindFilteredSnapshots` to iterate over snapshots This helper function takes a set of filters and/or a list of snapshots from the commandline. It returns a channel of *Snapshot. When snapshot ids are given, they are checked for validity and their corresponding Snapshots returned. The snapshot id "latest" is handled special to return either the last snapshot (no filters) or the last snapshot matching the filters. When no arguments are given, the filters are applied over all available snapshots and these are returned. --- src/cmds/restic/find.go | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/cmds/restic/find.go diff --git a/src/cmds/restic/find.go b/src/cmds/restic/find.go new file mode 100644 index 000000000..fa9d71694 --- /dev/null +++ b/src/cmds/restic/find.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + + "restic" + "restic/repository" +) + +// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. +func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { + out := make(chan *restic.Snapshot) + go func() { + defer close(out) + if len(snapshotIDs) != 0 { + var ( + id restic.ID + usedFilter bool + err error + ) + ids := make(restic.IDs, 0, len(snapshotIDs)) + // Process all snapshot IDs given as arguments. + for _, s := range snapshotIDs { + if s == "latest" { + id, err = restic.FindLatestSnapshot(repo, paths, tags, host) + if err != nil { + Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host) + usedFilter = true + continue + } + } else { + id, err = restic.FindSnapshot(repo, s) + if err != nil { + Warnf("Ignoring %q, it is not a snapshot id\n", s) + continue + } + } + ids = append(ids, id) + } + + // Give the user some indication their filters are not used. + if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) { + Warnf("Ignoring filters as there are explicit snapshot ids given\n") + } + + for _, id := range ids.Uniq() { + sn, err := restic.LoadSnapshot(repo, id) + if err != nil { + Warnf("Ignoring %q, could not load snapshot: %v\n", id, err) + continue + } + select { + case <-ctx.Done(): + return + case out <- sn: + } + } + return + } + + for id := range repo.List(restic.SnapshotFile, ctx.Done()) { + sn, err := restic.LoadSnapshot(repo, id) + if err != nil { + Warnf("Ignoring %q, could not load snapshot: %v\n", id, err) + continue + } + if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) { + continue + } + select { + case <-ctx.Done(): + return + case out <- sn: + } + } + }() + return out +} From 3c6c17abcd5a950bfcb80fb9c2163299ebfbffca Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:24:58 +0100 Subject: [PATCH 07/10] Refactor `forget` and `snapshots` command Implement filtering by using `FindFilteredSnapshots()` to iterate over the snapshots Refactor cmd_snapshots' `PrintSnapshots()` so its pretty printing can be used from both `forget` and `snapshots`. Use contexts. --- src/cmds/restic/cmd_forget.go | 110 ++++++++++++------------------- src/cmds/restic/cmd_snapshots.go | 53 +++++---------- 2 files changed, 57 insertions(+), 106 deletions(-) diff --git a/src/cmds/restic/cmd_forget.go b/src/cmds/restic/cmd_forget.go index 7c762728d..bc372fc56 100644 --- a/src/cmds/restic/cmd_forget.go +++ b/src/cmds/restic/cmd_forget.go @@ -1,14 +1,12 @@ package main import ( - "encoding/hex" + "context" "encoding/json" "restic" "sort" "strings" - "restic/errors" - "github.com/spf13/cobra" ) @@ -27,13 +25,12 @@ data after 'forget' was run successfully, see the 'prune' command. `, // ForgetOptions collects all options for the forget command. type ForgetOptions struct { - Last int - Hourly int - Daily int - Weekly int - Monthly int - Yearly int - + Last int + Hourly int + Daily int + Weekly int + Monthly int + Yearly int KeepTags []string Host string @@ -83,32 +80,43 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { return err } - // Process all snapshot IDs given as arguments. - if len(args) != 0 { - for _, s := range args { - // Parse argument as hex string. - if _, err := hex.DecodeString(s); err != nil { - Warnf("argument %q is not a snapshot ID, ignoring\n", s) - continue - } - id, err := restic.FindSnapshot(repo, s) - if err != nil { - Warnf("could not find a snapshot for ID %q, ignoring\n", s) - continue - } + // group by hostname and dirs + type key struct { + Hostname string + Paths []string + Tags []string + } + snapshotGroups := make(map[string]restic.Snapshots) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + if len(args) > 0 { + // When explicit snapshots args are given, remove them immediately. if !opts.DryRun { - h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()} - err = repo.Backend().Remove(h) - if err != nil { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + if err = repo.Backend().Remove(h); err != nil { return err } - - Verbosef("removed snapshot %v\n", id.Str()) + Verbosef("removed snapshot %v\n", sn.ID().Str()) } else { - Verbosef("would remove snapshot %v\n", id.Str()) + Verbosef("would have removed snapshot %v\n", sn.ID().Str()) } + } else { + var tags []string + if opts.GroupByTags { + tags = sn.Tags + sort.StringSlice(tags).Sort() + } + sort.StringSlice(sn.Paths).Sort() + k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths}) + if err != nil { + return err + } + snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) } + } + if len(args) > 0 { return nil } @@ -122,53 +130,17 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { Tags: opts.KeepTags, } - snapshots, err := restic.LoadAllSnapshots(repo) - if err != nil { - return err - } - - // Group snapshots by hostname and dirs. - type key struct { - Hostname string - Paths []string - Tags []string - } - - snapshotGroups := make(map[string]restic.Snapshots) - - for _, sn := range snapshots { - if opts.Host != "" && sn.Hostname != opts.Host { - continue - } - - if !sn.HasTags(opts.Tags) { - continue - } - - if !sn.HasPaths(opts.Paths) { - continue - } - - var tags []string - if opts.GroupByTags { - sort.StringSlice(sn.Tags).Sort() - tags = sn.Tags - } - sort.StringSlice(sn.Paths).Sort() - k, _ := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths}) - snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) - } - if len(snapshotGroups) == 0 { - return errors.Fatal("no snapshots remained after filtering") - } if policy.Empty() { Verbosef("no policy was specified, no snapshots will be removed\n") + return nil } removeSnapshots := 0 for k, snapshotGroup := range snapshotGroups { var key key - json.Unmarshal([]byte(k), &key) + if json.Unmarshal([]byte(k), &key) != nil { + return err + } if opts.GroupByTags { Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", ")) } else { diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go index 7ff52831d..7a3fa9879 100644 --- a/src/cmds/restic/cmd_snapshots.go +++ b/src/cmds/restic/cmd_snapshots.go @@ -1,19 +1,19 @@ package main import ( + "context" + "encoding/json" "fmt" "io" - "restic/errors" "sort" "github.com/spf13/cobra" - "encoding/json" "restic" ) var cmdSnapshots = &cobra.Command{ - Use: "snapshots", + Use: "snapshots [snapshotID ...]", Short: "list all snapshots", Long: ` The "snapshots" command lists all snapshots stored in the repository. @@ -26,6 +26,7 @@ The "snapshots" command lists all snapshots stored in the repository. // SnapshotOptions bundles all options for the snapshots command. type SnapshotOptions struct { Host string + Tags []string Paths []string } @@ -35,15 +36,12 @@ func init() { cmdRoot.AddCommand(cmdSnapshots) f := cmdSnapshots.Flags() - f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host") - f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)") + f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`") + f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)") + f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)") } func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { - if len(args) != 0 { - return errors.Fatal("wrong number of arguments") - } - repo, err := OpenRepository(gopts) if err != nil { return err @@ -57,32 +55,14 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } } - done := make(chan struct{}) - defer close(done) - - list := []*restic.Snapshot{} - for id := range repo.List(restic.SnapshotFile, done) { - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - Warnf("error loading snapshot %s: %v\n", id, err) - continue - } - - if (opts.Host == "" || opts.Host == sn.Hostname) && sn.HasPaths(opts.Paths) { - pos := sort.Search(len(list), func(i int) bool { - return list[i].Time.After(sn.Time) - }) - - if pos < len(list) { - list = append(list, nil) - copy(list[pos+1:], list[pos:]) - list[pos] = sn - } else { - list = append(list, sn) - } - } + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + var list restic.Snapshots + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + list = append(list, sn) } + sort.Sort(sort.Reverse(list)) if gopts.JSON { err := printSnapshotsJSON(gopts.stdout, list) @@ -97,7 +77,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } // PrintSnapshots prints a text table of the snapshots in list to stdout. -func PrintSnapshots(stdout io.Writer, list []*restic.Snapshot) { +func PrintSnapshots(stdout io.Writer, list restic.Snapshots) { // Determine the max widths for host and tag. maxHost, maxTag := 10, 6 @@ -165,7 +145,7 @@ func PrintSnapshots(stdout io.Writer, list []*restic.Snapshot) { tab.Write(stdout) } -// Snapshot helps to print Snaphots as JSON +// Snapshot helps to print Snaphots as JSON with their ID included. type Snapshot struct { *restic.Snapshot @@ -173,7 +153,7 @@ type Snapshot struct { } // printSnapshotsJSON writes the JSON representation of list to stdout. -func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error { +func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error { var snapshots []Snapshot @@ -187,5 +167,4 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error { } return json.NewEncoder(stdout).Encode(snapshots) - } From 3432e7edcd4c8443fb832a9448788cba90c6584d Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:28:44 +0100 Subject: [PATCH 08/10] Refactor `tag` to use `FindFilteredSnapshots()` --- src/cmds/restic/cmd_tag.go | 48 +++++++------------------------------- 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go index 1d8d5ebd2..17ed81919 100644 --- a/src/cmds/restic/cmd_tag.go +++ b/src/cmds/restic/cmd_tag.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/spf13/cobra" "restic" @@ -45,22 +47,14 @@ func init() { tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)") tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)") - tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`) + tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given") tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given") tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given") } -func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) { +func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) { var changed bool - sn, err := restic.LoadSnapshot(repo, snapshotID) - if err != nil { - return false, err - } - if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) { - return false, nil - } - if len(setTags) != 0 { // Setting the tag to an empty string really means no tags. if len(setTags) == 1 && setTags[0] == "" { @@ -126,37 +120,13 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { } } - var ids restic.IDs - if len(args) != 0 { - // When explit snapshot-IDs are given, the filtering does not matter anymore. - opts.Host = "" - opts.Tags = nil - opts.Paths = nil - - // Process all snapshot IDs given as arguments. - for _, s := range args { - snapshotID, err := restic.FindSnapshot(repo, s) - if err != nil { - Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err) - continue - } - ids = append(ids, snapshotID) - } - ids = ids.Uniq() - } else { - // If there were no arguments, just get all snapshots. - done := make(chan struct{}) - defer close(done) - for snapshotID := range repo.List(restic.SnapshotFile, done) { - ids = append(ids, snapshotID) - } - } - changeCnt := 0 - for _, id := range ids { - changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host) + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags) if err != nil { - Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err) + Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) continue } if changed { From 8a92687d9a1766914e54c1b283ce1efcf98ea03d Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:29:31 +0100 Subject: [PATCH 09/10] Refactor `find` and `ls` commands Implement filtering by using `FindFilteredSnapshots()` to iterate over the snapshots Refactor cmd_ls' `PrintNode()` into format.go, reuse its pretty printing in both `find` and `ls` commands. Use contexts. --- src/cmds/restic/cmd_find.go | 110 +++++++++++----------------- src/cmds/restic/cmd_ls.go | 78 ++++++-------------- src/cmds/restic/format.go | 24 ++++++ src/cmds/restic/integration_test.go | 4 +- 4 files changed, 90 insertions(+), 126 deletions(-) diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index 26ae92ad4..23c39485d 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -1,6 +1,7 @@ package main import ( + "context" "path/filepath" "strings" "time" @@ -28,8 +29,12 @@ repo. `, type FindOptions struct { Oldest string Newest string - Snapshot string + Snapshots []string CaseInsensitive bool + ListLong bool + Host string + Paths []string + Tags []string } var findOptions FindOptions @@ -40,8 +45,13 @@ func init() { f := cmdFind.Flags() f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time") f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time") - f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "snapshot ID to search in") + f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)") f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern") + f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") + + f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given") + f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given") + f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given") } type findPattern struct { @@ -50,11 +60,6 @@ type findPattern struct { ignoreCase bool } -type findResult struct { - node *restic.Node - path string -} - var timeFormats = []string{ "2006-01-02", "2006-01-02 15:04", @@ -79,14 +84,14 @@ func parseTime(str string) (time.Time, error) { return time.Time{}, errors.Fatalf("unable to parse time: %q", str) } -func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) { +func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error { debug.Log("checking tree %v\n", id) + tree, err := repo.LoadTree(id) if err != nil { - return nil, err + return err } - results := []findResult{} for _, node := range tree.Nodes { debug.Log(" testing entry %q\n", node.Name) @@ -97,7 +102,7 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path m, err := filepath.Match(pat.pattern, name) if err != nil { - return nil, err + return err } if m { @@ -112,46 +117,32 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path continue } - results = append(results, findResult{node: node, path: path}) + if snapshotID != nil { + Verbosef("Found matching entries in snapshot %s\n", *snapshotID) + snapshotID = nil + } + Printf(formatNode(prefix, node, findOptions.ListLong) + "\n") } else { debug.Log(" pattern does not match\n") } if node.Type == "dir" { - subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name)) - if err != nil { - return nil, err + if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil { + return err } - - results = append(results, subdirResults...) } } - return results, nil + return nil } -func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error { - debug.Log("searching in snapshot %s\n for entries within [%s %s]", id.Str(), pat.oldest, pat.newest) +func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error { + debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest) - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { + snapshotID := sn.ID().Str() + if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil { return err } - - results, err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator)) - if err != nil { - return err - } - - if len(results) == 0 { - return nil - } - Verbosef("found %d matching entries in snapshot %s\n", len(results), id) - for _, res := range results { - res.node.Name = filepath.Join(res.path, res.node.Name) - Printf(" %s\n", res.node) - } - return nil } @@ -160,21 +151,21 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { return errors.Fatal("wrong number of arguments") } - var ( - err error - pat findPattern - ) + var err error + pat := findPattern{pattern: args[0]} + if opts.CaseInsensitive { + pat.pattern = strings.ToLower(pat.pattern) + pat.ignoreCase = true + } if opts.Oldest != "" { - pat.oldest, err = parseTime(opts.Oldest) - if err != nil { + if pat.oldest, err = parseTime(opts.Oldest); err != nil { return err } } if opts.Newest != "" { - pat.newest, err = parseTime(opts.Newest) - if err != nil { + if pat.newest, err = parseTime(opts.Newest); err != nil { return err } } @@ -192,33 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { } } - err = repo.LoadIndex() - if err != nil { + if err = repo.LoadIndex(); err != nil { return err } - pat.pattern = args[0] - - if opts.CaseInsensitive { - pat.pattern = strings.ToLower(pat.pattern) - pat.ignoreCase = true - } - - if opts.Snapshot != "" { - snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot) - if err != nil { - return errors.Fatalf("invalid id %q: %v", args[1], err) - } - - return findInSnapshot(repo, pat, snapshotID) - } - - done := make(chan struct{}) - defer close(done) - for snapshotID := range repo.List(restic.SnapshotFile, done) { - err := findInSnapshot(repo, pat, snapshotID) - - if err != nil { + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { + if err = findInSnapshot(repo, sn, pat); err != nil { return err } } diff --git a/src/cmds/restic/cmd_ls.go b/src/cmds/restic/cmd_ls.go index c6c05bec1..7d613b45c 100644 --- a/src/cmds/restic/cmd_ls.go +++ b/src/cmds/restic/cmd_ls.go @@ -1,8 +1,7 @@ package main import ( - "fmt" - "os" + "context" "path/filepath" "github.com/spf13/cobra" @@ -13,7 +12,7 @@ import ( ) var cmdLs = &cobra.Command{ - Use: "ls [flags] snapshot-ID", + Use: "ls [flags] [snapshot-ID ...]", Short: "list files in a snapshot", Long: ` The "ls" command allows listing files and directories in a snapshot. @@ -21,7 +20,7 @@ The "ls" command allows listing files and directories in a snapshot. The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository. `, RunE: func(cmd *cobra.Command, args []string) error { - return runLs(globalOptions, args) + return runLs(lsOptions, globalOptions, args) }, } @@ -29,6 +28,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th type LsOptions struct { ListLong bool Host string + Tags []string Paths []string } @@ -40,42 +40,22 @@ func init() { flags := cmdLs.Flags() flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") - flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`) - flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"") + flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given") + flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given") + flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given") } -func printNode(prefix string, n *restic.Node) string { - if !lsOptions.ListLong { - return filepath.Join(prefix, n.Name) - } - - switch n.Type { - case "file": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) - case "dir": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) - case "symlink": - return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s", - n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget) - default: - return fmt.Sprintf("", n.Type, n.Name) - } -} - -func printTree(prefix string, repo *repository.Repository, id restic.ID) error { - tree, err := repo.LoadTree(id) +func printTree(repo *repository.Repository, id *restic.ID, prefix string) error { + tree, err := repo.LoadTree(*id) if err != nil { return err } for _, entry := range tree.Nodes { - Printf(printNode(prefix, entry) + "\n") + Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n") if entry.Type == "dir" && entry.Subtree != nil { - err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree) - if err != nil { + if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil { return err } } @@ -84,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error { return nil } -func runLs(gopts GlobalOptions, args []string) error { - if len(args) < 1 || len(args) > 2 { - return errors.Fatal("no snapshot ID given") +func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { + if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 { + return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.") } repo, err := OpenRepository(gopts) @@ -94,32 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error { return err } - err = repo.LoadIndex() - if err != nil { + if err = repo.LoadIndex(); err != nil { return err } - snapshotIDString := args[0] - var id restic.ID + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time) - if snapshotIDString == "latest" { - id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host) - if err != nil { - Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host) - } - } else { - id, err = restic.FindSnapshot(repo, snapshotIDString) - if err != nil { - Exitf(1, "invalid id %q: %v", snapshotIDString, err) + if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil { + return err } } - - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - return err - } - - Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time) - - return printTree(string(filepath.Separator), repo, *sn.Tree) + return nil } diff --git a/src/cmds/restic/format.go b/src/cmds/restic/format.go index 68fa29fb3..16c374699 100644 --- a/src/cmds/restic/format.go +++ b/src/cmds/restic/format.go @@ -2,7 +2,11 @@ package main import ( "fmt" + "os" + "path/filepath" "time" + + "restic" ) func formatBytes(c uint64) string { @@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string { sec := uint64(d / time.Second) return formatSeconds(sec) } + +func formatNode(prefix string, n *restic.Node, long bool) string { + if !long { + return filepath.Join(prefix, n.Name) + } + + switch n.Type { + case "file": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) + case "dir": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) + case "symlink": + return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s", + n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget) + default: + return fmt.Sprintf("", n.Type, n.Name) + } +} diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go index 9d7ea55fe..0adf495a3 100644 --- a/src/cmds/restic/integration_test.go +++ b/src/cmds/restic/integration_test.go @@ -142,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { globalOptions.Quiet = quiet }() - OK(t, runLs(gopts, []string{snapshotID})) + opts := LsOptions{} + + OK(t, runLs(opts, gopts, []string{snapshotID})) return strings.Split(string(buf.Bytes()), "\n") } From 8a05de537f2503055e9c52eec8af1ec5ea9213c6 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Wed, 8 Mar 2017 20:30:52 +0100 Subject: [PATCH 10/10] Refactor `prune` and `rebuild_index` Factor out and reuse `rebuildIndex()` in cmd_rebuild_index and cmd_prune. Use contexts. --- src/cmds/restic/cmd_prune.go | 35 ++++------------------------ src/cmds/restic/cmd_rebuild_index.go | 25 +++++++++++--------- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go index fec8126f2..03d14d300 100644 --- a/src/cmds/restic/cmd_prune.go +++ b/src/cmds/restic/cmd_prune.go @@ -1,8 +1,8 @@ package main import ( + "context" "fmt" - "os" "restic" "restic/debug" "restic/errors" @@ -81,8 +81,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { return err } - done := make(chan struct{}) - defer close(done) + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() var stats struct { blobs int @@ -92,7 +92,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { } Verbosef("counting files in repo\n") - for _ = range repo.List(restic.DataFile, done) { + for _ = range repo.List(restic.DataFile, ctx.Done()) { stats.packs++ } @@ -238,35 +238,10 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { bar.Done() } - Verbosef("creating new index\n") - - stats.packs = 0 - for _ = range repo.List(restic.DataFile, done) { - stats.packs++ - } - bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs") - idx, err = index.New(repo, bar) - if err != nil { + if err = rebuildIndex(ctx, repo); err != nil { return err } - var supersedes restic.IDs - for idxID := range repo.List(restic.IndexFile, done) { - h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()} - err := repo.Backend().Remove(h) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err) - } - - supersedes = append(supersedes, idxID) - } - - id, err := idx.Save(repo, supersedes) - if err != nil { - return err - } - Verbosef("saved new index as %v\n", id.Str()) - Verbosef("done\n") return nil } diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index 6f6647daa..e392b80ca 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -1,6 +1,7 @@ package main import ( + "context" "restic" "restic/index" @@ -35,25 +36,29 @@ func runRebuildIndex(gopts GlobalOptions) error { return err } - done := make(chan struct{}) - defer close(done) + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + return rebuildIndex(ctx, repo) +} +func rebuildIndex(ctx context.Context, repo restic.Repository) error { Verbosef("counting files in repo\n") var packs uint64 - for _ = range repo.List(restic.DataFile, done) { + for _ = range repo.List(restic.DataFile, ctx.Done()) { packs++ } - bar := newProgressMax(!gopts.Quiet, packs, "packs") + bar := newProgressMax(!globalOptions.Quiet, packs, "packs") idx, err := index.New(repo, bar) if err != nil { return err } - Verbosef("listing old index files\n") + Verbosef("finding old index files\n") + var supersedes restic.IDs - for id := range repo.List(restic.IndexFile, done) { + for id := range repo.List(restic.IndexFile, ctx.Done()) { supersedes = append(supersedes, id) } @@ -67,13 +72,11 @@ func runRebuildIndex(gopts GlobalOptions) error { Verbosef("remove %d old index files\n", len(supersedes)) for _, id := range supersedes { - err := repo.Backend().Remove(restic.Handle{ + if err := repo.Backend().Remove(restic.Handle{ Type: restic.IndexFile, Name: id.String(), - }) - - if err != nil { - Warnf("error deleting old index %v: %v\n", id.Str(), err) + }); err != nil { + Warnf("error removing old index %v: %v\n", id.Str(), err) } }