From cadcab5a196b768add304eb3a1aeb2c67ad3bfef Mon Sep 17 00:00:00 2001 From: Jan Niklas Richter Date: Wed, 14 Nov 2018 16:23:00 +0100 Subject: [PATCH 1/6] Add GroupBy option to snapshots command This commit adds a --group-by option to the snapshots command, which behaves similar to the --group-by option of forget. Valid option values are "host, paths, tags". If this option is given, the output of snapshots will be divided into multiple tables, according to the value given (i.e. "host" will create a table of snapshots for each host, that has a snapshot in the list). Also the JSON output will be grouped. The default behavior (when --group-by is not given) has not changed. More to this discussion can be found in issue #2037. --- cmd/restic/cmd_snapshots.go | 192 +++++++++++++++++++++++++++++++----- 1 file changed, 167 insertions(+), 25 deletions(-) diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index d9623b942..5ae5cfe85 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/table" "github.com/spf13/cobra" @@ -32,6 +33,7 @@ type SnapshotOptions struct { Paths []string Compact bool Last bool + GroupBy string } var snapshotOptions SnapshotOptions @@ -45,6 +47,13 @@ func init() { f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)") f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format") f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path") + f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags") +} + +type groupKey struct { + Hostname string + Paths []string + Tags []string } func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { @@ -61,28 +70,87 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } } + // group by hostname and dirs + snapshotGroups := make(map[string]restic.Snapshots) + + var GroupByTag bool + var GroupByHost bool + var GroupByPath bool + var GroupOptionList []string + + GroupOptionList = strings.Split(opts.GroupBy, ",") + + for _, option := range GroupOptionList { + switch option { + case "host": + GroupByHost = true + case "paths": + GroupByPath = true + case "tags": + GroupByTag = true + case "": + default: + return errors.Fatal("unknown grouping option: '" + option + "'") + } + } + ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() - var list restic.Snapshots for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { - list = append(list, sn) + // Determining grouping-keys + var tags []string + var hostname string + var paths []string + + if GroupByTag { + tags = sn.Tags + sort.StringSlice(tags).Sort() + } + if GroupByHost { + hostname = sn.Hostname + } + if GroupByPath { + paths = sn.Paths + } + + sort.StringSlice(sn.Paths).Sort() + var k []byte + var err error + + k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths}) + + if err != nil { + return err + } + snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) } - if opts.Last { - list = FilterLastSnapshots(list) + for k, list := range snapshotGroups { + if opts.Last { + list = FilterLastSnapshots(list) + } + sort.Sort(sort.Reverse(list)) + snapshotGroups[k] = list } - sort.Sort(sort.Reverse(list)) - if gopts.JSON { - err := printSnapshotsJSON(gopts.stdout, list) + err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, GroupByTag || GroupByHost || GroupByPath) if err != nil { - Warnf("error printing snapshot: %v\n", err) + Warnf("error printing snapshots: %v\n", err) } return nil } - PrintSnapshots(gopts.stdout, list, nil, opts.Compact) + + for k, list := range snapshotGroups { + err := PrintSnapshotGroupHeader(gopts.stdout, k, GroupByTag, GroupByHost, GroupByPath) + if err != nil { + Warnf("error printing snapshots: %v\n", err) + return nil + } + + PrintSnapshots(gopts.stdout, list, nil, opts.Compact) + } return nil } @@ -223,6 +291,40 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke tab.Write(stdout) } +// PrintSnapshotGroupHeader prints which group of the group-by option the +// following snapshots belong to. +// Prints nothing, if we did not group at all. +func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error { + if GroupByTag || GroupByHost || GroupByPath { + var key groupKey + var err error + + err = json.Unmarshal([]byte(groupKeyJSON), &key) + if err != nil { + return err + } + + // Info + fmt.Fprintf(stdout, "snapshots") + var infoStrings []string + if GroupByTag { + infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") + } + if GroupByHost { + infoStrings = append(infoStrings, "host ["+key.Hostname+"]") + } + if GroupByPath { + infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") + } + if infoStrings != nil { + fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", ")) + } + fmt.Fprintf(stdout, ":\n") + } + + return nil +} + // Snapshot helps to print Snaphots as JSON with their ID included. type Snapshot struct { *restic.Snapshot @@ -231,20 +333,60 @@ type Snapshot struct { ShortID string `json:"short_id"` } -// printSnapshotsJSON writes the JSON representation of list to stdout. -func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error { - - var snapshots []Snapshot - - for _, sn := range list { - - k := Snapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - } - snapshots = append(snapshots, k) - } - - return json.NewEncoder(stdout).Encode(snapshots) +// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included. +type SnapshotGroup struct { + GroupKey groupKey + Snapshots []Snapshot +} + +// printSnapshotsJSON writes the JSON representation of list to stdout. +func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error { + + if grouped { + var snapshotGroups []SnapshotGroup + + for k, list := range snGroups { + var key groupKey + var err error + var snapshots []Snapshot + + err = json.Unmarshal([]byte(k), &key) + if err != nil { + return err + } + + for _, sn := range list { + k := Snapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + } + snapshots = append(snapshots, k) + } + + group := SnapshotGroup{ + GroupKey: key, + Snapshots: snapshots, + } + snapshotGroups = append(snapshotGroups, group) + } + + return json.NewEncoder(stdout).Encode(snapshotGroups) + } else { + // Old behavior + var snapshots []Snapshot + + for _, list := range snGroups { + for _, sn := range list { + k := Snapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + } + snapshots = append(snapshots, k) + } + } + + return json.NewEncoder(stdout).Encode(snapshots) + } } From c9fd9b5275980818c79bcb44cac9b4c6d3d39aaf Mon Sep 17 00:00:00 2001 From: Jan Niklas Richter Date: Fri, 30 Nov 2018 14:34:19 +0100 Subject: [PATCH 2/6] Fix json tags for grouped snapshot output This commit will add json tags to the structs for json output, so all json variables of the snapshot command output are lowercase and snake-case. Furthermore it adds some internal code changes based on the feedback in the pull request #2087. --- cmd/restic/cmd_snapshots.go | 93 +++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 5ae5cfe85..eebf53b7f 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -51,9 +51,9 @@ func init() { } type groupKey struct { - Hostname string - Paths []string - Tags []string + Hostname string `json:"hostname"` + Paths []string `json:"paths"` + Tags []string `json:"tags"` } func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { @@ -295,32 +295,33 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke // following snapshots belong to. // Prints nothing, if we did not group at all. func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error { - if GroupByTag || GroupByHost || GroupByPath { - var key groupKey - var err error - - err = json.Unmarshal([]byte(groupKeyJSON), &key) - if err != nil { - return err - } - - // Info - fmt.Fprintf(stdout, "snapshots") - var infoStrings []string - if GroupByTag { - infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") - } - if GroupByHost { - infoStrings = append(infoStrings, "host ["+key.Hostname+"]") - } - if GroupByPath { - infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") - } - if infoStrings != nil { - fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", ")) - } - fmt.Fprintf(stdout, ":\n") + if !GroupByTag && !GroupByHost && !GroupByPath { + return nil } + var key groupKey + var err error + + err = json.Unmarshal([]byte(groupKeyJSON), &key) + if err != nil { + return err + } + + // Info + fmt.Fprintf(stdout, "snapshots") + var infoStrings []string + if GroupByTag { + infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") + } + if GroupByHost { + infoStrings = append(infoStrings, "host ["+key.Hostname+"]") + } + if GroupByPath { + infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") + } + if infoStrings != nil { + fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", ")) + } + fmt.Fprintf(stdout, ":\n") return nil } @@ -335,8 +336,8 @@ type Snapshot struct { // SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included. type SnapshotGroup struct { - GroupKey groupKey - Snapshots []Snapshot + GroupKey groupKey `json:"group_key"` + Snapshots []Snapshot `json:"snapshots"` } // printSnapshotsJSON writes the JSON representation of list to stdout. @@ -372,21 +373,21 @@ func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapsho } return json.NewEncoder(stdout).Encode(snapshotGroups) - } else { - // Old behavior - var snapshots []Snapshot - - for _, list := range snGroups { - for _, sn := range list { - k := Snapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - } - snapshots = append(snapshots, k) - } - } - - return json.NewEncoder(stdout).Encode(snapshots) } + + // Old behavior + var snapshots []Snapshot + + for _, list := range snGroups { + for _, sn := range list { + k := Snapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + } + snapshots = append(snapshots, k) + } + } + + return json.NewEncoder(stdout).Encode(snapshots) } From c4475ac58f93e613a340d75ca78ea23a1b0ee55a Mon Sep 17 00:00:00 2001 From: Jan Niklas Richter Date: Fri, 30 Nov 2018 15:37:49 +0100 Subject: [PATCH 3/6] Move snapshot grouping code into own function to deduplicate code This commit moves the code which is used to group snapshots in the snapshots command into an own function to deduplicate code shared by the snapshots command and forget command. --- cmd/restic/cmd_snapshots.go | 148 ++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 66 deletions(-) diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index eebf53b7f..8b77d2aa8 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -70,60 +70,16 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } } - // group by hostname and dirs - snapshotGroups := make(map[string]restic.Snapshots) - - var GroupByTag bool - var GroupByHost bool - var GroupByPath bool - var GroupOptionList []string - - GroupOptionList = strings.Split(opts.GroupBy, ",") - - for _, option := range GroupOptionList { - switch option { - case "host": - GroupByHost = true - case "paths": - GroupByPath = true - case "tags": - GroupByTag = true - case "": - default: - return errors.Fatal("unknown grouping option: '" + option + "'") - } - } - ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() + var snapshots restic.Snapshots for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { - // Determining grouping-keys - var tags []string - var hostname string - var paths []string - - if GroupByTag { - tags = sn.Tags - sort.StringSlice(tags).Sort() - } - if GroupByHost { - hostname = sn.Hostname - } - if GroupByPath { - paths = sn.Paths - } - - sort.StringSlice(sn.Paths).Sort() - var k []byte - var err error - - k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths}) - - if err != nil { - return err - } - snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) + snapshots = append(snapshots, sn) + } + snapshotGroups, grouped, err := GroupSnapshots(snapshots, opts.GroupBy) + if err != nil { + return err } for k, list := range snapshotGroups { @@ -135,7 +91,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } if gopts.JSON { - err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, GroupByTag || GroupByHost || GroupByPath) + err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, grouped) if err != nil { Warnf("error printing snapshots: %v\n", err) } @@ -143,12 +99,13 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } for k, list := range snapshotGroups { - err := PrintSnapshotGroupHeader(gopts.stdout, k, GroupByTag, GroupByHost, GroupByPath) - if err != nil { - Warnf("error printing snapshots: %v\n", err) - return nil + if grouped { + err := PrintSnapshotGroupHeader(gopts.stdout, k) + if err != nil { + Warnf("error printing snapshots: %v\n", err) + return nil + } } - PrintSnapshots(gopts.stdout, list, nil, opts.Compact) } @@ -191,6 +148,65 @@ func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots { return results } +// GroupSnapshots takes a list of snapshots and a grouping criteria and creates +// a group list of snapshots. +func GroupSnapshots(snapshots restic.Snapshots, options string) (map[string]restic.Snapshots, bool, error) { + // group by hostname and dirs + snapshotGroups := make(map[string]restic.Snapshots) + + var GroupByTag bool + var GroupByHost bool + var GroupByPath bool + var GroupOptionList []string + + GroupOptionList = strings.Split(options, ",") + + for _, option := range GroupOptionList { + switch option { + case "host": + GroupByHost = true + case "paths": + GroupByPath = true + case "tags": + GroupByTag = true + case "": + default: + return nil, false, errors.Fatal("unknown grouping option: '" + option + "'") + } + } + + for _, sn := range snapshots { + // Determining grouping-keys + var tags []string + var hostname string + var paths []string + + if GroupByTag { + tags = sn.Tags + sort.StringSlice(tags).Sort() + } + if GroupByHost { + hostname = sn.Hostname + } + if GroupByPath { + paths = sn.Paths + } + + sort.StringSlice(sn.Paths).Sort() + var k []byte + var err error + + k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths}) + + if err != nil { + return nil, false, err + } + snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) + } + + return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil +} + // PrintSnapshots prints a text table of the snapshots in list to stdout. func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) { // keep the reasons a snasphot is being kept in a map, so that it doesn't @@ -294,10 +310,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke // PrintSnapshotGroupHeader prints which group of the group-by option the // following snapshots belong to. // Prints nothing, if we did not group at all. -func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error { - if !GroupByTag && !GroupByHost && !GroupByPath { - return nil - } +func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { var key groupKey var err error @@ -306,16 +319,20 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag return err } + if key.Hostname == "" && key.Tags == nil && key.Paths == nil { + return nil + } + // Info fmt.Fprintf(stdout, "snapshots") var infoStrings []string - if GroupByTag { - infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") - } - if GroupByHost { + if key.Hostname != "" { infoStrings = append(infoStrings, "host ["+key.Hostname+"]") } - if GroupByPath { + if key.Tags != nil { + infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") + } + if key.Paths != nil { infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") } if infoStrings != nil { @@ -342,7 +359,6 @@ type SnapshotGroup struct { // printSnapshotsJSON writes the JSON representation of list to stdout. func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error { - if grouped { var snapshotGroups []SnapshotGroup From 3d5a0c799b2f5cf4a171fab1a0e347f17097bad8 Mon Sep 17 00:00:00 2001 From: Jan Niklas Richter Date: Fri, 30 Nov 2018 15:54:44 +0100 Subject: [PATCH 4/6] Add changelog of group-by option for snapshots command --- changelog/unreleased/pull-2087 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/pull-2087 diff --git a/changelog/unreleased/pull-2087 b/changelog/unreleased/pull-2087 new file mode 100644 index 000000000..36967c997 --- /dev/null +++ b/changelog/unreleased/pull-2087 @@ -0,0 +1,10 @@ +Enhancement: Add group-by option to snapshots command + +We have added an option to group the output of the snapshots command, similar +to the output of the forget command. The option has been called "--group-by" +and accepts any combination of the values "host", "paths" and "tags", separated +by commas. Default behavior (not specifying --group-by) has not been changed. +We have added support of the grouping to the JSON output. + +https://github.com/restic/restic/issues/2037 +https://github.com/restic/restic/pull/2087 From 733519d895dacc633bf64746ed182730448544ee Mon Sep 17 00:00:00 2001 From: Jan Niklas Richter Date: Fri, 4 Jan 2019 19:24:03 +0100 Subject: [PATCH 5/6] Refactor duplicate code for grouping snapshots This commit is a followup to the addition of the --group-by flag for the snapshots command. Adding the grouping code there introduced duplicated code (the forget command also does grouping). This commit refactors boths sides to only use shared code. --- cmd/restic/cmd_forget.go | 232 ++++++++++++------------------ cmd/restic/cmd_snapshots.go | 76 +--------- internal/restic/snapshot_group.go | 76 ++++++++++ 3 files changed, 171 insertions(+), 213 deletions(-) create mode 100644 internal/restic/snapshot_group.go diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index a9b7246be..b047f1d45 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -4,10 +4,7 @@ import ( "context" "encoding/json" "io" - "sort" - "strings" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" ) @@ -91,178 +88,129 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { return err } - // group by hostname and dirs - type key struct { - Hostname string - Paths []string - Tags []string - } - snapshotGroups := make(map[string]restic.Snapshots) - - var GroupByTag bool - var GroupByHost bool - var GroupByPath bool - var GroupOptionList []string - - GroupOptionList = strings.Split(opts.GroupBy, ",") - - for _, option := range GroupOptionList { - switch option { - case "host": - GroupByHost = true - case "paths": - GroupByPath = true - case "tags": - GroupByTag = true - case "": - default: - return errors.Fatal("unknown grouping option: '" + option + "'") - } - } - removeSnapshots := 0 ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() + + var snapshots restic.Snapshots + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { - if len(args) > 0 { - // When explicit snapshots args are given, remove them immediately. + snapshots = append(snapshots, sn) + } + + if len(args) > 0 { + // When explicit snapshots args are given, remove them immediately. + for _, sn := range snapshots { if !opts.DryRun { h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(gopts.ctx, h); err != nil { return err } - Verbosef("removed snapshot %v\n", sn.ID().Str()) + if !gopts.JSON { + Verbosef("removed snapshot %v\n", sn.ID().Str()) + } removeSnapshots++ } else { - Verbosef("would have removed snapshot %v\n", sn.ID().Str()) + if !gopts.JSON { + Verbosef("would have removed snapshot %v\n", sn.ID().Str()) + } } - } else { - // Determining grouping-keys - var tags []string - var hostname string - var paths []string - - if GroupByTag { - tags = sn.Tags - sort.StringSlice(tags).Sort() - } - if GroupByHost { - hostname = sn.Hostname - } - if GroupByPath { - paths = sn.Paths - } - - sort.StringSlice(sn.Paths).Sort() - var k []byte - var err error - - k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths}) - - if err != nil { - return err - } - snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) } - } - - policy := restic.ExpirePolicy{ - Last: opts.Last, - Hourly: opts.Hourly, - Daily: opts.Daily, - Weekly: opts.Weekly, - Monthly: opts.Monthly, - Yearly: opts.Yearly, - Within: opts.Within, - Tags: opts.KeepTags, - } - - if policy.Empty() && len(args) == 0 { - Verbosef("no policy was specified, no snapshots will be removed\n") - } - - if !policy.Empty() { - if !gopts.JSON { - Verbosef("Applying Policy: %v\n", policy) + } else { + snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy) + if err != nil { + return err } - var jsonGroups []*ForgetGroup + policy := restic.ExpirePolicy{ + Last: opts.Last, + Hourly: opts.Hourly, + Daily: opts.Daily, + Weekly: opts.Weekly, + Monthly: opts.Monthly, + Yearly: opts.Yearly, + Within: opts.Within, + Tags: opts.KeepTags, + } - for k, snapshotGroup := range snapshotGroups { - var key key - if json.Unmarshal([]byte(k), &key) != nil { - return err - } - - var fg ForgetGroup - // Info + if policy.Empty() && len(args) == 0 { if !gopts.JSON { - Verbosef("snapshots") - } - var infoStrings []string - if GroupByTag { - infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") - fg.Tags = key.Tags - } - if GroupByHost { - infoStrings = append(infoStrings, "host ["+key.Hostname+"]") - fg.Host = key.Hostname - } - if GroupByPath { - infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") - fg.Paths = key.Paths - } - if infoStrings != nil && !gopts.JSON { - Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")") + Verbosef("no policy was specified, no snapshots will be removed\n") } + } + + if !policy.Empty() { if !gopts.JSON { - Verbosef(":\n\n") + Verbosef("Applying Policy: %v\n", policy) } - keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) + var jsonGroups []*ForgetGroup - if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { - Printf("keep %d snapshots:\n", len(keep)) - PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact) - Printf("\n") - } - addJSONSnapshots(&fg.Keep, keep) - - if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { - Printf("remove %d snapshots:\n", len(remove)) - PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact) - Printf("\n") - } - addJSONSnapshots(&fg.Remove, remove) - - fg.Reasons = reasons - - jsonGroups = append(jsonGroups, &fg) - - removeSnapshots += len(remove) - - if !opts.DryRun { - for _, sn := range remove { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} - err = repo.Backend().Remove(gopts.ctx, h) + for k, snapshotGroup := range snapshotGroups { + if gopts.Verbose >= 1 && !gopts.JSON { + err = PrintSnapshotGroupHeader(gopts.stdout, k) if err != nil { return err } } - } - } - if gopts.JSON { - err = printJSONForget(gopts.stdout, jsonGroups) - if err != nil { - return err + var key restic.SnapshotGroupKey + if json.Unmarshal([]byte(k), &key) != nil { + return err + } + + var fg ForgetGroup + fg.Tags = key.Tags + fg.Host = key.Hostname + fg.Paths = key.Paths + + keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) + + if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { + Printf("keep %d snapshots:\n", len(keep)) + PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact) + Printf("\n") + } + addJSONSnapshots(&fg.Keep, keep) + + if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { + Printf("remove %d snapshots:\n", len(remove)) + PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact) + Printf("\n") + } + addJSONSnapshots(&fg.Remove, remove) + + fg.Reasons = reasons + + jsonGroups = append(jsonGroups, &fg) + + removeSnapshots += len(remove) + + if !opts.DryRun { + for _, sn := range remove { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + err = repo.Backend().Remove(gopts.ctx, h) + if err != nil { + return err + } + } + } + } + + if gopts.JSON { + err = printJSONForget(gopts.stdout, jsonGroups) + if err != nil { + return err + } } } } if removeSnapshots > 0 && opts.Prune { - Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots) + if !gopts.JSON { + Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots) + } if !opts.DryRun { return pruneRepository(gopts, repo) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 8b77d2aa8..94cd08836 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -8,7 +8,6 @@ import ( "sort" "strings" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/table" "github.com/spf13/cobra" @@ -50,12 +49,6 @@ func init() { f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags") } -type groupKey struct { - Hostname string `json:"hostname"` - Paths []string `json:"paths"` - Tags []string `json:"tags"` -} - func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { repo, err := OpenRepository(gopts) if err != nil { @@ -77,7 +70,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { snapshots = append(snapshots, sn) } - snapshotGroups, grouped, err := GroupSnapshots(snapshots, opts.GroupBy) + snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) if err != nil { return err } @@ -148,65 +141,6 @@ func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots { return results } -// GroupSnapshots takes a list of snapshots and a grouping criteria and creates -// a group list of snapshots. -func GroupSnapshots(snapshots restic.Snapshots, options string) (map[string]restic.Snapshots, bool, error) { - // group by hostname and dirs - snapshotGroups := make(map[string]restic.Snapshots) - - var GroupByTag bool - var GroupByHost bool - var GroupByPath bool - var GroupOptionList []string - - GroupOptionList = strings.Split(options, ",") - - for _, option := range GroupOptionList { - switch option { - case "host": - GroupByHost = true - case "paths": - GroupByPath = true - case "tags": - GroupByTag = true - case "": - default: - return nil, false, errors.Fatal("unknown grouping option: '" + option + "'") - } - } - - for _, sn := range snapshots { - // Determining grouping-keys - var tags []string - var hostname string - var paths []string - - if GroupByTag { - tags = sn.Tags - sort.StringSlice(tags).Sort() - } - if GroupByHost { - hostname = sn.Hostname - } - if GroupByPath { - paths = sn.Paths - } - - sort.StringSlice(sn.Paths).Sort() - var k []byte - var err error - - k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths}) - - if err != nil { - return nil, false, err - } - snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) - } - - return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil -} - // PrintSnapshots prints a text table of the snapshots in list to stdout. func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) { // keep the reasons a snasphot is being kept in a map, so that it doesn't @@ -311,7 +245,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke // following snapshots belong to. // Prints nothing, if we did not group at all. func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { - var key groupKey + var key restic.SnapshotGroupKey var err error err = json.Unmarshal([]byte(groupKeyJSON), &key) @@ -353,8 +287,8 @@ type Snapshot struct { // SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included. type SnapshotGroup struct { - GroupKey groupKey `json:"group_key"` - Snapshots []Snapshot `json:"snapshots"` + GroupKey restic.SnapshotGroupKey `json:"group_key"` + Snapshots []Snapshot `json:"snapshots"` } // printSnapshotsJSON writes the JSON representation of list to stdout. @@ -363,7 +297,7 @@ func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapsho var snapshotGroups []SnapshotGroup for k, list := range snGroups { - var key groupKey + var key restic.SnapshotGroupKey var err error var snapshots []Snapshot diff --git a/internal/restic/snapshot_group.go b/internal/restic/snapshot_group.go new file mode 100644 index 000000000..f18e0d767 --- /dev/null +++ b/internal/restic/snapshot_group.go @@ -0,0 +1,76 @@ +package restic + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/restic/restic/internal/errors" +) + +// SnapshotGroupKey is the structure for identifying groups in a grouped +// snapshot list. This is used by GroupSnapshots() +type SnapshotGroupKey struct { + Hostname string `json:"hostname"` + Paths []string `json:"paths"` + Tags []string `json:"tags"` +} + +// GroupSnapshots takes a list of snapshots and a grouping criteria and creates +// a group list of snapshots. +func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) { + // group by hostname and dirs + snapshotGroups := make(map[string]Snapshots) + + var GroupByTag bool + var GroupByHost bool + var GroupByPath bool + var GroupOptionList []string + + GroupOptionList = strings.Split(options, ",") + + for _, option := range GroupOptionList { + switch option { + case "host": + GroupByHost = true + case "paths": + GroupByPath = true + case "tags": + GroupByTag = true + case "": + default: + return nil, false, errors.Fatal("unknown grouping option: '" + option + "'") + } + } + + for _, sn := range snapshots { + // Determining grouping-keys + var tags []string + var hostname string + var paths []string + + if GroupByTag { + tags = sn.Tags + sort.StringSlice(tags).Sort() + } + if GroupByHost { + hostname = sn.Hostname + } + if GroupByPath { + paths = sn.Paths + } + + sort.StringSlice(sn.Paths).Sort() + var k []byte + var err error + + k, err = json.Marshal(SnapshotGroupKey{Tags: tags, Hostname: hostname, Paths: paths}) + + if err != nil { + return nil, false, err + } + snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) + } + + return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil +} From 9a26be4e5b7bd1099cffeb40570a3a4ed3540461 Mon Sep 17 00:00:00 2001 From: Jan Niklas Richter Date: Fri, 4 Jan 2019 19:39:13 +0100 Subject: [PATCH 6/6] Add documentation for --group-by flag for snapshots command --- doc/045_working_with_repos.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 6890e32aa..850e29579 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -56,6 +56,31 @@ Or filter by host: Combining filters is also possible. +Furthermore you can group the output by the same filters (host, paths, tags): + +.. code-block:: console + + $ restic -r /srv/restic-repo snapshots --group-by host + + enter password for repository: + snapshots for (host [kasimir]) + ID Date Host Tags Directory + ---------------------------------------------------------------------- + 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work + 79766175 2015-05-08 21:40:19 kasimir /home/user/work + 2 snapshots + snapshots for (host [luigi]) + ID Date Host Tags Directory + ---------------------------------------------------------------------- + bdbd3439 2015-05-08 21:45:17 luigi /home/art + 9f0bc19e 2015-05-08 21:46:11 luigi /srv + 2 snapshots + snapshots for (host [kazik]) + ID Date Host Tags Directory + ---------------------------------------------------------------------- + 590c8fc8 2015-05-08 21:47:38 kazik /srv + 1 snapshots + Checking a repo's integrity and consistency ===========================================