Merge pull request #2087 from ArcticXWolf/add_group_by_option_for_snapshots

Add GroupBy option to snapshots command

changelog/unreleased/pull-2087

@@ -0,0 +1,10 @@
+Enhancement: Add group-by option to snapshots command
+
+We have added an option to group the output of the snapshots command, similar
+to the output of the forget command. The option has been called "--group-by"
+and accepts any combination of the values "host", "paths" and "tags", separated
+by commas. Default behavior (not specifying --group-by) has not been changed.
+We have added support of the grouping to the JSON output.
+
+https://github.com/restic/restic/issues/2037
+https://github.com/restic/restic/pull/2087

cmd/restic/cmd_forget.go

@@ -4,10 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"sort"
-	"strings"
 
-	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -91,81 +88,40 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	// group by hostname and dirs
-	type key struct {
-		Hostname string
-		Paths    []string
-		Tags     []string
-	}
-	snapshotGroups := make(map[string]restic.Snapshots)
-
-	var GroupByTag bool
-	var GroupByHost bool
-	var GroupByPath bool
-	var GroupOptionList []string
-
-	GroupOptionList = strings.Split(opts.GroupBy, ",")
-
-	for _, option := range GroupOptionList {
-		switch option {
-		case "host":
-			GroupByHost = true
-		case "paths":
-			GroupByPath = true
-		case "tags":
-			GroupByTag = true
-		case "":
-		default:
-			return errors.Fatal("unknown grouping option: '" + option + "'")
-		}
-	}
-
 	removeSnapshots := 0
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
+
+	var snapshots restic.Snapshots
+
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		snapshots = append(snapshots, sn)
+	}
+
 	if len(args) > 0 {
 		// When explicit snapshots args are given, remove them immediately.
+		for _, sn := range snapshots {
 			if !opts.DryRun {
 				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 				if err = repo.Backend().Remove(gopts.ctx, h); err != nil {
 					return err
 				}
+				if !gopts.JSON {
 					Verbosef("removed snapshot %v\n", sn.ID().Str())
+				}
 				removeSnapshots++
 			} else {
+				if !gopts.JSON {
 					Verbosef("would have removed snapshot %v\n", sn.ID().Str())
 				}
+			}
+		}
 	} else {
-			// Determining grouping-keys
+		snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
-			var tags []string
-			var hostname string
-			var paths []string
-
-			if GroupByTag {
-				tags = sn.Tags
-				sort.StringSlice(tags).Sort()
-			}
-			if GroupByHost {
-				hostname = sn.Hostname
-			}
-			if GroupByPath {
-				paths = sn.Paths
-			}
-
-			sort.StringSlice(sn.Paths).Sort()
-			var k []byte
-			var err error
-
-			k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths})
-
 		if err != nil {
 			return err
 		}
-			snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
-		}
-	}
 
 		policy := restic.ExpirePolicy{
 			Last:    opts.Last,
@@ -179,8 +135,10 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		if policy.Empty() && len(args) == 0 {
+			if !gopts.JSON {
 				Verbosef("no policy was specified, no snapshots will be removed\n")
 			}
+		}
 
 		if !policy.Empty() {
 			if !gopts.JSON {
@@ -190,35 +148,22 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 			var jsonGroups []*ForgetGroup
 
 			for k, snapshotGroup := range snapshotGroups {
-			var key key
+				if gopts.Verbose >= 1 && !gopts.JSON {
+					err = PrintSnapshotGroupHeader(gopts.stdout, k)
+					if err != nil {
+						return err
+					}
+				}
+
+				var key restic.SnapshotGroupKey
 				if json.Unmarshal([]byte(k), &key) != nil {
 					return err
 				}
 
 				var fg ForgetGroup
-			// Info
-			if !gopts.JSON {
-				Verbosef("snapshots")
-			}
-			var infoStrings []string
-			if GroupByTag {
-				infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
 				fg.Tags = key.Tags
-			}
-			if GroupByHost {
-				infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
 				fg.Host = key.Hostname
-			}
-			if GroupByPath {
-				infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
 				fg.Paths = key.Paths
-			}
-			if infoStrings != nil && !gopts.JSON {
-				Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")")
-			}
-			if !gopts.JSON {
-				Verbosef(":\n\n")
-			}
 
 				keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
 
@@ -260,9 +205,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 				}
 			}
 		}
+	}
 
 	if removeSnapshots > 0 && opts.Prune {
+		if !gopts.JSON {
 			Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
+		}
 		if !opts.DryRun {
 			return pruneRepository(gopts, repo)
 		}

cmd/restic/cmd_snapshots.go

@@ -32,6 +32,7 @@ type SnapshotOptions struct {
 	Paths   []string
 	Compact bool
 	Last    bool
+	GroupBy string
 }
 
 var snapshotOptions SnapshotOptions
@@ -45,6 +46,7 @@ func init() {
 	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
 	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
+	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
 }
 
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
@@ -64,25 +66,41 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
 
-	var list restic.Snapshots
+	var snapshots restic.Snapshots
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
-		list = append(list, sn)
+		snapshots = append(snapshots, sn)
+	}
+	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
+	if err != nil {
+		return err
 	}
 
+	for k, list := range snapshotGroups {
 		if opts.Last {
 			list = FilterLastSnapshots(list)
 		}
-
 		sort.Sort(sort.Reverse(list))
+		snapshotGroups[k] = list
+	}
 
 	if gopts.JSON {
-		err := printSnapshotsJSON(gopts.stdout, list)
+		err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, grouped)
 		if err != nil {
-			Warnf("error printing snapshot: %v\n", err)
+			Warnf("error printing snapshots: %v\n", err)
 		}
 		return nil
 	}
+
+	for k, list := range snapshotGroups {
+		if grouped {
+			err := PrintSnapshotGroupHeader(gopts.stdout, k)
+			if err != nil {
+				Warnf("error printing snapshots: %v\n", err)
+				return nil
+			}
+		}
 		PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
+	}
 
 	return nil
 }
@@ -223,6 +241,42 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 	tab.Write(stdout)
 }
 
+// PrintSnapshotGroupHeader prints which group of the group-by option the
+// following snapshots belong to.
+// Prints nothing, if we did not group at all.
+func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
+	var key restic.SnapshotGroupKey
+	var err error
+
+	err = json.Unmarshal([]byte(groupKeyJSON), &key)
+	if err != nil {
+		return err
+	}
+
+	if key.Hostname == "" && key.Tags == nil && key.Paths == nil {
+		return nil
+	}
+
+	// Info
+	fmt.Fprintf(stdout, "snapshots")
+	var infoStrings []string
+	if key.Hostname != "" {
+		infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
+	}
+	if key.Tags != nil {
+		infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
+	}
+	if key.Paths != nil {
+		infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
+	}
+	if infoStrings != nil {
+		fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
+	}
+	fmt.Fprintf(stdout, ":\n")
+
+	return nil
+}
+
 // Snapshot helps to print Snaphots as JSON with their ID included.
 type Snapshot struct {
 	*restic.Snapshot
@@ -231,13 +285,28 @@ type Snapshot struct {
 	ShortID string     `json:"short_id"`
 }
 
-// printSnapshotsJSON writes the JSON representation of list to stdout.
+// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
-func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
+type SnapshotGroup struct {
+	GroupKey  restic.SnapshotGroupKey `json:"group_key"`
+	Snapshots []Snapshot              `json:"snapshots"`
+}
 
+// printSnapshotsJSON writes the JSON representation of list to stdout.
+func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {
+	if grouped {
+		var snapshotGroups []SnapshotGroup
+
+		for k, list := range snGroups {
+			var key restic.SnapshotGroupKey
+			var err error
 			var snapshots []Snapshot
 
-	for _, sn := range list {
+			err = json.Unmarshal([]byte(k), &key)
+			if err != nil {
+				return err
+			}
 
+			for _, sn := range list {
 				k := Snapshot{
 					Snapshot: sn,
 					ID:       sn.ID(),
@@ -246,5 +315,29 @@ func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
 				snapshots = append(snapshots, k)
 			}
 
+			group := SnapshotGroup{
+				GroupKey:  key,
+				Snapshots: snapshots,
+			}
+			snapshotGroups = append(snapshotGroups, group)
+		}
+
+		return json.NewEncoder(stdout).Encode(snapshotGroups)
+	}
+
+	// Old behavior
+	var snapshots []Snapshot
+
+	for _, list := range snGroups {
+		for _, sn := range list {
+			k := Snapshot{
+				Snapshot: sn,
+				ID:       sn.ID(),
+				ShortID:  sn.ID().Str(),
+			}
+			snapshots = append(snapshots, k)
+		}
+	}
+
 	return json.NewEncoder(stdout).Encode(snapshots)
 }

doc/045_working_with_repos.rst

@@ -56,6 +56,31 @@ Or filter by host:
 
 Combining filters is also possible.
 
+Furthermore you can group the output by the same filters (host, paths, tags):
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo snapshots --group-by host
+
+    enter password for repository:
+    snapshots for (host [kasimir])
+    ID        Date                 Host    Tags   Directory
+    ----------------------------------------------------------------------
+    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
+    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
+    2 snapshots
+    snapshots for (host [luigi])
+    ID        Date                 Host    Tags   Directory
+    ----------------------------------------------------------------------
+    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
+    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+    2 snapshots
+    snapshots for (host [kazik])
+    ID        Date                 Host    Tags   Directory
+    ----------------------------------------------------------------------
+    590c8fc8  2015-05-08 21:47:38  kazik          /srv
+    1 snapshots
+
 
 Checking a repo's integrity and consistency
 ===========================================

internal/restic/snapshot_group.go

@@ -0,0 +1,76 @@
+package restic
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+)
+
+// SnapshotGroupKey is the structure for identifying groups in a grouped
+// snapshot list. This is used by GroupSnapshots()
+type SnapshotGroupKey struct {
+	Hostname string   `json:"hostname"`
+	Paths    []string `json:"paths"`
+	Tags     []string `json:"tags"`
+}
+
+// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
+// a group list of snapshots.
+func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) {
+	// group by hostname and dirs
+	snapshotGroups := make(map[string]Snapshots)
+
+	var GroupByTag bool
+	var GroupByHost bool
+	var GroupByPath bool
+	var GroupOptionList []string
+
+	GroupOptionList = strings.Split(options, ",")
+
+	for _, option := range GroupOptionList {
+		switch option {
+		case "host":
+			GroupByHost = true
+		case "paths":
+			GroupByPath = true
+		case "tags":
+			GroupByTag = true
+		case "":
+		default:
+			return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
+		}
+	}
+
+	for _, sn := range snapshots {
+		// Determining grouping-keys
+		var tags []string
+		var hostname string
+		var paths []string
+
+		if GroupByTag {
+			tags = sn.Tags
+			sort.StringSlice(tags).Sort()
+		}
+		if GroupByHost {
+			hostname = sn.Hostname
+		}
+		if GroupByPath {
+			paths = sn.Paths
+		}
+
+		sort.StringSlice(sn.Paths).Sort()
+		var k []byte
+		var err error
+
+		k, err = json.Marshal(SnapshotGroupKey{Tags: tags, Hostname: hostname, Paths: paths})
+
+		if err != nil {
+			return nil, false, err
+		}
+		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
+	}
+
+	return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
+}