forked from TrueCloudLab/restic
Refactor duplicate code for grouping snapshots
This commit is a followup to the addition of the --group-by flag for the snapshots command. Adding the grouping code there introduced duplicated code (the forget command also does grouping). This commit refactors boths sides to only use shared code.
This commit is contained in:
Jan Niklas Richter
Alexander Neumann
parent
3d5a0c799b
commit
733519d895
3 changed files with 171 additions and 213 deletions
|
|
@ -4,10 +4,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -91,178 +88,129 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// group by hostname and dirs
|
||||
type key struct {
|
||||
Hostname string
|
||||
Paths []string
|
||||
Tags []string
|
||||
}
|
||||
snapshotGroups := make(map[string]restic.Snapshots)
|
||||
|
||||
var GroupByTag bool
|
||||
var GroupByHost bool
|
||||
var GroupByPath bool
|
||||
var GroupOptionList []string
|
||||
|
||||
GroupOptionList = strings.Split(opts.GroupBy, ",")
|
||||
|
||||
for _, option := range GroupOptionList {
|
||||
switch option {
|
||||
case "host":
|
||||
GroupByHost = true
|
||||
case "paths":
|
||||
GroupByPath = true
|
||||
case "tags":
|
||||
GroupByTag = true
|
||||
case "":
|
||||
default:
|
||||
return errors.Fatal("unknown grouping option: '" + option + "'")
|
||||
}
|
||||
}
|
||||
|
||||
removeSnapshots := 0
|
||||
|
||||
ctx, cancel := context.WithCancel(gopts.ctx)
|
||||
defer cancel()
|
||||
|
||||
var snapshots restic.Snapshots
|
||||
|
||||
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
|
||||
if len(args) > 0 {
|
||||
// When explicit snapshots args are given, remove them immediately.
|
||||
snapshots = append(snapshots, sn)
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
// When explicit snapshots args are given, remove them immediately.
|
||||
for _, sn := range snapshots {
|
||||
if !opts.DryRun {
|
||||
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
|
||||
if err = repo.Backend().Remove(gopts.ctx, h); err != nil {
|
||||
return err
|
||||
}
|
||||
Verbosef("removed snapshot %v\n", sn.ID().Str())
|
||||
if !gopts.JSON {
|
||||
Verbosef("removed snapshot %v\n", sn.ID().Str())
|
||||
}
|
||||
removeSnapshots++
|
||||
} else {
|
||||
Verbosef("would have removed snapshot %v\n", sn.ID().Str())
|
||||
if !gopts.JSON {
|
||||
Verbosef("would have removed snapshot %v\n", sn.ID().Str())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Determining grouping-keys
|
||||
var tags []string
|
||||
var hostname string
|
||||
var paths []string
|
||||
|
||||
if GroupByTag {
|
||||
tags = sn.Tags
|
||||
sort.StringSlice(tags).Sort()
|
||||
}
|
||||
if GroupByHost {
|
||||
hostname = sn.Hostname
|
||||
}
|
||||
if GroupByPath {
|
||||
paths = sn.Paths
|
||||
}
|
||||
|
||||
sort.StringSlice(sn.Paths).Sort()
|
||||
var k []byte
|
||||
var err error
|
||||
|
||||
k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
|
||||
}
|
||||
}
|
||||
|
||||
policy := restic.ExpirePolicy{
|
||||
Last: opts.Last,
|
||||
Hourly: opts.Hourly,
|
||||
Daily: opts.Daily,
|
||||
Weekly: opts.Weekly,
|
||||
Monthly: opts.Monthly,
|
||||
Yearly: opts.Yearly,
|
||||
Within: opts.Within,
|
||||
Tags: opts.KeepTags,
|
||||
}
|
||||
|
||||
if policy.Empty() && len(args) == 0 {
|
||||
Verbosef("no policy was specified, no snapshots will be removed\n")
|
||||
}
|
||||
|
||||
if !policy.Empty() {
|
||||
if !gopts.JSON {
|
||||
Verbosef("Applying Policy: %v\n", policy)
|
||||
} else {
|
||||
snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var jsonGroups []*ForgetGroup
|
||||
policy := restic.ExpirePolicy{
|
||||
Last: opts.Last,
|
||||
Hourly: opts.Hourly,
|
||||
Daily: opts.Daily,
|
||||
Weekly: opts.Weekly,
|
||||
Monthly: opts.Monthly,
|
||||
Yearly: opts.Yearly,
|
||||
Within: opts.Within,
|
||||
Tags: opts.KeepTags,
|
||||
}
|
||||
|
||||
for k, snapshotGroup := range snapshotGroups {
|
||||
var key key
|
||||
if json.Unmarshal([]byte(k), &key) != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var fg ForgetGroup
|
||||
// Info
|
||||
if policy.Empty() && len(args) == 0 {
|
||||
if !gopts.JSON {
|
||||
Verbosef("snapshots")
|
||||
}
|
||||
var infoStrings []string
|
||||
if GroupByTag {
|
||||
infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
|
||||
fg.Tags = key.Tags
|
||||
}
|
||||
if GroupByHost {
|
||||
infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
|
||||
fg.Host = key.Hostname
|
||||
}
|
||||
if GroupByPath {
|
||||
infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
|
||||
fg.Paths = key.Paths
|
||||
}
|
||||
if infoStrings != nil && !gopts.JSON {
|
||||
Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")")
|
||||
Verbosef("no policy was specified, no snapshots will be removed\n")
|
||||
}
|
||||
}
|
||||
|
||||
if !policy.Empty() {
|
||||
if !gopts.JSON {
|
||||
Verbosef(":\n\n")
|
||||
Verbosef("Applying Policy: %v\n", policy)
|
||||
}
|
||||
|
||||
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
|
||||
var jsonGroups []*ForgetGroup
|
||||
|
||||
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||
Printf("keep %d snapshots:\n", len(keep))
|
||||
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
|
||||
Printf("\n")
|
||||
}
|
||||
addJSONSnapshots(&fg.Keep, keep)
|
||||
|
||||
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||
Printf("remove %d snapshots:\n", len(remove))
|
||||
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
|
||||
Printf("\n")
|
||||
}
|
||||
addJSONSnapshots(&fg.Remove, remove)
|
||||
|
||||
fg.Reasons = reasons
|
||||
|
||||
jsonGroups = append(jsonGroups, &fg)
|
||||
|
||||
removeSnapshots += len(remove)
|
||||
|
||||
if !opts.DryRun {
|
||||
for _, sn := range remove {
|
||||
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
|
||||
err = repo.Backend().Remove(gopts.ctx, h)
|
||||
for k, snapshotGroup := range snapshotGroups {
|
||||
if gopts.Verbose >= 1 && !gopts.JSON {
|
||||
err = PrintSnapshotGroupHeader(gopts.stdout, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if gopts.JSON {
|
||||
err = printJSONForget(gopts.stdout, jsonGroups)
|
||||
if err != nil {
|
||||
return err
|
||||
var key restic.SnapshotGroupKey
|
||||
if json.Unmarshal([]byte(k), &key) != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var fg ForgetGroup
|
||||
fg.Tags = key.Tags
|
||||
fg.Host = key.Hostname
|
||||
fg.Paths = key.Paths
|
||||
|
||||
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
|
||||
|
||||
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||
Printf("keep %d snapshots:\n", len(keep))
|
||||
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
|
||||
Printf("\n")
|
||||
}
|
||||
addJSONSnapshots(&fg.Keep, keep)
|
||||
|
||||
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||
Printf("remove %d snapshots:\n", len(remove))
|
||||
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
|
||||
Printf("\n")
|
||||
}
|
||||
addJSONSnapshots(&fg.Remove, remove)
|
||||
|
||||
fg.Reasons = reasons
|
||||
|
||||
jsonGroups = append(jsonGroups, &fg)
|
||||
|
||||
removeSnapshots += len(remove)
|
||||
|
||||
if !opts.DryRun {
|
||||
for _, sn := range remove {
|
||||
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
|
||||
err = repo.Backend().Remove(gopts.ctx, h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if gopts.JSON {
|
||||
err = printJSONForget(gopts.stdout, jsonGroups)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if removeSnapshots > 0 && opts.Prune {
|
||||
Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
|
||||
if !gopts.JSON {
|
||||
Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
|
||||
}
|
||||
if !opts.DryRun {
|
||||
return pruneRepository(gopts, repo)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/table"
|
||||
"github.com/spf13/cobra"
|
||||
|
|
@ -50,12 +49,6 @@ func init() {
|
|||
f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
|
||||
}
|
||||
|
||||
type groupKey struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Paths []string `json:"paths"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
|
||||
repo, err := OpenRepository(gopts)
|
||||
if err != nil {
|
||||
|
|
@ -77,7 +70,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
|
|||
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
|
||||
snapshots = append(snapshots, sn)
|
||||
}
|
||||
snapshotGroups, grouped, err := GroupSnapshots(snapshots, opts.GroupBy)
|
||||
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -148,65 +141,6 @@ func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
|
|||
return results
|
||||
}
|
||||
|
||||
// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
|
||||
// a group list of snapshots.
|
||||
func GroupSnapshots(snapshots restic.Snapshots, options string) (map[string]restic.Snapshots, bool, error) {
|
||||
// group by hostname and dirs
|
||||
snapshotGroups := make(map[string]restic.Snapshots)
|
||||
|
||||
var GroupByTag bool
|
||||
var GroupByHost bool
|
||||
var GroupByPath bool
|
||||
var GroupOptionList []string
|
||||
|
||||
GroupOptionList = strings.Split(options, ",")
|
||||
|
||||
for _, option := range GroupOptionList {
|
||||
switch option {
|
||||
case "host":
|
||||
GroupByHost = true
|
||||
case "paths":
|
||||
GroupByPath = true
|
||||
case "tags":
|
||||
GroupByTag = true
|
||||
case "":
|
||||
default:
|
||||
return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
|
||||
}
|
||||
}
|
||||
|
||||
for _, sn := range snapshots {
|
||||
// Determining grouping-keys
|
||||
var tags []string
|
||||
var hostname string
|
||||
var paths []string
|
||||
|
||||
if GroupByTag {
|
||||
tags = sn.Tags
|
||||
sort.StringSlice(tags).Sort()
|
||||
}
|
||||
if GroupByHost {
|
||||
hostname = sn.Hostname
|
||||
}
|
||||
if GroupByPath {
|
||||
paths = sn.Paths
|
||||
}
|
||||
|
||||
sort.StringSlice(sn.Paths).Sort()
|
||||
var k []byte
|
||||
var err error
|
||||
|
||||
k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})
|
||||
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
|
||||
}
|
||||
|
||||
return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
|
||||
}
|
||||
|
||||
// PrintSnapshots prints a text table of the snapshots in list to stdout.
|
||||
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
|
||||
// keep the reasons a snasphot is being kept in a map, so that it doesn't
|
||||
|
|
@ -311,7 +245,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
|
|||
// following snapshots belong to.
|
||||
// Prints nothing, if we did not group at all.
|
||||
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
|
||||
var key groupKey
|
||||
var key restic.SnapshotGroupKey
|
||||
var err error
|
||||
|
||||
err = json.Unmarshal([]byte(groupKeyJSON), &key)
|
||||
|
|
@ -353,8 +287,8 @@ type Snapshot struct {
|
|||
|
||||
// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
|
||||
type SnapshotGroup struct {
|
||||
GroupKey groupKey `json:"group_key"`
|
||||
Snapshots []Snapshot `json:"snapshots"`
|
||||
GroupKey restic.SnapshotGroupKey `json:"group_key"`
|
||||
Snapshots []Snapshot `json:"snapshots"`
|
||||
}
|
||||
|
||||
// printSnapshotsJSON writes the JSON representation of list to stdout.
|
||||
|
|
@ -363,7 +297,7 @@ func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapsho
|
|||
var snapshotGroups []SnapshotGroup
|
||||
|
||||
for k, list := range snGroups {
|
||||
var key groupKey
|
||||
var key restic.SnapshotGroupKey
|
||||
var err error
|
||||
var snapshots []Snapshot
|
||||
|
||||
|
|
|
|||
76
internal/restic/snapshot_group.go
Normal file
76
internal/restic/snapshot_group.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package restic
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
// SnapshotGroupKey is the structure for identifying groups in a grouped
|
||||
// snapshot list. This is used by GroupSnapshots()
|
||||
type SnapshotGroupKey struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Paths []string `json:"paths"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
|
||||
// a group list of snapshots.
|
||||
func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) {
|
||||
// group by hostname and dirs
|
||||
snapshotGroups := make(map[string]Snapshots)
|
||||
|
||||
var GroupByTag bool
|
||||
var GroupByHost bool
|
||||
var GroupByPath bool
|
||||
var GroupOptionList []string
|
||||
|
||||
GroupOptionList = strings.Split(options, ",")
|
||||
|
||||
for _, option := range GroupOptionList {
|
||||
switch option {
|
||||
case "host":
|
||||
GroupByHost = true
|
||||
case "paths":
|
||||
GroupByPath = true
|
||||
case "tags":
|
||||
GroupByTag = true
|
||||
case "":
|
||||
default:
|
||||
return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
|
||||
}
|
||||
}
|
||||
|
||||
for _, sn := range snapshots {
|
||||
// Determining grouping-keys
|
||||
var tags []string
|
||||
var hostname string
|
||||
var paths []string
|
||||
|
||||
if GroupByTag {
|
||||
tags = sn.Tags
|
||||
sort.StringSlice(tags).Sort()
|
||||
}
|
||||
if GroupByHost {
|
||||
hostname = sn.Hostname
|
||||
}
|
||||
if GroupByPath {
|
||||
paths = sn.Paths
|
||||
}
|
||||
|
||||
sort.StringSlice(sn.Paths).Sort()
|
||||
var k []byte
|
||||
var err error
|
||||
|
||||
k, err = json.Marshal(SnapshotGroupKey{Tags: tags, Hostname: hostname, Paths: paths})
|
||||
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
|
||||
}
|
||||
|
||||
return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
|
||||
}
|
||||
Loading…
Reference in a new issue