KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #866 from middelink/widespread-tags

Browse source 
Add --tag filtering to every command, where applicable
This commit is contained in:
Alexander Neumann 2017-03-09 09:55:26 +01:00
commit 93e4e4f4fb
24 changed files with 316 additions and 344 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.travis.yml
View file

@ -2,7 +2,6 @@ language: go
sudo: false
go:
- 1.6.4
- 1.7.5
- 1.8
- tip
@ -17,8 +16,6 @@ env:
matrix:
exclude:
- os: osx
go: 1.6.4
- os: osx
go: 1.7.5
- os: osx

2
CONTRIBUTING.md
View file

@ -77,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary:
[...]
$ bin/restic version
restic compiled manually
compiled at unknown time with go1.6
compiled at unknown time with go1.7
The following commands can be used to run all the tests:

2
README.md
View file

@ -34,7 +34,7 @@ You can download the latest pre-compiled binary from the [restic release page](h
Build restic
============
Install Go/Golang (at least version 1.6), then run `go run build.go`,
Install Go/Golang (at least version 1.7), then run `go run build.go`,
afterwards you'll find the binary in the current directory:
$ go run build.go

2
Vagrantfile vendored
View file

@ -1,7 +1,7 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
GO_VERSION = '1.6'
GO_VERSION = '1.7'
def packages_freebsd
return <<-EOF

2
doc/Manual.md
View file

@ -27,7 +27,7 @@ $ pacaur -S restic-git
# Building restic
restic is written in the Go programming language and you need at least Go version 1.6.
restic is written in the Go programming language and you need at least Go version 1.7.
Building restic may also work with older versions of Go, but that's not supported.
See the [Getting started](https://golang.org/doc/install) guide of the Go project for
instructions how to install Go.

6
src/cmds/restic/cmd_backup.go
View file

@ -67,12 +67,12 @@ func init() {
f := cmdBackup.Flags()
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}
@ -391,7 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentSnapshotID == nil {
id, err := restic.FindLatestSnapshot(repo, target, opts.Hostname)
id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
if err == nil {
parentSnapshotID = &id
} else if err != restic.ErrNoSnapshotFound {

2
src/cmds/restic/cmd_check.go
View file

@ -24,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore.
},
}
// CheckOptions bundle all options for the 'check' command.
// CheckOptions bundles all options for the 'check' command.
type CheckOptions struct {
ReadData bool
CheckUnused bool

112
src/cmds/restic/cmd_find.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"path/filepath"
"strings"
"time"
@ -24,12 +25,16 @@ repo. `,
},
}
// FindOptions bundle all options for the find command.
// FindOptions bundles all options for the find command.
type FindOptions struct {
Oldest string
Newest string
Snapshot string
Snapshots []string
CaseInsensitive bool
ListLong bool
Host string
Paths []string
Tags []string
}
var findOptions FindOptions
@ -40,8 +45,13 @@ func init() {
f := cmdFind.Flags()
f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time")
f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time")
f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "snapshot ID to search in")
f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
}
type findPattern struct {
@ -50,11 +60,6 @@ type findPattern struct {
ignoreCase bool
}
type findResult struct {
node *restic.Node
path string
}
var timeFormats = []string{
"2006-01-02",
"2006-01-02 15:04",
@ -79,14 +84,14 @@ func parseTime(str string) (time.Time, error) {
return time.Time{}, errors.Fatalf("unable to parse time: %q", str)
}
func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) {
func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error {
debug.Log("checking tree %v\n", id)
tree, err := repo.LoadTree(id)
if err != nil {
return nil, err
return err
}
results := []findResult{}
for _, node := range tree.Nodes {
debug.Log(" testing entry %q\n", node.Name)
@ -97,7 +102,7 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path
m, err := filepath.Match(pat.pattern, name)
if err != nil {
return nil, err
return err
}
if m {
@ -112,46 +117,32 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path
continue
}
results = append(results, findResult{node: node, path: path})
if snapshotID != nil {
Verbosef("Found matching entries in snapshot %s\n", *snapshotID)
snapshotID = nil
}
Printf(formatNode(prefix, node, findOptions.ListLong) + "\n")
} else {
debug.Log(" pattern does not match\n")
}
if node.Type == "dir" {
subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name))
if err != nil {
return nil, err
if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil {
return err
}
results = append(results, subdirResults...)
}
}
return results, nil
return nil
}
func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)
func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
snapshotID := sn.ID().Str()
if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil {
return err
}
results, err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator))
if err != nil {
return err
}
if len(results) == 0 {
return nil
}
Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
for _, res := range results {
res.node.Name = filepath.Join(res.path, res.node.Name)
Printf(" %s\n", res.node)
}
return nil
}
@ -160,21 +151,21 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("wrong number of arguments")
}
var (
err error
pat findPattern
)
var err error
pat := findPattern{pattern: args[0]}
if opts.CaseInsensitive {
pat.pattern = strings.ToLower(pat.pattern)
pat.ignoreCase = true
}
if opts.Oldest != "" {
pat.oldest, err = parseTime(opts.Oldest)
if err != nil {
if pat.oldest, err = parseTime(opts.Oldest); err != nil {
return err
}
}
if opts.Newest != "" {
pat.newest, err = parseTime(opts.Newest)
if err != nil {
if pat.newest, err = parseTime(opts.Newest); err != nil {
return err
}
}
@ -192,33 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
}
}
err = repo.LoadIndex()
if err != nil {
if err = repo.LoadIndex(); err != nil {
return err
}
pat.pattern = args[0]
if opts.CaseInsensitive {
pat.pattern = strings.ToLower(pat.pattern)
pat.ignoreCase = true
}
if opts.Snapshot != "" {
snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot)
if err != nil {
return errors.Fatalf("invalid id %q: %v", args[1], err)
}
return findInSnapshot(repo, pat, snapshotID)
}
done := make(chan struct{})
defer close(done)
for snapshotID := range repo.List(restic.SnapshotFile, done) {
err := findInSnapshot(repo, pat, snapshotID)
if err != nil {
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
if err = findInSnapshot(repo, sn, pat); err != nil {
return err
}
}

110
src/cmds/restic/cmd_forget.go
View file

@ -1,14 +1,12 @@
package main
import (
"encoding/hex"
"context"
"encoding/json"
"restic"
"sort"
"strings"
"restic/errors"
"github.com/spf13/cobra"
)
@ -27,13 +25,12 @@ data after 'forget' was run successfully, see the 'prune' command. `,
// ForgetOptions collects all options for the forget command.
type ForgetOptions struct {
Last int
Hourly int
Daily int
Weekly int
Monthly int
Yearly int
Last int
Hourly int
Daily int
Weekly int
Monthly int
Yearly int
KeepTags []string
Host string
@ -83,32 +80,43 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
return err
}
// Process all snapshot IDs given as arguments.
if len(args) != 0 {
for _, s := range args {
// Parse argument as hex string.
if _, err := hex.DecodeString(s); err != nil {
Warnf("argument %q is not a snapshot ID, ignoring\n", s)
continue
}
id, err := restic.FindSnapshot(repo, s)
if err != nil {
Warnf("could not find a snapshot for ID %q, ignoring\n", s)
continue
}
// group by hostname and dirs
type key struct {
Hostname string
Paths []string
Tags []string
}
snapshotGroups := make(map[string]restic.Snapshots)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
if len(args) > 0 {
// When explicit snapshots args are given, remove them immediately.
if !opts.DryRun {
h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()}
err = repo.Backend().Remove(h)
if err != nil {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(h); err != nil {
return err
}
Verbosef("removed snapshot %v\n", id.Str())
Verbosef("removed snapshot %v\n", sn.ID().Str())
} else {
Verbosef("would remove snapshot %v\n", id.Str())
Verbosef("would have removed snapshot %v\n", sn.ID().Str())
}
} else {
var tags []string
if opts.GroupByTags {
tags = sn.Tags
sort.StringSlice(tags).Sort()
}
sort.StringSlice(sn.Paths).Sort()
k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
if err != nil {
return err
}
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
}
}
if len(args) > 0 {
return nil
}
@ -122,53 +130,17 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
Tags: opts.KeepTags,
}
snapshots, err := restic.LoadAllSnapshots(repo)
if err != nil {
return err
}
// Group snapshots by hostname and dirs.
type key struct {
Hostname string
Paths []string
Tags []string
}
snapshotGroups := make(map[string]restic.Snapshots)
for _, sn := range snapshots {
if opts.Host != "" && sn.Hostname != opts.Host {
continue
}
if !sn.HasTags(opts.Tags) {
continue
}
if !sn.HasPaths(opts.Paths) {
continue
}
var tags []string
if opts.GroupByTags {
sort.StringSlice(sn.Tags).Sort()
tags = sn.Tags
}
sort.StringSlice(sn.Paths).Sort()
k, _ := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
}
if len(snapshotGroups) == 0 {
return errors.Fatal("no snapshots remained after filtering")
}
if policy.Empty() {
Verbosef("no policy was specified, no snapshots will be removed\n")
return nil
}
removeSnapshots := 0
for k, snapshotGroup := range snapshotGroups {
var key key
json.Unmarshal([]byte(k), &key)
if json.Unmarshal([]byte(k), &key) != nil {
return err
}
if opts.GroupByTags {
Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", "))
} else {

18
src/cmds/restic/cmd_key.go
View file

@ -1,13 +1,13 @@
package main
import (
"context"
"fmt"
"restic"
"github.com/spf13/cobra"
"restic/errors"
"restic/repository"
"github.com/spf13/cobra"
)
var cmdKey = &cobra.Command{
@ -25,15 +25,12 @@ func init() {
cmdRoot.AddCommand(cmdKey)
}
func listKeys(s *repository.Repository) error {
func listKeys(ctx context.Context, s *repository.Repository) error {
tab := NewTable()
tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created")
tab.RowFormat = "%s%-10s %-10s %-10s %s"
done := make(chan struct{})
defer close(done)
for id := range s.List(restic.KeyFile, done) {
for id := range s.List(restic.KeyFile, ctx.Done()) {
k, err := repository.LoadKey(s, id.String())
if err != nil {
Warnf("LoadKey() failed: %v\n", err)
@ -124,6 +121,9 @@ func runKey(gopts GlobalOptions, args []string) error {
return errors.Fatal("wrong number of arguments")
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
repo, err := OpenRepository(gopts)
if err != nil {
return err
@ -137,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error {
return err
}
return listKeys(repo)
return listKeys(ctx, repo)
case "add":
lock, err := lockRepo(repo)
defer unlockRepo(lock)

78
src/cmds/restic/cmd_ls.go
View file

@ -1,8 +1,7 @@
package main
import (
"fmt"
"os"
"context"
"path/filepath"
"github.com/spf13/cobra"
@ -13,7 +12,7 @@ import (
)
var cmdLs = &cobra.Command{
Use: "ls [flags] snapshot-ID",
Use: "ls [flags] [snapshot-ID ...]",
Short: "list files in a snapshot",
Long: `
The "ls" command allows listing files and directories in a snapshot.
@ -21,7 +20,7 @@ The "ls" command allows listing files and directories in a snapshot.
The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runLs(globalOptions, args)
return runLs(lsOptions, globalOptions, args)
},
}
@ -29,6 +28,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th
type LsOptions struct {
ListLong bool
Host string
Tags []string
Paths []string
}
@ -40,42 +40,22 @@ func init() {
flags := cmdLs.Flags()
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}
func printNode(prefix string, n *restic.Node) string {
if !lsOptions.ListLong {
return filepath.Join(prefix, n.Name)
}
switch n.Type {
case "file":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
case "dir":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
case "symlink":
return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
default:
return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
}
}
func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
tree, err := repo.LoadTree(id)
func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
tree, err := repo.LoadTree(*id)
if err != nil {
return err
}
for _, entry := range tree.Nodes {
Printf(printNode(prefix, entry) + "\n")
Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")
if entry.Type == "dir" && entry.Subtree != nil {
err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
if err != nil {
if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
return err
}
}
@ -84,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
return nil
}
func runLs(gopts GlobalOptions, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errors.Fatal("no snapshot ID given")
func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
}
repo, err := OpenRepository(gopts)
@ -94,32 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error {
return err
}
err = repo.LoadIndex()
if err != nil {
if err = repo.LoadIndex(); err != nil {
return err
}
snapshotIDString := args[0]
var id restic.ID
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
if snapshotIDString == "latest" {
id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
if err != nil {
Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
}
} else {
id, err = restic.FindSnapshot(repo, snapshotIDString)
if err != nil {
Exitf(1, "invalid id %q: %v", snapshotIDString, err)
if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
return err
}
}
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
return err
}
Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
return printTree(string(filepath.Separator), repo, *sn.Tree)
return nil
}

16
src/cmds/restic/cmd_mount.go
View file

@ -35,6 +35,9 @@ type MountOptions struct {
OwnerRoot bool
AllowRoot bool
AllowOther bool
Host string
Tags []string
Paths []string
}
var mountOptions MountOptions
@ -42,9 +45,14 @@ var mountOptions MountOptions
func init() {
cmdRoot.AddCommand(cmdMount)
cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
cmdMount.Flags().BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory")
cmdMount.Flags().BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
mountFlags := cmdMount.Flags()
mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory")
mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
}
func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@ -91,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
Printf("Don't forget to umount after quitting!\n")
root := fs.Tree{}
root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot))
root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host))
debug.Log("serving mount at %v", mountpoint)
err = fs.Serve(c, &root)

35
src/cmds/restic/cmd_prune.go
View file

@ -1,8 +1,8 @@
package main
import (
"context"
"fmt"
"os"
"restic"
"restic/debug"
"restic/errors"
@ -81,8 +81,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
return err
}
done := make(chan struct{})
defer close(done)
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var stats struct {
blobs int
@ -92,7 +92,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
}
Verbosef("counting files in repo\n")
for _ = range repo.List(restic.DataFile, done) {
for _ = range repo.List(restic.DataFile, ctx.Done()) {
stats.packs++
}
@ -238,35 +238,10 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
bar.Done()
}
Verbosef("creating new index\n")
stats.packs = 0
for _ = range repo.List(restic.DataFile, done) {
stats.packs++
}
bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
idx, err = index.New(repo, bar)
if err != nil {
if err = rebuildIndex(ctx, repo); err != nil {
return err
}
var supersedes restic.IDs
for idxID := range repo.List(restic.IndexFile, done) {
h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()}
err := repo.Backend().Remove(h)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
}
supersedes = append(supersedes, idxID)
}
id, err := idx.Save(repo, supersedes)
if err != nil {
return err
}
Verbosef("saved new index as %v\n", id.Str())
Verbosef("done\n")
return nil
}

25
src/cmds/restic/cmd_rebuild_index.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"restic"
"restic/index"
@ -35,25 +36,29 @@ func runRebuildIndex(gopts GlobalOptions) error {
return err
}
done := make(chan struct{})
defer close(done)
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
return rebuildIndex(ctx, repo)
}
func rebuildIndex(ctx context.Context, repo restic.Repository) error {
Verbosef("counting files in repo\n")
var packs uint64
for _ = range repo.List(restic.DataFile, done) {
for _ = range repo.List(restic.DataFile, ctx.Done()) {
packs++
}
bar := newProgressMax(!gopts.Quiet, packs, "packs")
bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
idx, err := index.New(repo, bar)
if err != nil {
return err
}
Verbosef("listing old index files\n")
Verbosef("finding old index files\n")
var supersedes restic.IDs
for id := range repo.List(restic.IndexFile, done) {
for id := range repo.List(restic.IndexFile, ctx.Done()) {
supersedes = append(supersedes, id)
}
@ -67,13 +72,11 @@ func runRebuildIndex(gopts GlobalOptions) error {
Verbosef("remove %d old index files\n", len(supersedes))
for _, id := range supersedes {
err := repo.Backend().Remove(restic.Handle{
if err := repo.Backend().Remove(restic.Handle{
Type: restic.IndexFile,
Name: id.String(),
})
if err != nil {
Warnf("error deleting old index %v: %v\n", id.Str(), err)
}); err != nil {
Warnf("error removing old index %v: %v\n", id.Str(), err)
}
}

4
src/cmds/restic/cmd_restore.go
View file

@ -31,6 +31,7 @@ type RestoreOptions struct {
Target string
Host string
Paths []string
Tags []string
}
var restoreOptions RestoreOptions
@ -44,6 +45,7 @@ func init() {
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
}
@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
var id restic.ID
if snapshotIDString == "latest" {
id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host)
id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host)
if err != nil {
Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
}

55
src/cmds/restic/cmd_snapshots.go
View file

@ -1,19 +1,19 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"restic/errors"
"sort"
"github.com/spf13/cobra"
"encoding/json"
"restic"
)
var cmdSnapshots = &cobra.Command{
Use: "snapshots",
Use: "snapshots [snapshotID ...]",
Short: "list all snapshots",
Long: `
The "snapshots" command lists all snapshots stored in the repository.
@ -23,9 +23,10 @@ The "snapshots" command lists all snapshots stored in the repository.
},
}
// SnapshotOptions bundle all options for the snapshots command.
// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
Host string
Tags []string
Paths []string
}
@ -35,15 +36,12 @@ func init() {
cmdRoot.AddCommand(cmdSnapshots)
f := cmdSnapshots.Flags()
f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host")
f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)")
f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
}
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatal("wrong number of arguments")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
@ -57,32 +55,14 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
}
}
done := make(chan struct{})
defer close(done)
list := []*restic.Snapshot{}
for id := range repo.List(restic.SnapshotFile, done) {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
Warnf("error loading snapshot %s: %v\n", id, err)
continue
}
if (opts.Host == "" || opts.Host == sn.Hostname) && sn.HasPaths(opts.Paths) {
pos := sort.Search(len(list), func(i int) bool {
return list[i].Time.After(sn.Time)
})
if pos < len(list) {
list = append(list, nil)
copy(list[pos+1:], list[pos:])
list[pos] = sn
} else {
list = append(list, sn)
}
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var list restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
list = append(list, sn)
}
sort.Sort(sort.Reverse(list))
if gopts.JSON {
err := printSnapshotsJSON(gopts.stdout, list)
@ -97,7 +77,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
}
// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list []*restic.Snapshot) {
func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
// Determine the max widths for host and tag.
maxHost, maxTag := 10, 6
@ -165,7 +145,7 @@ func PrintSnapshots(stdout io.Writer, list []*restic.Snapshot) {
tab.Write(stdout)
}
// Snapshot helps to print Snaphots as JSON
// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
*restic.Snapshot
@ -173,7 +153,7 @@ type Snapshot struct {
}
// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error {
func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
var snapshots []Snapshot
@ -187,5 +167,4 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error {
}
return json.NewEncoder(stdout).Encode(snapshots)
}

48
src/cmds/restic/cmd_tag.go
View file

@ -1,6 +1,8 @@
package main
import (
"context"
"github.com/spf13/cobra"
"restic"
@ -45,22 +47,14 @@ func init() {
tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`)
tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
}
func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) {
func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
var changed bool
sn, err := restic.LoadSnapshot(repo, snapshotID)
if err != nil {
return false, err
}
if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
return false, nil
}
if len(setTags) != 0 {
// Setting the tag to an empty string really means no tags.
if len(setTags) == 1 && setTags[0] == "" {
@ -126,37 +120,13 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
}
}
var ids restic.IDs
if len(args) != 0 {
// When explit snapshot-IDs are given, the filtering does not matter anymore.
opts.Host = ""
opts.Tags = nil
opts.Paths = nil
// Process all snapshot IDs given as arguments.
for _, s := range args {
snapshotID, err := restic.FindSnapshot(repo, s)
if err != nil {
Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err)
continue
}
ids = append(ids, snapshotID)
}
ids = ids.Uniq()
} else {
// If there were no arguments, just get all snapshots.
done := make(chan struct{})
defer close(done)
for snapshotID := range repo.List(restic.SnapshotFile, done) {
ids = append(ids, snapshotID)
}
}
changeCnt := 0
for _, id := range ids {
changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host)
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
if err != nil {
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err)
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)
continue
}
if changed {

78
src/cmds/restic/find.go Normal file
View file

@ -0,0 +1,78 @@
package main
import (
"context"
"restic"
"restic/repository"
)
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot)
go func() {
defer close(out)
if len(snapshotIDs) != 0 {
var (
id restic.ID
usedFilter bool
err error
)
ids := make(restic.IDs, 0, len(snapshotIDs))
// Process all snapshot IDs given as arguments.
for _, s := range snapshotIDs {
if s == "latest" {
id, err = restic.FindLatestSnapshot(repo, paths, tags, host)
if err != nil {
Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
usedFilter = true
continue
}
} else {
id, err = restic.FindSnapshot(repo, s)
if err != nil {
Warnf("Ignoring %q, it is not a snapshot id\n", s)
continue
}
}
ids = append(ids, id)
}
// Give the user some indication their filters are not used.
if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) {
Warnf("Ignoring filters as there are explicit snapshot ids given\n")
}
for _, id := range ids.Uniq() {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
continue
}
select {
case <-ctx.Done():
return
case out <- sn:
}
}
return
}
for id := range repo.List(restic.SnapshotFile, ctx.Done()) {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
continue
}
if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
continue
}
select {
case <-ctx.Done():
return
case out <- sn:
}
}
}()
return out
}

24
src/cmds/restic/format.go
View file

@ -2,7 +2,11 @@ package main
import (
"fmt"
"os"
"path/filepath"
"time"
"restic"
)
func formatBytes(c uint64) string {
@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string {
sec := uint64(d / time.Second)
return formatSeconds(sec)
}
func formatNode(prefix string, n *restic.Node, long bool) string {
if !long {
return filepath.Join(prefix, n.Name)
}
switch n.Type {
case "file":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
case "dir":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
case "symlink":
return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
default:
return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
}
}

9
src/cmds/restic/global.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"io"
"io/ioutil"
@ -33,6 +34,7 @@ type GlobalOptions struct {
NoLock bool
JSON bool
ctx context.Context
password string
stdout io.Writer
stderr io.Writer
@ -49,6 +51,13 @@ func init() {
globalOptions.password = pw
}
var cancel context.CancelFunc
globalOptions.ctx, cancel = context.WithCancel(context.Background())
AddCleanupHandler(func() error {
cancel()
return nil
})
f := cmdRoot.PersistentFlags()
f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")

2
src/cmds/restic/integration_helpers_test.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"io/ioutil"
"os"
@ -194,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
gopts := GlobalOptions{
Repo: env.repo,
Quiet: true,
ctx: context.Background(),
password: TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,

4
src/cmds/restic/integration_test.go
View file

@ -142,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
globalOptions.Quiet = quiet
}()
OK(t, runLs(gopts, []string{snapshotID}))
opts := LsOptions{}
OK(t, runLs(opts, gopts, []string{snapshotID}))
return strings.Split(string(buf.Bytes()), "\n")
}

17
src/restic/fuse/snapshot.go
View file

@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{})
type SnapshotsDir struct {
repo restic.Repository
ownerIsRoot bool
paths []string
tags []string
host string
// knownSnapshots maps snapshot timestamp to the snapshot
sync.RWMutex
@ -40,12 +43,15 @@ type SnapshotsDir struct {
}
// NewSnapshotsDir returns a new dir object for the snapshots.
func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir {
func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir {
debug.Log("fuse mount initiated")
return &SnapshotsDir{
repo: repo,
knownSnapshots: make(map[string]SnapshotWithId),
ownerIsRoot: ownerIsRoot,
paths: paths,
tags: tags,
host: host,
knownSnapshots: make(map[string]SnapshotWithId),
processed: restic.NewIDSet(),
}
}
@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
return err
}
// Filter snapshots we don't care for.
if (sn.host != "" && sn.host != snapshot.Hostname) ||
!snapshot.HasTags(sn.tags) ||
!snapshot.HasPaths(sn.paths) {
continue
}
timestamp := snapshot.Time.Format(time.RFC3339)
for i := 1; ; i++ {
if _, ok := sn.knownSnapshots[timestamp]; !ok {

6
src/restic/snapshot.go
View file

@ -177,8 +177,8 @@ func (sn *Snapshot) SamePaths(paths []string) bool {
// ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
var ErrNoSnapshotFound = errors.New("no snapshot found")
// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters.
func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) {
// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters.
func FindLatestSnapshot(repo Repository, targets []string, tags []string, hostname string) (ID, error) {
var (
latest time.Time
latestID ID
@ -190,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID,
if err != nil {
return ID{}, errors.Errorf("Error listing snapshot: %v", err)
}
if snapshot.Time.After(latest) && snapshot.HasPaths(targets) && (hostname == "" || hostname == snapshot.Hostname) {
if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) {
latest = snapshot.Time
latestID = snapshotID
found = true