alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge branch 'find-file'

Browse source
This commit is contained in:
Alexander Neumann 2014-12-07 17:24:20 +01:00
commit 2fd29dd5f6
15 changed files with 466 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
archiver.go
View file

@ -384,7 +384,7 @@ func (arch *Archiver) LoadTree(path string, parentSnapshot backend.ID) (*Tree, e
return nil, arrar.Annotate(err, "load old snapshot")
}
if snapshot.Content == nil {
if snapshot.Tree == nil {
return nil, errors.New("snapshot without tree!")
}
@ -394,7 +394,7 @@ func (arch *Archiver) LoadTree(path string, parentSnapshot backend.ID) (*Tree, e
return nil, err
}
oldTree, err = LoadTree(arch.ch, snapshot.Content)
oldTree, err = LoadTree(arch.ch, snapshot.Tree)
if err != nil {
return nil, arrar.Annotate(err, "load old tree")
}
@ -520,7 +520,7 @@ func (arch *Archiver) Snapshot(dir string, t *Tree, parentSnapshot backend.ID) (
if err != nil {
return nil, nil, err
}
sn.Content = blob.ID
sn.Tree = blob.ID
// save bloblist
blob, err = arch.SaveJSON(backend.Map, arch.bl)

2
bloblist_test.go
View file

@ -14,7 +14,7 @@ import (
"github.com/restic/restic/backend"
)
var maxWorkers = flag.Uint("workers", 100, "number of workers to test BlobList concurrent access against")
var maxWorkers = flag.Uint("workers", 20, "number of workers to test BlobList concurrent access against")
func randomID() []byte {
buf := make([]byte, backend.IDSize)

45
cmd/restic/cmd_backup.go
View file

@ -1,7 +1,6 @@
package main
import (
"errors"
"fmt"
"os"
"strings"
@ -12,8 +11,16 @@ import (
"golang.org/x/crypto/ssh/terminal"
)
type CmdBackup struct{}
func init() {
commands["backup"] = commandBackup
_, err := parser.AddCommand("backup",
"save file/directory",
"The backup command creates a snapshot of a file or directory",
&CmdBackup{})
if err != nil {
panic(err)
}
}
func format_bytes(c uint64) string {
@ -56,13 +63,21 @@ func print_tree2(indent int, t *restic.Tree) {
}
}
func commandBackup(be backend.Server, key *restic.Key, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errors.New("usage: backup [dir|file] [snapshot-id]")
func (cmd CmdBackup) Usage() string {
return "DIR/FILE [snapshot-ID]"
}
func (cmd CmdBackup) Execute(args []string) error {
if len(args) == 0 || len(args) > 2 {
return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
var parentSnapshotID backend.ID
var err error
target := args[0]
if len(args) > 1 {
@ -97,8 +112,6 @@ func commandBackup(be backend.Server, key *restic.Key, args []string) error {
}(ch)
}
fmt.Printf("done\n")
// TODO: add filter
// arch.Filter = func(dir string, fi os.FileInfo) bool {
// return true
@ -159,7 +172,7 @@ func commandBackup(be backend.Server, key *restic.Key, args []string) error {
}(ch)
}
sn, id, err := arch.Snapshot(target, t, parentSnapshotID)
_, id, err := arch.Snapshot(target, t, parentSnapshotID)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
}
@ -170,9 +183,17 @@ func commandBackup(be backend.Server, key *restic.Key, args []string) error {
close(arch.ScannerStats)
}
fmt.Printf("\nsnapshot %s saved: %v\n", id, sn)
duration := time.Now().Sub(start)
fmt.Printf("duration: %s, %.2fMiB/s\n", duration, float64(arch.Stats.Bytes)/float64(duration/time.Second)/(1<<20))
plen, err := backend.PrefixLength(be, backend.Snapshot)
if err != nil {
return err
}
fmt.Printf("\nsnapshot %s saved\n", id[:plen])
sec := uint64(time.Since(start) / time.Second)
fmt.Printf("duration: %s, %.2fMiB/s\n",
format_duration(sec),
float64(arch.Stats.Bytes)/float64(sec)/(1<<20))
return nil
}

23
cmd/restic/cmd_cat.go
View file

@ -10,13 +10,30 @@ import (
"github.com/restic/restic/backend"
)
type CmdCat struct{}
func init() {
commands["cat"] = commandCat
_, err := parser.AddCommand("cat",
"dump something",
"The cat command dumps data structures or data from a repository",
&CmdCat{})
if err != nil {
panic(err)
}
}
func commandCat(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdCat) Usage() string {
return "[blob|tree|snapshot|key|lock] ID"
}
func (cmd CmdCat) Execute(args []string) error {
if len(args) != 2 {
return errors.New("usage: cat [blob|tree|snapshot|key|lock] ID")
return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
tpe := args[0]

199
cmd/restic/cmd_find.go Normal file
View file

@ -0,0 +1,199 @@
package main
import (
"fmt"
"path/filepath"
"time"
"github.com/restic/restic"
"github.com/restic/restic/backend"
)
type findQuery struct {
name string
minModTime time.Time
maxModTime time.Time
}
type findResult struct {
node *restic.Node
path string
}
type CmdFind struct {
Oldest string `short:"o" long:"oldest" description:"Oldest modification date/time"`
Newest string `short:"n" long:"newest" description:"Newest modification date/time"`
Snapshot string `short:"s" long:"snapshot" description:"Snapshot ID to search in"`
oldest, newest time.Time
pattern string
}
var timeFormats = []string{
"2006-01-02",
"2006-01-02 15:04",
"2006-01-02 15:04:05",
"2006-01-02 15:04:05 -0700",
"2006-01-02 15:04:05 MST",
"02.01.2006",
"02.01.2006 15:04",
"02.01.2006 15:04:05",
"02.01.2006 15:04:05 -0700",
"02.01.2006 15:04:05 MST",
"Mon Jan 2 15:04:05 -0700 MST 2006",
}
func init() {
_, err := parser.AddCommand("find",
"find a file/directory",
"The find command searches for files or directories in snapshots",
&CmdFind{})
if err != nil {
panic(err)
}
}
func parseTime(str string) (time.Time, error) {
for _, fmt := range timeFormats {
if t, err := time.ParseInLocation(fmt, str, time.Local); err == nil {
return t, nil
}
}
return time.Time{}, fmt.Errorf("unable to parse time: %q", str)
}
func (c CmdFind) findInTree(ch *restic.ContentHandler, id backend.ID, path string) ([]findResult, error) {
debug("checking tree %v\n", id)
tree, err := restic.LoadTree(ch, id)
if err != nil {
return nil, err
}
results := []findResult{}
for _, node := range tree {
debug(" testing entry %q\n", node.Name)
m, err := filepath.Match(c.pattern, node.Name)
if err != nil {
return nil, err
}
if m {
debug(" pattern matches\n")
if !c.oldest.IsZero() && node.ModTime.Before(c.oldest) {
debug(" ModTime is older than %s\n", c.oldest)
continue
}
if !c.newest.IsZero() && node.ModTime.After(c.newest) {
debug(" ModTime is newer than %s\n", c.newest)
continue
}
results = append(results, findResult{node: node, path: path})
} else {
debug(" pattern does not match\n")
}
if node.Type == "dir" {
subdirResults, err := c.findInTree(ch, node.Subtree, filepath.Join(path, node.Name))
if err != nil {
return nil, err
}
results = append(results, subdirResults...)
}
}
return results, nil
}
func (c CmdFind) findInSnapshot(be backend.Server, key *restic.Key, id backend.ID) error {
debug("searching in snapshot %s\n for entries within [%s %s]", id, c.oldest, c.newest)
ch, err := restic.NewContentHandler(be, key)
if err != nil {
return err
}
sn, err := ch.LoadSnapshot(id)
if err != nil {
return err
}
results, err := c.findInTree(ch, sn.Tree, "")
if err != nil {
return err
}
if len(results) == 0 {
return nil
}
fmt.Printf("found %d matching entries in snapshot %s\n", len(results), id)
for _, res := range results {
res.node.Name = filepath.Join(res.path, res.node.Name)
fmt.Printf(" %s\n", res.node)
}
return nil
}
func (CmdFind) Usage() string {
return "[find-OPTIONS] PATTERN"
}
func (c CmdFind) Execute(args []string) error {
if len(args) != 1 {
return fmt.Errorf("invalid number of arguments, Usage: %s", c.Usage())
}
var err error
if c.Oldest != "" {
c.oldest, err = parseTime(c.Oldest)
if err != nil {
return err
}
}
if c.Newest != "" {
c.newest, err = parseTime(c.Newest)
if err != nil {
return err
}
}
be, key, err := OpenRepo()
if err != nil {
return err
}
c.pattern = args[0]
if c.Snapshot != "" {
snapshotID, err := backend.FindSnapshot(be, c.Snapshot)
if err != nil {
return fmt.Errorf("invalid id %q: %v", args[1], err)
}
return c.findInSnapshot(be, key, snapshotID)
}
list, err := be.List(backend.Snapshot)
if err != nil {
return err
}
for _, snapshotID := range list {
err := c.findInSnapshot(be, key, snapshotID)
if err != nil {
return err
}
}
return nil
}

28
cmd/restic/cmd_fsck.go
View file

@ -1,15 +1,22 @@
package main
import (
"errors"
"fmt"
"github.com/restic/restic"
"github.com/restic/restic/backend"
)
type CmdFsck struct{}
func init() {
commands["fsck"] = commandFsck
_, err := parser.AddCommand("fsck",
"check the repository",
"The fsck command check the integrity and consistency of the repository",
&CmdFsck{})
if err != nil {
panic(err)
}
}
func fsckFile(ch *restic.ContentHandler, IDs []backend.ID) error {
@ -81,7 +88,7 @@ func fsck_snapshot(be backend.Server, key *restic.Key, id backend.ID) error {
return err
}
if sn.Content == nil {
if sn.Tree == nil {
return fmt.Errorf("snapshot %v has no content", sn.ID)
}
@ -89,12 +96,21 @@ func fsck_snapshot(be backend.Server, key *restic.Key, id backend.ID) error {
return fmt.Errorf("snapshot %v has no map", sn.ID)
}
return fsckTree(ch, sn.Content)
return fsckTree(ch, sn.Tree)
}
func commandFsck(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdFsck) Usage() string {
return "fsck [all|snapshot-ID]"
}
func (cmd CmdFsck) Execute(args []string) error {
if len(args) == 0 {
return errors.New("usage: fsck [all|snapshot-id]")
return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
if len(args) == 1 && args[0] != "all" {

23
cmd/restic/cmd_key.go
View file

@ -10,8 +10,16 @@ import (
"github.com/restic/restic/backend"
)
type CmdKey struct{}
func init() {
commands["key"] = commandKey
_, err := parser.AddCommand("key",
"manage keys",
"The key command manages keys (passwords) of a repository",
&CmdKey{})
if err != nil {
panic(err)
}
}
func list_keys(be backend.Server, key *restic.Key) error {
@ -103,9 +111,18 @@ func change_password(be backend.Server, key *restic.Key) error {
return nil
}
func commandKey(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdKey) Usage() string {
return "[list|add|rm|change] [ID]"
}
func (cmd CmdKey) Execute(args []string) error {
if len(args) < 1 || (args[0] == "rm" && len(args) != 2) {
return errors.New("usage: key [list|add|rm|change] [ID]")
return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
switch args[0] {

25
cmd/restic/cmd_list.go
View file

@ -3,18 +3,33 @@ package main
import (
"errors"
"fmt"
"github.com/restic/restic"
"github.com/restic/restic/backend"
)
type CmdList struct{}
func init() {
commands["list"] = commandList
_, err := parser.AddCommand("list",
"lists data",
"The list command lists structures or data of a repository",
&CmdList{})
if err != nil {
panic(err)
}
}
func commandList(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdList) Usage() string {
return "[data|trees|snapshots|keys|locks]"
}
func (cmd CmdList) Execute(args []string) error {
if len(args) != 1 {
return errors.New("usage: list [data|trees|snapshots|keys|locks]")
return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
var (

26
cmd/restic/cmd_ls.go
View file

@ -1,7 +1,6 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
@ -10,8 +9,16 @@ import (
"github.com/restic/restic/backend"
)
type CmdLs struct{}
func init() {
commands["ls"] = commandLs
_, err := parser.AddCommand("ls",
"list files",
"The ls command lists all files and directories in a snapshot",
&CmdLs{})
if err != nil {
panic(err)
}
}
func print_node(prefix string, n *restic.Node) string {
@ -52,9 +59,18 @@ func print_tree(prefix string, ch *restic.ContentHandler, id backend.ID) error {
return nil
}
func commandLs(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdLs) Usage() string {
return "ls snapshot-ID [DIR]"
}
func (cmd CmdLs) Execute(be backend.Server, key *restic.Key, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errors.New("usage: ls SNAPSHOT_ID [dir]")
return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
id, err := backend.FindSnapshot(be, args[0])
@ -74,5 +90,5 @@ func commandLs(be backend.Server, key *restic.Key, args []string) error {
fmt.Printf("snapshot of %s at %s:\n", sn.Dir, sn.Time)
return print_tree("", ch, sn.Content)
return print_tree("", ch, sn.Tree)
}

24
cmd/restic/cmd_restore.go
View file

@ -1,7 +1,6 @@
package main
import (
"errors"
"fmt"
"os"
@ -9,13 +8,30 @@ import (
"github.com/restic/restic/backend"
)
type CmdRestore struct{}
func init() {
commands["restore"] = commandRestore
_, err := parser.AddCommand("restore",
"restore a snapshot",
"The restore command restores a snapshot to a directory",
&CmdRestore{})
if err != nil {
panic(err)
}
}
func commandRestore(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdRestore) Usage() string {
return "snapshot-ID TARGETDIR"
}
func (cmd CmdRestore) Execute(args []string) error {
if len(args) != 2 {
return errors.New("usage: restore ID dir")
return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
id, err := backend.FindSnapshot(be, args[0])

24
cmd/restic/cmd_snapshots.go
View file

@ -1,7 +1,6 @@
package main
import (
"errors"
"fmt"
"io"
"os"
@ -72,13 +71,30 @@ func reltime(t time.Time) string {
}
}
type CmdSnapshots struct{}
func init() {
commands["snapshots"] = commandSnapshots
_, err := parser.AddCommand("snapshots",
"show snapshots",
"The snapshots command lists all snapshots stored in a repository",
&CmdSnapshots{})
if err != nil {
panic(err)
}
}
func commandSnapshots(be backend.Server, key *restic.Key, args []string) error {
func (cmd CmdSnapshots) Usage() string {
return ""
}
func (cmd CmdSnapshots) Execute(args []string) error {
if len(args) != 0 {
return errors.New("usage: snapshots")
return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage())
}
be, key, err := OpenRepo()
if err != nil {
return err
}
ch, err := restic.NewContentHandler(be, key)

140
cmd/restic/main.go
View file

@ -1,13 +1,11 @@
package main
import (
"errors"
"fmt"
"log"
"net/url"
"os"
"runtime"
"sort"
"strings"
"golang.org/x/crypto/ssh/terminal"
@ -22,6 +20,8 @@ var opts struct {
Repo string `short:"r" long:"repo" description:"Repository directory to backup to/restore from"`
}
var parser = flags.NewParser(&opts, flags.Default)
func errx(code int, format string, data ...interface{}) {
if len(format) > 0 && format[len(format)-1] != '\n' {
format += "\n"
@ -30,10 +30,6 @@ func errx(code int, format string, data ...interface{}) {
os.Exit(code)
}
type commandFunc func(backend.Server, *restic.Key, []string) error
var commands = make(map[string]commandFunc)
func readPassword(env string, prompt string) string {
if env != "" {
@ -54,7 +50,13 @@ func readPassword(env string, prompt string) string {
return string(pw)
}
func commandInit(repo string) error {
type CmdInit struct{}
func (cmd CmdInit) Execute(args []string) error {
if opts.Repo == "" {
return errors.New("Please specify repository location (-r)")
}
pw := readPassword("RESTIC_PASSWORD", "enter password for new backend: ")
pw2 := readPassword("RESTIC_PASSWORD", "enter password again: ")
@ -62,15 +64,15 @@ func commandInit(repo string) error {
errx(1, "passwords do not match")
}
be, err := create(repo)
be, err := create(opts.Repo)
if err != nil {
fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", repo, err)
fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", opts.Repo, err)
os.Exit(1)
}
_, err = restic.CreateKey(be, pw)
if err != nil {
fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", repo, err)
fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err)
os.Exit(1)
}
@ -125,73 +127,99 @@ func create(u string) (backend.Server, error) {
return backend.CreateSFTP(url.Path[1:], "ssh", args...)
}
func OpenRepo() (backend.Server, *restic.Key, error) {
be, err := open(opts.Repo)
if err != nil {
return nil, nil, err
}
key, err := restic.SearchKey(be, readPassword("RESTIC_PASSWORD", "Enter Password for Repository: "))
if err != nil {
return nil, nil, fmt.Errorf("unable to open repo: %v", err)
}
return be, key, nil
}
func init() {
// set GOMAXPROCS to number of CPUs
runtime.GOMAXPROCS(runtime.NumCPU())
_, err := parser.AddCommand("init",
"create repository",
"The init command creates a new repository",
&CmdInit{})
if err != nil {
panic(err)
}
}
func main() {
// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
log.SetOutput(os.Stdout)
opts.Repo = os.Getenv("RESTIC_REPOSITORY")
args, err := flags.Parse(&opts)
_, err := parser.Parse()
if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
os.Exit(0)
}
if opts.Repo == "" {
fmt.Fprintf(os.Stderr, "no repository specified, use -r or RESTIC_REPOSITORY variable\n")
if err != nil {
os.Exit(1)
}
if len(args) == 0 {
cmds := []string{"init"}
for k := range commands {
cmds = append(cmds, k)
}
sort.Strings(cmds)
fmt.Printf("nothing to do, available commands: [%v]\n", strings.Join(cmds, "|"))
os.Exit(0)
}
// fmt.Printf("parser: %#v\n", parser)
// fmt.Printf("%#v\n", parser.Active.Name)
cmd := args[0]
// if opts.Repo == "" {
// fmt.Fprintf(os.Stderr, "no repository specified, use -r or RESTIC_REPOSITORY variable\n")
// os.Exit(1)
// }
switch cmd {
case "init":
err = commandInit(opts.Repo)
if err != nil {
errx(1, "error executing command %q: %v", cmd, err)
}
return
// if len(args) == 0 {
// cmds := []string{"init"}
// for k := range commands {
// cmds = append(cmds, k)
// }
// sort.Strings(cmds)
// fmt.Printf("nothing to do, available commands: [%v]\n", strings.Join(cmds, "|"))
// os.Exit(0)
// }
case "version":
fmt.Printf("%v\n", version)
return
}
// cmd := args[0]
f, ok := commands[cmd]
if !ok {
errx(1, "unknown command: %q\n", cmd)
}
// switch cmd {
// case "init":
// err = commandInit(opts.Repo)
// if err != nil {
// errx(1, "error executing command %q: %v", cmd, err)
// }
// return
// read_password("enter password: ")
repo, err := open(opts.Repo)
if err != nil {
errx(1, "unable to open repo: %v", err)
}
// case "version":
// fmt.Printf("%v\n", version)
// return
// }
key, err := restic.SearchKey(repo, readPassword("RESTIC_PASSWORD", "Enter Password for Repository: "))
if err != nil {
errx(2, "unable to open repo: %v", err)
}
// f, ok := commands[cmd]
// if !ok {
// errx(1, "unknown command: %q\n", cmd)
// }
err = f(repo, key, args[1:])
if err != nil {
errx(1, "error executing command %q: %v", cmd, err)
}
// // read_password("enter password: ")
// repo, err := open(opts.Repo)
// if err != nil {
// errx(1, "unable to open repo: %v", err)
// }
restic.PoolAlloc()
// key, err := restic.SearchKey(repo, readPassword("RESTIC_PASSWORD", "Enter Password for Repository: "))
// if err != nil {
// errx(2, "unable to open repo: %v", err)
// }
// err = f(repo, key, args[1:])
// if err != nil {
// errx(1, "error executing command %q: %v", cmd, err)
// }
// restic.PoolAlloc()
}

2
restorer.go
View file

@ -93,7 +93,7 @@ func (res *Restorer) RestoreTo(dir string) error {
return err
}
return res.to(dir, res.sn.Content)
return res.to(dir, res.sn.Tree)
}
func (res *Restorer) Snapshot() *Snapshot {

2
snapshot.go
View file

@ -13,7 +13,7 @@ import (
type Snapshot struct {
Time time.Time `json:"time"`
Parent backend.ID `json:"parent,omitempty"`
Content backend.ID `json:"content"`
Tree backend.ID `json:"tree"`
Map backend.ID `json:"map"`
Dir string `json:"dir"`
Hostname string `json:"hostname,omitempty"`

2
snapshot_test.go
View file

@ -11,7 +11,7 @@ import (
func testSnapshot(t *testing.T, be backend.Server) {
var err error
sn := restic.NewSnapshot("/home/foobar")
sn.Content, err = backend.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")
sn.Tree, err = backend.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")
ok(t, err)
sn.Time, err = time.Parse(time.RFC3339Nano, "2014-08-03T17:49:05.378595539+02:00")
ok(t, err)