KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Refactor output of find to allow for json and normal output.

Browse source 
Rather complicated solution becaused I wanted to retain the streaming
character of the output, which means for json I have to manually add
headers and footers per snapshot scanned + a list around the whole
set.

As the json ouput is now partly handcrafted, add proper testing to catch
unintentional changes to the output, making it non-json compliant.

Closes #869
This commit is contained in:
Pauline Middelink 2017-03-09 16:06:28 +01:00
parent 8a05de537f
commit a9707a5728
2 changed files with 154 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
src/cmds/restic/cmd_find.go
View file

@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"time"
@ -84,7 +85,94 @@ func parseTime(str string) (time.Time, error) {
return time.Time{}, errors.Fatalf("unable to parse time: %q", str)
}
func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error {
type statefulOutput struct {
ListLong bool
JSON bool
inuse bool
newsn *restic.Snapshot
oldsn *restic.Snapshot
hits int
}
func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
type findNode restic.Node
b, err := json.Marshal(struct {
// Add these attributes
Path string `json:"path,omitempty"`
Permissions string `json:"permissions,omitempty"`
*findNode
// Make the following attributes disappear
Name byte `json:"name,omitempty"`
Inode byte `json:"inode,omitempty"`
ExtendedAttributes byte `json:"extended_attributes,omitempty"`
Device byte `json:"device,omitempty"`
Content byte `json:"content,omitempty"`
Subtree byte `json:"subtree,omitempty"`
}{
Path: filepath.Join(prefix, node.Name),
Permissions: node.Mode.String(),
findNode: (*findNode)(node),
})
if err != nil {
Warnf("Marshall failed: %v\n", err)
return
}
if !s.inuse {
Printf("[")
s.inuse = true
}
if s.newsn != s.oldsn {
if s.oldsn != nil {
Printf("],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID())
}
Printf(`{"matches":[`)
s.oldsn = s.newsn
s.hits = 0
}
if s.hits > 0 {
Printf(",")
}
Printf(string(b))
s.hits++
}
func (s *statefulOutput) PrintNormal(prefix string, node *restic.Node) {
if s.newsn != s.oldsn {
if s.oldsn != nil {
Verbosef("\n")
}
s.oldsn = s.newsn
Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID())
}
Printf(formatNode(prefix, node, s.ListLong) + "\n")
}
func (s *statefulOutput) Print(prefix string, node *restic.Node) {
if s.JSON {
s.PrintJSON(prefix, node)
} else {
s.PrintNormal(prefix, node)
}
}
func (s *statefulOutput) Finish() {
if s.JSON {
// do some finishing up
if s.oldsn != nil {
Printf("],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID())
}
if s.inuse {
Printf("]\n")
} else {
Printf("[]\n")
}
return
}
}
func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, prefix string, state *statefulOutput) error {
debug.Log("checking tree %v\n", id)
tree, err := repo.LoadTree(id)
@ -117,17 +205,13 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, pref
continue
}
if snapshotID != nil {
Verbosef("Found matching entries in snapshot %s\n", *snapshotID)
snapshotID = nil
}
Printf(formatNode(prefix, node, findOptions.ListLong) + "\n")
state.Print(prefix, node)
} else {
debug.Log(" pattern does not match\n")
}
if node.Type == "dir" {
if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil {
if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), state); err != nil {
return err
}
}
@ -136,11 +220,11 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, pref
return nil
}
func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error {
func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern, state *statefulOutput) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
snapshotID := sn.ID().Str()
if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil {
state.newsn = sn
if err := findInTree(repo, &pat, *sn.Tree, string(filepath.Separator), state); err != nil {
return err
}
return nil
@ -189,11 +273,13 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
state := statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
if err = findInSnapshot(repo, sn, pat); err != nil {
if err = findInSnapshot(repo, sn, pat, &state); err != nil {
return err
}
}
state.Finish()
return nil
}

65
src/cmds/restic/integration_test.go
View file

@ -149,18 +149,20 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
return strings.Split(string(buf.Bytes()), "\n")
}
func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern string) []byte {
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
globalOptions.JSON = wantJSON
defer func() {
globalOptions.stdout = os.Stdout
globalOptions.JSON = false
}()
opts := FindOptions{}
OK(t, runFind(opts, gopts, []string{pattern}))
return strings.Split(string(buf.Bytes()), "\n")
return buf.Bytes()
}
func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
@ -1037,14 +1039,61 @@ func TestFind(t *testing.T) {
testRunBackup(t, []string{env.testdata}, opts, gopts)
testRunCheck(t, gopts)
results := testRunFind(t, gopts, "unexistingfile")
Assert(t, len(results) != 0, "unexisting file found in repo (%v)", datafile)
results := testRunFind(t, false, gopts, "unexistingfile")
Assert(t, len(results) == 0, "unexisting file found in repo (%v)", datafile)
results = testRunFind(t, gopts, "testfile")
Assert(t, len(results) != 1, "file not found in repo (%v)", datafile)
results = testRunFind(t, false, gopts, "testfile")
lines := strings.Split(string(results), "\n")
Assert(t, len(lines) == 2, "expected one file found in repo (%v)", datafile)
results = testRunFind(t, gopts, "test")
Assert(t, len(results) < 2, "less than two file found in repo (%v)", datafile)
results = testRunFind(t, false, gopts, "testfile*")
lines = strings.Split(string(results), "\n")
Assert(t, len(lines) == 4, "expected three files found in repo (%v)", datafile)
})
}
type testMatch struct {
Path string `json:"path,omitempty"`
Permissions string `json:"permissions,omitempty"`
Size uint64 `json:"size,omitempty"`
Date time.Time `json:"date,omitempty"`
UID uint32 `json:"uid,omitempty"`
GID uint32 `json:"gid,omitempty"`
}
type testMatches struct {
Hits int `json:"hits,omitempty"`
SnapshotID string `json:"snapshot,omitempty"`
Matches []testMatch `json:"matches,omitempty"`
}
func TestFindJSON(t *testing.T) {
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
datafile := filepath.Join("testdata", "backup-data.tar.gz")
testRunInit(t, gopts)
SetupTarTestFixture(t, env.testdata, datafile)
opts := BackupOptions{}
testRunBackup(t, []string{env.testdata}, opts, gopts)
testRunCheck(t, gopts)
results := testRunFind(t, true, gopts, "unexistingfile")
matches := []testMatches{}
OK(t, json.Unmarshal(results, &matches))
Assert(t, len(matches) == 0, "expected no match in repo (%v)", datafile)
results = testRunFind(t, true, gopts, "testfile")
OK(t, json.Unmarshal(results, &matches))
Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
Assert(t, len(matches[0].Matches) == 1, "expected a single file to match (%v)", datafile)
Assert(t, matches[0].Hits == 1, "expected hits to show 1 match (%v)", datafile)
results = testRunFind(t, true, gopts, "testfile*")
OK(t, json.Unmarshal(results, &matches))
Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", datafile)
Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
})
}