From b5b3c0eaf8eacd9d4695e5ef8a245fd3a899008a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 3 Sep 2016 20:55:22 +0200 Subject: [PATCH] Add repository.SaveTree --- src/restic/archiver/archive_reader.go | 20 +---- src/restic/archiver/archiver.go | 2 +- src/restic/repository.go | 6 +- src/restic/repository/repository.go | 95 ++++++++++++------------ src/restic/repository/repository_test.go | 53 ------------- src/restic/testing.go | 12 +-- src/restic/tree_test.go | 2 +- 7 files changed, 61 insertions(+), 129 deletions(-) diff --git a/src/restic/archiver/archive_reader.go b/src/restic/archiver/archive_reader.go index 1868ee0c1..fb3803e34 100644 --- a/src/restic/archiver/archive_reader.go +++ b/src/restic/archiver/archive_reader.go @@ -1,7 +1,6 @@ package archiver import ( - "encoding/json" "io" "restic" "restic/debug" @@ -12,23 +11,6 @@ import ( "github.com/restic/chunker" ) -// saveTreeJSON stores a tree in the repository. -func saveTreeJSON(repo restic.Repository, item interface{}) (restic.ID, error) { - data, err := json.Marshal(item) - if err != nil { - return restic.ID{}, errors.Wrap(err, "") - } - data = append(data, '\n') - - // check if tree has been saved before - id := restic.Hash(data) - if repo.Index().Has(id, restic.TreeBlob) { - return id, nil - } - - return repo.SaveJSON(restic.TreeBlob, item) -} - // ArchiveReader reads from the reader and archives the data. Returned is the // resulting snapshot and its ID. func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string) (*restic.Snapshot, restic.ID, error) { @@ -93,7 +75,7 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam }, } - treeID, err := saveTreeJSON(repo, tree) + treeID, err := repo.SaveTree(tree) if err != nil { return nil, restic.ID{}, err } diff --git a/src/restic/archiver/archiver.go b/src/restic/archiver/archiver.go index 6decc5fd5..3be272adc 100644 --- a/src/restic/archiver/archiver.go +++ b/src/restic/archiver/archiver.go @@ -122,7 +122,7 @@ func (arch *Archiver) SaveTreeJSON(item interface{}) (restic.ID, error) { return id, nil } - return arch.repo.SaveJSON(restic.TreeBlob, item) + return arch.repo.SaveBlob(restic.TreeBlob, data, id) } func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*restic.Node, error) { diff --git a/src/restic/repository.go b/src/restic/repository.go index c84ba4fd9..959c0bd3c 100644 --- a/src/restic/repository.go +++ b/src/restic/repository.go @@ -27,17 +27,17 @@ type Repository interface { Flush() error - SaveJSON(BlobType, interface{}) (ID, error) SaveUnpacked(FileType, []byte) (ID, error) SaveJSONUnpacked(FileType, interface{}) (ID, error) LoadJSONUnpacked(FileType, ID, interface{}) error LoadAndDecrypt(FileType, ID) ([]byte, error) - LoadTree(ID) (*Tree, error) LoadBlob(BlobType, ID, []byte) (int, error) - SaveBlob(BlobType, []byte, ID) (ID, error) + + LoadTree(ID) (*Tree, error) + SaveTree(t *Tree) (ID, error) } // Deleter removes all data stored in a backend/repo. diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index ca5fc92b6..a7258e090 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -227,25 +227,6 @@ func (r *Repository) SaveAndEncrypt(t restic.BlobType, data []byte, id *restic.I return *id, r.savePacker(packer) } -// SaveJSON serialises item as JSON and encrypts and saves it in a pack in the -// backend as type t. -func (r *Repository) SaveJSON(t restic.BlobType, item interface{}) (restic.ID, error) { - debug.Log("Repo.SaveJSON", "save %v blob", t) - buf := getBuf()[:0] - defer freeBuf(buf) - - wr := bytes.NewBuffer(buf) - - enc := json.NewEncoder(wr) - err := enc.Encode(item) - if err != nil { - return restic.ID{}, errors.Errorf("json.Encode: %v", err) - } - - buf = wr.Bytes() - return r.SaveAndEncrypt(t, buf, nil) -} - // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the // backend as type t, without a pack. It returns the storage hash. func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (restic.ID, error) { @@ -565,33 +546,6 @@ func (r *Repository) Close() error { return r.be.Close() } -// LoadTree loads a tree from the repository. -func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) { - debug.Log("repo.LoadTree", "load tree %v", id.Str()) - - size, err := r.idx.LookupSize(id, restic.TreeBlob) - if err != nil { - return nil, err - } - - debug.Log("repo.LoadTree", "size is %d, create buffer", size) - buf := make([]byte, size) - - n, err := r.loadBlob(id, restic.TreeBlob, buf) - if err != nil { - return nil, err - } - buf = buf[:n] - - t := &restic.Tree{} - err = json.Unmarshal(buf, t) - if err != nil { - return nil, err - } - - return t, nil -} - // LoadBlob loads a blob of type t from the repository to the buffer. func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) { debug.Log("repo.LoadBlob", "load blob %v into buf %p", id.Str(), buf) @@ -624,3 +578,52 @@ func (r *Repository) SaveBlob(t restic.BlobType, buf []byte, id restic.ID) (rest } return r.SaveAndEncrypt(t, buf, i) } + +// LoadTree loads a tree from the repository. +func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) { + debug.Log("repo.LoadTree", "load tree %v", id.Str()) + + size, err := r.idx.LookupSize(id, restic.TreeBlob) + if err != nil { + return nil, err + } + + debug.Log("repo.LoadTree", "size is %d, create buffer", size) + buf := make([]byte, size) + + n, err := r.loadBlob(id, restic.TreeBlob, buf) + if err != nil { + return nil, err + } + buf = buf[:n] + + t := &restic.Tree{} + err = json.Unmarshal(buf, t) + if err != nil { + return nil, err + } + + return t, nil +} + +// SaveTree stores a tree into the repository and returns the ID. The ID is +// checked against the index. The tree is only stored when the index does not +// contain the ID. +func (r *Repository) SaveTree(t *restic.Tree) (restic.ID, error) { + buf, err := json.Marshal(t) + if err != nil { + return restic.ID{}, errors.Wrap(err, "MarshalJSON") + } + + // append a newline so that the data is always consistent (json.Encoder + // adds a newline after each object) + buf = append(buf, '\n') + + id := restic.Hash(buf) + if r.idx.Has(id, restic.TreeBlob) { + return id, nil + } + + _, err = r.SaveBlob(restic.TreeBlob, buf, id) + return id, err +} diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 3910f57be..3295ab5d9 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/rand" "crypto/sha256" - "encoding/json" "io" mrand "math/rand" "path/filepath" @@ -15,58 +14,6 @@ import ( . "restic/test" ) -type testJSONStruct struct { - Foo uint32 - Bar string - Baz []byte -} - -var repoTests = []testJSONStruct{ - testJSONStruct{Foo: 23, Bar: "Teststring", Baz: []byte("xx")}, -} - -func TestSaveJSON(t *testing.T) { - repo := SetupRepo() - defer TeardownRepo(repo) - - for _, obj := range repoTests { - data, err := json.Marshal(obj) - OK(t, err) - data = append(data, '\n') - h := sha256.Sum256(data) - - id, err := repo.SaveJSON(restic.TreeBlob, obj) - OK(t, err) - - Assert(t, h == id, - "TestSaveJSON: wrong plaintext ID: expected %02x, got %02x", - h, id) - } -} - -func BenchmarkSaveJSON(t *testing.B) { - repo := SetupRepo() - defer TeardownRepo(repo) - - obj := repoTests[0] - - data, err := json.Marshal(obj) - OK(t, err) - data = append(data, '\n') - h := sha256.Sum256(data) - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - id, err := repo.SaveJSON(restic.TreeBlob, obj) - OK(t, err) - - Assert(t, h == id, - "TestSaveJSON: wrong plaintext ID: expected %02x, got %02x", - h, id) - } -} - var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} func TestSave(t *testing.T) { diff --git a/src/restic/testing.go b/src/restic/testing.go index 85cab0d69..039b908f7 100644 --- a/src/restic/testing.go +++ b/src/restic/testing.go @@ -64,17 +64,16 @@ const ( maxNodes = 32 ) -func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, ID) { +func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) { data, err := json.Marshal(tree) if err != nil { fs.t.Fatalf("json.Marshal(tree) returned error: %v", err) - return false, ID{} + return false, nil, ID{} } data = append(data, '\n') id := Hash(data) - return fs.blobIsKnown(id, TreeBlob), id - + return fs.blobIsKnown(id, TreeBlob), data, id } func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool { @@ -132,11 +131,12 @@ func (fs fakeFileSystem) saveTree(seed int64, depth int) ID { tree.Nodes = append(tree.Nodes, node) } - if known, id := fs.treeIsKnown(&tree); known { + known, buf, id := fs.treeIsKnown(&tree) + if known { return id } - id, err := fs.repo.SaveJSON(TreeBlob, tree) + _, err := fs.repo.SaveBlob(TreeBlob, buf, id) if err != nil { fs.t.Fatal(err) } diff --git a/src/restic/tree_test.go b/src/restic/tree_test.go index 3c581ec68..779842552 100644 --- a/src/restic/tree_test.go +++ b/src/restic/tree_test.go @@ -97,7 +97,7 @@ func TestLoadTree(t *testing.T) { // save tree tree := restic.NewTree() - id, err := repo.SaveJSON(restic.TreeBlob, tree) + id, err := repo.SaveTree(tree) OK(t, err) // save packs