From b6f25aa6907c706b29d644b00fdc80a6934568a7 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 17 Feb 2015 23:05:23 +0100 Subject: [PATCH] Remove Each(), add basic stats --- backend/generic.go | 24 ---------------- cmd/restic/cmd_key.go | 7 ++--- cmd/restic/cmd_list.go | 15 ++-------- key.go | 28 ++++++++++++------ server.go | 64 +++++++++++++++++++++++++++--------------- server_test.go | 15 ++++++++++ 6 files changed, 82 insertions(+), 71 deletions(-) diff --git a/backend/generic.go b/backend/generic.go index 07a7cde2e..1e16ee2ec 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -24,30 +24,6 @@ var ( const hashSize = sha256.Size -// Each lists all entries of type t in the backend and calls function f() with -// the id and data. -func Each(be interface { - Lister - Getter -}, t Type, f func(id ID, data []byte, err error)) error { - ids, err := be.List(t) - if err != nil { - return err - } - - for _, id := range ids { - data, err := be.Get(t, id) - if err != nil { - f(id, nil, err) - continue - } - - f(id, data, nil) - } - - return nil -} - // Each lists all entries of type t in the backend and calls function f() with // the id. func EachID(be Lister, t Type, f func(ID)) error { diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 09d4de557..7a5846082 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "errors" "fmt" "os" @@ -32,10 +31,10 @@ func list_keys(s restic.Server) error { return err } - s.Each(backend.Key, func(id backend.ID, data []byte, err error) { - k := restic.Key{} - err = json.Unmarshal(data, &k) + s.EachID(backend.Key, func(id backend.ID) { + k, err := restic.LoadKey(s, id) if err != nil { + fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err) return } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 58e2b4301..77b779815 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -33,17 +33,12 @@ func (cmd CmdList) Execute(args []string) error { return err } - var ( - t backend.Type - each func(backend.Type, func(backend.ID, []byte, error)) error = s.Each - ) + var t backend.Type switch args[0] { case "data": t = backend.Data - each = s.EachDecrypted case "trees": t = backend.Tree - each = s.EachDecrypted case "snapshots": t = backend.Snapshot case "keys": @@ -54,11 +49,7 @@ func (cmd CmdList) Execute(args []string) error { return errors.New("invalid type") } - return each(t, func(id backend.ID, data []byte, err error) { - if t == backend.Data || t == backend.Tree { - fmt.Printf("%s %s\n", id, backend.Hash(data)) - } else { - fmt.Printf("%s\n", id) - } + return s.EachID(t, func(id backend.ID) { + fmt.Printf("%s\n", id) }) } diff --git a/key.go b/key.go index 436206171..8475e744e 100644 --- a/key.go +++ b/key.go @@ -84,15 +84,7 @@ func CreateKey(s Server, password string) (*Key, error) { // OpenKey tries do decrypt the key specified by id with the given password. func OpenKey(s Server, id backend.ID, password string) (*Key, error) { - // extract data from repo - data, err := s.Get(backend.Key, id) - if err != nil { - return nil, err - } - - // restore json - k := &Key{} - err = json.Unmarshal(data, k) + k, err := LoadKey(s, id) if err != nil { return nil, err } @@ -148,6 +140,24 @@ func SearchKey(s Server, password string) (*Key, error) { return nil, ErrNoKeyFound } +// LoadKey loads a key from the backend. +func LoadKey(s Server, id backend.ID) (*Key, error) { + // extract data from repo + data, err := s.Get(backend.Key, id) + if err != nil { + return nil, err + } + + // restore json + k := &Key{} + err = json.Unmarshal(data, k) + if err != nil { + return nil, err + } + + return k, err +} + // AddKey adds a new key to an already existing repository. func AddKey(s Server, password string, template *Key) (*Key, error) { // fill meta data about key diff --git a/server.go b/server.go index ce6b3c287..79a8f8d8c 100644 --- a/server.go +++ b/server.go @@ -26,12 +26,6 @@ func NewServerWithKey(be backend.Backend, key *Key) Server { return Server{be: be, key: key} } -// Each lists all entries of type t in the backend and calls function f() with -// the id and data. -func (s Server) Each(t backend.Type, f func(id backend.ID, data []byte, err error)) error { - return backend.Each(s.be, t, f) -} - // Each lists all entries of type t in the backend and calls function f() with // the id. func (s Server) EachID(t backend.Type, f func(backend.ID)) error { @@ -348,27 +342,53 @@ func (s Server) Key() *Key { return s.key } -// Each calls Each() with the given parameters, Decrypt() on the ciphertext -// and, on successful decryption, f with the plaintext. -func (s Server) EachDecrypted(t backend.Type, f func(backend.ID, []byte, error)) error { - if s.key == nil { - return errors.New("key for server not set") +type ServerStats struct { + Blobs, Trees uint + Bytes uint64 +} + +// Stats returns statistics for this backend and the server. +func (s Server) Stats() (ServerStats, error) { + blobs := backend.NewIDSet() + + // load all trees, in parallel + worker := func(wg *sync.WaitGroup, c <-chan backend.ID) { + for id := range c { + tree, err := LoadTree(s, id) + // ignore error and advance to next tree + if err != nil { + return + } + + for _, id := range tree.Map.StorageIDs() { + blobs.Insert(id) + } + } + wg.Done() } - return s.Each(t, func(id backend.ID, data []byte, e error) { - if e != nil { - f(id, nil, e) - return - } + idCh := make(chan backend.ID) - buf, err := s.key.Decrypt([]byte{}, data) - if err != nil { - f(id, nil, err) - return - } + // start workers + var wg sync.WaitGroup + for i := 0; i < maxConcurrency; i++ { + wg.Add(1) + go worker(&wg, idCh) + } - f(id, buf, nil) + // list ids + trees := 0 + err := s.EachID(backend.Tree, func(id backend.ID) { + trees++ + idCh <- id }) + + close(idCh) + + // wait for workers + wg.Wait() + + return ServerStats{Blobs: uint(blobs.Len()), Trees: uint(trees)}, err } // Proxy methods to backend diff --git a/server_test.go b/server_test.go index 9b9172de8..5c3f2dd46 100644 --- a/server_test.go +++ b/server_test.go @@ -123,3 +123,18 @@ func BenchmarkSaveFrom(t *testing.B) { ok(t, err) } } + +func TestServerStats(t *testing.T) { + be := setupBackend(t) + defer teardownBackend(t, be) + key := setupKey(t, be, "geheim") + server := restic.NewServerWithKey(be, key) + + // archive a few files + sn := snapshot(t, server, *benchArchiveDirectory) + t.Logf("archived snapshot %v", sn.ID) + + stats, err := server.Stats() + ok(t, err) + t.Logf("stats: %v", stats) +}