Restructure backend
This commit is contained in:
parent
0e1045301a
commit
661c1e9aa1
6 changed files with 95 additions and 28 deletions
|
@ -8,15 +8,12 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
MinPrefixLength = 4
|
MinPrefixLength = 4
|
||||||
)
|
)
|
||||||
|
|
||||||
var idPool = sync.Pool{New: func() interface{} { return ID(make([]byte, IDSize)) }}
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrNoIDPrefixFound = errors.New("no ID found")
|
ErrNoIDPrefixFound = errors.New("no ID found")
|
||||||
ErrMultipleIDMatches = errors.New("multiple IDs with prefix found")
|
ErrMultipleIDMatches = errors.New("multiple IDs with prefix found")
|
||||||
|
@ -24,7 +21,10 @@ var (
|
||||||
|
|
||||||
// Each lists all entries of type t in the backend and calls function f() with
|
// Each lists all entries of type t in the backend and calls function f() with
|
||||||
// the id and data.
|
// the id and data.
|
||||||
func Each(be Server, t Type, f func(id ID, data []byte, err error)) error {
|
func Each(be interface {
|
||||||
|
lister
|
||||||
|
getter
|
||||||
|
}, t Type, f func(id ID, data []byte, err error)) error {
|
||||||
ids, err := be.List(t)
|
ids, err := be.List(t)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -45,7 +45,7 @@ func Each(be Server, t Type, f func(id ID, data []byte, err error)) error {
|
||||||
|
|
||||||
// Each lists all entries of type t in the backend and calls function f() with
|
// Each lists all entries of type t in the backend and calls function f() with
|
||||||
// the id.
|
// the id.
|
||||||
func EachID(be Server, t Type, f func(ID)) error {
|
func EachID(be lister, t Type, f func(ID)) error {
|
||||||
ids, err := be.List(t)
|
ids, err := be.List(t)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -101,7 +101,7 @@ func Hash(data []byte) ID {
|
||||||
// Find loads the list of all blobs of type t and searches for IDs which start
|
// Find loads the list of all blobs of type t and searches for IDs which start
|
||||||
// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
|
// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
|
||||||
// more than one is found, nil and ErrMultipleIDMatches is returned.
|
// more than one is found, nil and ErrMultipleIDMatches is returned.
|
||||||
func Find(be Server, t Type, prefix string) (ID, error) {
|
func Find(be lister, t Type, prefix string) (ID, error) {
|
||||||
p, err := hex.DecodeString(prefix)
|
p, err := hex.DecodeString(prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -134,7 +134,7 @@ func Find(be Server, t Type, prefix string) (ID, error) {
|
||||||
|
|
||||||
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
|
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
|
||||||
// the string as closely as possible.
|
// the string as closely as possible.
|
||||||
func FindSnapshot(be Server, s string) (ID, error) {
|
func FindSnapshot(be lister, s string) (ID, error) {
|
||||||
// parse ID directly
|
// parse ID directly
|
||||||
if id, err := ParseID(s); err == nil {
|
if id, err := ParseID(s); err == nil {
|
||||||
return id, nil
|
return id, nil
|
||||||
|
@ -151,7 +151,7 @@ func FindSnapshot(be Server, s string) (ID, error) {
|
||||||
|
|
||||||
// PrefixLength returns the number of bytes required so that all prefixes of
|
// PrefixLength returns the number of bytes required so that all prefixes of
|
||||||
// all IDs of type t are unique.
|
// all IDs of type t are unique.
|
||||||
func PrefixLength(be Lister, t Type) (int, error) {
|
func PrefixLength(be lister, t Type) (int, error) {
|
||||||
// load all IDs of the given type
|
// load all IDs of the given type
|
||||||
list, err := be.List(t)
|
list, err := be.List(t)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -46,9 +46,40 @@ func str2id(s string) backend.ID {
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
type IDList backend.IDs
|
type mockBackend struct {
|
||||||
|
list func(backend.Type) (backend.IDs, error)
|
||||||
|
get func(backend.Type, backend.ID) ([]byte, error)
|
||||||
|
create func(backend.Type, []byte) (backend.ID, error)
|
||||||
|
test func(backend.Type, backend.ID) (bool, error)
|
||||||
|
remove func(backend.Type, backend.ID) error
|
||||||
|
close func() error
|
||||||
|
}
|
||||||
|
|
||||||
var samples = IDList{
|
func (m mockBackend) List(t backend.Type) (backend.IDs, error) {
|
||||||
|
return m.list(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mockBackend) Get(t backend.Type, id backend.ID) ([]byte, error) {
|
||||||
|
return m.get(t, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mockBackend) Create(t backend.Type, data []byte) (backend.ID, error) {
|
||||||
|
return m.create(t, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mockBackend) Test(t backend.Type, id backend.ID) (bool, error) {
|
||||||
|
return m.test(t, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mockBackend) Remove(t backend.Type, id backend.ID) error {
|
||||||
|
return m.remove(t, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mockBackend) Close() error {
|
||||||
|
return m.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
var samples = backend.IDs{
|
||||||
str2id("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
|
str2id("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
|
||||||
str2id("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
|
str2id("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
|
||||||
str2id("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
|
str2id("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
|
||||||
|
@ -59,20 +90,25 @@ var samples = IDList{
|
||||||
str2id("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
|
str2id("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l IDList) List(backend.Type) (backend.IDs, error) {
|
|
||||||
return backend.IDs(l), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrefixLength(t *testing.T) {
|
func TestPrefixLength(t *testing.T) {
|
||||||
l, err := backend.PrefixLength(samples, backend.Snapshot)
|
list := samples
|
||||||
|
|
||||||
|
m := mockBackend{}
|
||||||
|
m.list = func(t backend.Type) (backend.IDs, error) {
|
||||||
|
return list, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
l, err := backend.PrefixLength(m, backend.Snapshot)
|
||||||
ok(t, err)
|
ok(t, err)
|
||||||
equals(t, 10, l)
|
equals(t, 10, l)
|
||||||
|
|
||||||
l, err = backend.PrefixLength(samples[:3], backend.Snapshot)
|
list = samples[:3]
|
||||||
|
l, err = backend.PrefixLength(m, backend.Snapshot)
|
||||||
ok(t, err)
|
ok(t, err)
|
||||||
equals(t, 10, l)
|
equals(t, 10, l)
|
||||||
|
|
||||||
l, err = backend.PrefixLength(samples[3:], backend.Snapshot)
|
list = samples[3:]
|
||||||
|
l, err = backend.PrefixLength(m, backend.Snapshot)
|
||||||
ok(t, err)
|
ok(t, err)
|
||||||
equals(t, 4, l)
|
equals(t, 4, l)
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
// IDSize contains the size of an ID, in bytes.
|
// IDSize contains the size of an ID, in bytes.
|
||||||
|
@ -14,6 +15,8 @@ const IDSize = sha256.Size
|
||||||
// References content within a repository.
|
// References content within a repository.
|
||||||
type ID []byte
|
type ID []byte
|
||||||
|
|
||||||
|
var idPool = sync.Pool{New: func() interface{} { return ID(make([]byte, IDSize)) }}
|
||||||
|
|
||||||
// ParseID converts the given string to an ID.
|
// ParseID converts the given string to an ID.
|
||||||
func ParseID(s string) (ID, error) {
|
func ParseID(s string) (ID, error) {
|
||||||
b, err := hex.DecodeString(s)
|
b, err := hex.DecodeString(s)
|
||||||
|
|
|
@ -21,19 +21,43 @@ var (
|
||||||
ErrAlreadyPresent = errors.New("blob is already present in backend")
|
ErrAlreadyPresent = errors.New("blob is already present in backend")
|
||||||
)
|
)
|
||||||
|
|
||||||
type Lister interface {
|
type lister interface {
|
||||||
List(Type) (IDs, error)
|
List(Type) (IDs, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Server interface {
|
type getter interface {
|
||||||
Create(Type, []byte) (ID, error)
|
|
||||||
Get(Type, ID) ([]byte, error)
|
Get(Type, ID) ([]byte, error)
|
||||||
Lister
|
}
|
||||||
|
|
||||||
|
type creater interface {
|
||||||
|
Create(Type, []byte) (ID, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type tester interface {
|
||||||
Test(Type, ID) (bool, error)
|
Test(Type, ID) (bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type remover interface {
|
||||||
Remove(Type, ID) error
|
Remove(Type, ID) error
|
||||||
Version() uint
|
}
|
||||||
|
|
||||||
|
type closer interface {
|
||||||
Close() error
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type deleter interface {
|
||||||
|
Delete() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type locationer interface {
|
||||||
Location() string
|
Location() string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type backend interface {
|
||||||
|
lister
|
||||||
|
getter
|
||||||
|
creater
|
||||||
|
tester
|
||||||
|
remover
|
||||||
|
closer
|
||||||
|
}
|
||||||
|
|
|
@ -328,3 +328,8 @@ func (b *Local) Version() uint {
|
||||||
func (b *Local) Close() error {
|
func (b *Local) Close() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delete removes the repository and all files.
|
||||||
|
func (b *Local) Delete() error {
|
||||||
|
return os.RemoveAll(b.p)
|
||||||
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
|
||||||
"sort"
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -41,10 +40,10 @@ func teardownBackend(t *testing.T, b *backend.Local) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ok(t, os.RemoveAll(b.Location()))
|
ok(t, b.Delete())
|
||||||
}
|
}
|
||||||
|
|
||||||
func testBackend(b backend.Server, t *testing.T) {
|
func testBackend(b *backend.Local, t *testing.T) {
|
||||||
for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} {
|
for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} {
|
||||||
// detect non-existing files
|
// detect non-existing files
|
||||||
for _, test := range TestStrings {
|
for _, test := range TestStrings {
|
||||||
|
@ -126,10 +125,10 @@ func TestBackend(t *testing.T) {
|
||||||
assert(t, err != nil, "opening invalid repository at /invalid-restic-test should have failed, but err is nil")
|
assert(t, err != nil, "opening invalid repository at /invalid-restic-test should have failed, but err is nil")
|
||||||
assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-restic-test should have failed, but b is not nil: %v", b))
|
assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-restic-test should have failed, but b is not nil: %v", b))
|
||||||
|
|
||||||
b = setupBackend(t)
|
s := setupBackend(t)
|
||||||
defer teardownBackend(t, b)
|
defer teardownBackend(t, s)
|
||||||
|
|
||||||
testBackend(b, t)
|
testBackend(s, t)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLocalBackendCreationFailures(t *testing.T) {
|
func TestLocalBackendCreationFailures(t *testing.T) {
|
||||||
|
|
Loading…
Reference in a new issue