Copy ID from backend to restic

src/restic/id.go

@@ -0,0 +1,109 @@
+package restic
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+// Hash returns the ID for data.
+func Hash(data []byte) ID {
+	return sha256.Sum256(data)
+}
+
+// IDSize contains the size of an ID, in bytes.
+const IDSize = sha256.Size
+
+// ID references content within a repository.
+type ID [IDSize]byte
+
+// ParseID converts the given string to an ID.
+func ParseID(s string) (ID, error) {
+	b, err := hex.DecodeString(s)
+
+	if err != nil {
+		return ID{}, errors.Wrap(err, "hex.DecodeString")
+	}
+
+	if len(b) != IDSize {
+		return ID{}, errors.New("invalid length for hash")
+	}
+
+	id := ID{}
+	copy(id[:], b)
+
+	return id, nil
+}
+
+func (id ID) String() string {
+	return hex.EncodeToString(id[:])
+}
+
+const shortStr = 4
+
+// Str returns the shortened string version of id.
+func (id *ID) Str() string {
+	if id == nil {
+		return "[nil]"
+	}
+
+	if id.IsNull() {
+		return "[null]"
+	}
+
+	return hex.EncodeToString(id[:shortStr])
+}
+
+// IsNull returns true iff id only consists of null bytes.
+func (id ID) IsNull() bool {
+	var nullID ID
+
+	return id == nullID
+}
+
+// Equal compares an ID to another other.
+func (id ID) Equal(other ID) bool {
+	return id == other
+}
+
+// EqualString compares this ID to another one, given as a string.
+func (id ID) EqualString(other string) (bool, error) {
+	s, err := hex.DecodeString(other)
+	if err != nil {
+		return false, errors.Wrap(err, "hex.DecodeString")
+	}
+
+	id2 := ID{}
+	copy(id2[:], s)
+
+	return id == id2, nil
+}
+
+// Compare compares this ID to another one, returning -1, 0, or 1.
+func (id ID) Compare(other ID) int {
+	return bytes.Compare(other[:], id[:])
+}
+
+// MarshalJSON returns the JSON encoding of id.
+func (id ID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(id.String())
+}
+
+// UnmarshalJSON parses the JSON-encoded data and stores the result in id.
+func (id *ID) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return errors.Wrap(err, "Unmarshal")
+	}
+
+	_, err = hex.Decode(id[:], []byte(s))
+	if err != nil {
+		return errors.Wrap(err, "hex.Decode")
+	}
+
+	return nil
+}

src/restic/id_int_test.go

@@ -0,0 +1,16 @@
+package restic
+
+import "testing"
+
+func TestIDMethods(t *testing.T) {
+	var id ID
+
+	if id.Str() != "[null]" {
+		t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[null]", id.Str())
+	}
+
+	var pid *ID
+	if pid.Str() != "[nil]" {
+		t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[nil]", pid.Str())
+	}
+}

src/restic/id_test.go

@@ -0,0 +1,60 @@
+package restic
+
+import (
+	"reflect"
+	"testing"
+)
+
+var TestStrings = []struct {
+	id   string
+	data string
+}{
+	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
+	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
+	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
+	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
+}
+
+func TestID(t *testing.T) {
+	for _, test := range TestStrings {
+		id, err := ParseID(test.id)
+		if err != nil {
+			t.Error(err)
+		}
+
+		id2, err := ParseID(test.id)
+		if err != nil {
+			t.Error(err)
+		}
+		if !id.Equal(id2) {
+			t.Errorf("ID.Equal() does not work as expected")
+		}
+
+		ret, err := id.EqualString(test.id)
+		if err != nil {
+			t.Error(err)
+		}
+		if !ret {
+			t.Error("ID.EqualString() returned wrong value")
+		}
+
+		// test json marshalling
+		buf, err := id.MarshalJSON()
+		if err != nil {
+			t.Error(err)
+		}
+		want := `"` + test.id + `"`
+		if string(buf) != want {
+			t.Errorf("string comparison failed, wanted %q, got %q", want, string(buf))
+		}
+
+		var id3 ID
+		err = id3.UnmarshalJSON(buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(id, id3) {
+			t.Error("ids are not equal")
+		}
+	}
+}

src/restic/ids.go

@@ -0,0 +1,69 @@
+package restic
+
+import (
+	"encoding/hex"
+	"fmt"
+)
+
+// IDs is an ordered list of IDs that implements sort.Interface.
+type IDs []ID
+
+func (ids IDs) Len() int {
+	return len(ids)
+}
+
+func (ids IDs) Less(i, j int) bool {
+	if len(ids[i]) < len(ids[j]) {
+		return true
+	}
+
+	for k, b := range ids[i] {
+		if b == ids[j][k] {
+			continue
+		}
+
+		if b < ids[j][k] {
+			return true
+		}
+
+		return false
+	}
+
+	return false
+}
+
+func (ids IDs) Swap(i, j int) {
+	ids[i], ids[j] = ids[j], ids[i]
+}
+
+// Uniq returns list without duplicate IDs. The returned list retains the order
+// of the original list so that the order of the first occurrence of each ID
+// stays the same.
+func (ids IDs) Uniq() (list IDs) {
+	seen := NewIDSet()
+
+	for _, id := range ids {
+		if seen.Has(id) {
+			continue
+		}
+
+		list = append(list, id)
+		seen.Insert(id)
+	}
+
+	return list
+}
+
+type shortID ID
+
+func (id shortID) String() string {
+	return hex.EncodeToString(id[:shortStr])
+}
+
+func (ids IDs) String() string {
+	elements := make([]shortID, 0, len(ids))
+	for _, id := range ids {
+		elements = append(elements, shortID(id))
+	}
+	return fmt.Sprintf("%v", elements)
+}

src/restic/ids_test.go

@@ -0,0 +1,55 @@
+package restic
+
+import (
+	"reflect"
+	"testing"
+)
+
+var uniqTests = []struct {
+	before, after IDs
+}{
+	{
+		IDs{
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+			TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+		},
+		IDs{
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+			TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+		},
+	},
+	{
+		IDs{
+			TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+		},
+		IDs{
+			TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+		},
+	},
+	{
+		IDs{
+			TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+			TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+		},
+		IDs{
+			TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+			TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
+			TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+		},
+	},
+}
+
+func TestUniqIDs(t *testing.T) {
+	for i, test := range uniqTests {
+		uniq := test.before.Uniq()
+		if !reflect.DeepEqual(uniq, test.after) {
+			t.Errorf("uniqIDs() test %v failed\n  wanted: %v\n  got: %v", i, test.after, uniq)
+		}
+	}
+}

src/restic/idset.go

@@ -0,0 +1,111 @@
+package restic
+
+import "sort"
+
+// IDSet is a set of IDs.
+type IDSet map[ID]struct{}
+
+// NewIDSet returns a new IDSet, populated with ids.
+func NewIDSet(ids ...ID) IDSet {
+	m := make(IDSet)
+	for _, id := range ids {
+		m[id] = struct{}{}
+	}
+
+	return m
+}
+
+// Has returns true iff id is contained in the set.
+func (s IDSet) Has(id ID) bool {
+	_, ok := s[id]
+	return ok
+}
+
+// Insert adds id to the set.
+func (s IDSet) Insert(id ID) {
+	s[id] = struct{}{}
+}
+
+// Delete removes id from the set.
+func (s IDSet) Delete(id ID) {
+	delete(s, id)
+}
+
+// List returns a slice of all IDs in the set.
+func (s IDSet) List() IDs {
+	list := make(IDs, 0, len(s))
+	for id := range s {
+		list = append(list, id)
+	}
+
+	sort.Sort(list)
+
+	return list
+}
+
+// Equals returns true iff s equals other.
+func (s IDSet) Equals(other IDSet) bool {
+	if len(s) != len(other) {
+		return false
+	}
+
+	for id := range s {
+		if _, ok := other[id]; !ok {
+			return false
+		}
+	}
+
+	// length + one-way comparison is sufficient implication of equality
+
+	return true
+}
+
+// Merge adds the blobs in other to the current set.
+func (s IDSet) Merge(other IDSet) {
+	for id := range other {
+		s.Insert(id)
+	}
+}
+
+// Intersect returns a new set containing the IDs that are present in both sets.
+func (s IDSet) Intersect(other IDSet) (result IDSet) {
+	result = NewIDSet()
+
+	set1 := s
+	set2 := other
+
+	// iterate over the smaller set
+	if len(set2) < len(set1) {
+		set1, set2 = set2, set1
+	}
+
+	for id := range set1 {
+		if set2.Has(id) {
+			result.Insert(id)
+		}
+	}
+
+	return result
+}
+
+// Sub returns a new set containing all IDs that are present in s but not in
+// other.
+func (s IDSet) Sub(other IDSet) (result IDSet) {
+	result = NewIDSet()
+	for id := range s {
+		if !other.Has(id) {
+			result.Insert(id)
+		}
+	}
+
+	return result
+}
+
+func (s IDSet) String() string {
+	str := s.List().String()
+	if len(str) < 2 {
+		return "{}"
+	}
+
+	return "{" + str[1:len(str)-1] + "}"
+}

src/restic/idset_test.go

@@ -0,0 +1,32 @@
+package restic
+
+import (
+	"testing"
+)
+
+var idsetTests = []struct {
+	id   ID
+	seen bool
+}{
+	{TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), false},
+	{TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), false},
+	{TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
+	{TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
+	{TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true},
+	{TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), false},
+	{TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
+	{TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true},
+	{TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), true},
+	{TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
+}
+
+func TestIDSet(t *testing.T) {
+	set := NewIDSet()
+	for i, test := range idsetTests {
+		seen := set.Has(test.id)
+		if seen != test.seen {
+			t.Errorf("IDSet test %v failed: wanted %v, got %v", i, test.seen, seen)
+		}
+		set.Insert(test.id)
+	}
+}

src/restic/testing.go

@@ -210,3 +210,13 @@ func TestResetRepository(t testing.TB, repo Repository) {
 
 	repo.SetIndex(repository.NewMasterIndex())
 }
+
+// TestParseID parses s as a backend.ID and panics if that fails.
+func TestParseID(s string) ID {
+	id, err := ParseID(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return id
+}