diff --git a/src/restic/id.go b/src/restic/id.go new file mode 100644 index 000000000..2e9308888 --- /dev/null +++ b/src/restic/id.go @@ -0,0 +1,109 @@ +package restic + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + + "github.com/pkg/errors" +) + +// Hash returns the ID for data. +func Hash(data []byte) ID { + return sha256.Sum256(data) +} + +// IDSize contains the size of an ID, in bytes. +const IDSize = sha256.Size + +// ID references content within a repository. +type ID [IDSize]byte + +// ParseID converts the given string to an ID. +func ParseID(s string) (ID, error) { + b, err := hex.DecodeString(s) + + if err != nil { + return ID{}, errors.Wrap(err, "hex.DecodeString") + } + + if len(b) != IDSize { + return ID{}, errors.New("invalid length for hash") + } + + id := ID{} + copy(id[:], b) + + return id, nil +} + +func (id ID) String() string { + return hex.EncodeToString(id[:]) +} + +const shortStr = 4 + +// Str returns the shortened string version of id. +func (id *ID) Str() string { + if id == nil { + return "[nil]" + } + + if id.IsNull() { + return "[null]" + } + + return hex.EncodeToString(id[:shortStr]) +} + +// IsNull returns true iff id only consists of null bytes. +func (id ID) IsNull() bool { + var nullID ID + + return id == nullID +} + +// Equal compares an ID to another other. +func (id ID) Equal(other ID) bool { + return id == other +} + +// EqualString compares this ID to another one, given as a string. +func (id ID) EqualString(other string) (bool, error) { + s, err := hex.DecodeString(other) + if err != nil { + return false, errors.Wrap(err, "hex.DecodeString") + } + + id2 := ID{} + copy(id2[:], s) + + return id == id2, nil +} + +// Compare compares this ID to another one, returning -1, 0, or 1. +func (id ID) Compare(other ID) int { + return bytes.Compare(other[:], id[:]) +} + +// MarshalJSON returns the JSON encoding of id. +func (id ID) MarshalJSON() ([]byte, error) { + return json.Marshal(id.String()) +} + +// UnmarshalJSON parses the JSON-encoded data and stores the result in id. +func (id *ID) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return errors.Wrap(err, "Unmarshal") + } + + _, err = hex.Decode(id[:], []byte(s)) + if err != nil { + return errors.Wrap(err, "hex.Decode") + } + + return nil +} diff --git a/src/restic/id_int_test.go b/src/restic/id_int_test.go new file mode 100644 index 000000000..a60a11b89 --- /dev/null +++ b/src/restic/id_int_test.go @@ -0,0 +1,16 @@ +package restic + +import "testing" + +func TestIDMethods(t *testing.T) { + var id ID + + if id.Str() != "[null]" { + t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[null]", id.Str()) + } + + var pid *ID + if pid.Str() != "[nil]" { + t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[nil]", pid.Str()) + } +} diff --git a/src/restic/id_test.go b/src/restic/id_test.go new file mode 100644 index 000000000..2e9634a19 --- /dev/null +++ b/src/restic/id_test.go @@ -0,0 +1,60 @@ +package restic + +import ( + "reflect" + "testing" +) + +var TestStrings = []struct { + id string + data string +}{ + {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"}, + {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"}, + {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, +} + +func TestID(t *testing.T) { + for _, test := range TestStrings { + id, err := ParseID(test.id) + if err != nil { + t.Error(err) + } + + id2, err := ParseID(test.id) + if err != nil { + t.Error(err) + } + if !id.Equal(id2) { + t.Errorf("ID.Equal() does not work as expected") + } + + ret, err := id.EqualString(test.id) + if err != nil { + t.Error(err) + } + if !ret { + t.Error("ID.EqualString() returned wrong value") + } + + // test json marshalling + buf, err := id.MarshalJSON() + if err != nil { + t.Error(err) + } + want := `"` + test.id + `"` + if string(buf) != want { + t.Errorf("string comparison failed, wanted %q, got %q", want, string(buf)) + } + + var id3 ID + err = id3.UnmarshalJSON(buf) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(id, id3) { + t.Error("ids are not equal") + } + } +} diff --git a/src/restic/ids.go b/src/restic/ids.go new file mode 100644 index 000000000..cc5ad18da --- /dev/null +++ b/src/restic/ids.go @@ -0,0 +1,69 @@ +package restic + +import ( + "encoding/hex" + "fmt" +) + +// IDs is an ordered list of IDs that implements sort.Interface. +type IDs []ID + +func (ids IDs) Len() int { + return len(ids) +} + +func (ids IDs) Less(i, j int) bool { + if len(ids[i]) < len(ids[j]) { + return true + } + + for k, b := range ids[i] { + if b == ids[j][k] { + continue + } + + if b < ids[j][k] { + return true + } + + return false + } + + return false +} + +func (ids IDs) Swap(i, j int) { + ids[i], ids[j] = ids[j], ids[i] +} + +// Uniq returns list without duplicate IDs. The returned list retains the order +// of the original list so that the order of the first occurrence of each ID +// stays the same. +func (ids IDs) Uniq() (list IDs) { + seen := NewIDSet() + + for _, id := range ids { + if seen.Has(id) { + continue + } + + list = append(list, id) + seen.Insert(id) + } + + return list +} + +type shortID ID + +func (id shortID) String() string { + return hex.EncodeToString(id[:shortStr]) +} + +func (ids IDs) String() string { + elements := make([]shortID, 0, len(ids)) + for _, id := range ids { + elements = append(elements, shortID(id)) + } + return fmt.Sprintf("%v", elements) +} diff --git a/src/restic/ids_test.go b/src/restic/ids_test.go new file mode 100644 index 000000000..9ce02607b --- /dev/null +++ b/src/restic/ids_test.go @@ -0,0 +1,55 @@ +package restic + +import ( + "reflect" + "testing" +) + +var uniqTests = []struct { + before, after IDs +}{ + { + IDs{ + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + }, + IDs{ + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + }, + }, + { + IDs{ + TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + }, + IDs{ + TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + }, + }, + { + IDs{ + TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + }, + IDs{ + TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), + TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + }, + }, +} + +func TestUniqIDs(t *testing.T) { + for i, test := range uniqTests { + uniq := test.before.Uniq() + if !reflect.DeepEqual(uniq, test.after) { + t.Errorf("uniqIDs() test %v failed\n wanted: %v\n got: %v", i, test.after, uniq) + } + } +} diff --git a/src/restic/idset.go b/src/restic/idset.go new file mode 100644 index 000000000..c31ca7747 --- /dev/null +++ b/src/restic/idset.go @@ -0,0 +1,111 @@ +package restic + +import "sort" + +// IDSet is a set of IDs. +type IDSet map[ID]struct{} + +// NewIDSet returns a new IDSet, populated with ids. +func NewIDSet(ids ...ID) IDSet { + m := make(IDSet) + for _, id := range ids { + m[id] = struct{}{} + } + + return m +} + +// Has returns true iff id is contained in the set. +func (s IDSet) Has(id ID) bool { + _, ok := s[id] + return ok +} + +// Insert adds id to the set. +func (s IDSet) Insert(id ID) { + s[id] = struct{}{} +} + +// Delete removes id from the set. +func (s IDSet) Delete(id ID) { + delete(s, id) +} + +// List returns a slice of all IDs in the set. +func (s IDSet) List() IDs { + list := make(IDs, 0, len(s)) + for id := range s { + list = append(list, id) + } + + sort.Sort(list) + + return list +} + +// Equals returns true iff s equals other. +func (s IDSet) Equals(other IDSet) bool { + if len(s) != len(other) { + return false + } + + for id := range s { + if _, ok := other[id]; !ok { + return false + } + } + + // length + one-way comparison is sufficient implication of equality + + return true +} + +// Merge adds the blobs in other to the current set. +func (s IDSet) Merge(other IDSet) { + for id := range other { + s.Insert(id) + } +} + +// Intersect returns a new set containing the IDs that are present in both sets. +func (s IDSet) Intersect(other IDSet) (result IDSet) { + result = NewIDSet() + + set1 := s + set2 := other + + // iterate over the smaller set + if len(set2) < len(set1) { + set1, set2 = set2, set1 + } + + for id := range set1 { + if set2.Has(id) { + result.Insert(id) + } + } + + return result +} + +// Sub returns a new set containing all IDs that are present in s but not in +// other. +func (s IDSet) Sub(other IDSet) (result IDSet) { + result = NewIDSet() + for id := range s { + if !other.Has(id) { + result.Insert(id) + } + } + + return result +} + +func (s IDSet) String() string { + str := s.List().String() + if len(str) < 2 { + return "{}" + } + + return "{" + str[1:len(str)-1] + "}" +} diff --git a/src/restic/idset_test.go b/src/restic/idset_test.go new file mode 100644 index 000000000..5525eab79 --- /dev/null +++ b/src/restic/idset_test.go @@ -0,0 +1,32 @@ +package restic + +import ( + "testing" +) + +var idsetTests = []struct { + id ID + seen bool +}{ + {TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), false}, + {TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), false}, + {TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true}, + {TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), false}, + {TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true}, + {TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), true}, + {TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, +} + +func TestIDSet(t *testing.T) { + set := NewIDSet() + for i, test := range idsetTests { + seen := set.Has(test.id) + if seen != test.seen { + t.Errorf("IDSet test %v failed: wanted %v, got %v", i, test.seen, seen) + } + set.Insert(test.id) + } +} diff --git a/src/restic/testing.go b/src/restic/testing.go index 0b6ed6b49..cf2500b17 100644 --- a/src/restic/testing.go +++ b/src/restic/testing.go @@ -210,3 +210,13 @@ func TestResetRepository(t testing.TB, repo Repository) { repo.SetIndex(repository.NewMasterIndex()) } + +// TestParseID parses s as a backend.ID and panics if that fails. +func TestParseID(s string) ID { + id, err := ParseID(s) + if err != nil { + panic(err) + } + + return id +}