KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #392 from restic/fix-build

Browse source 
Allow saving duplicate blobs in the repacker
This commit is contained in:
Alexander Neumann 2016-01-17 22:02:09 +01:00
commit 4d7e802c44
6 changed files with 49 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
checker/repacker.go
View file

@ -136,7 +136,7 @@ func repackBlob(src, dst *repository.Repository, id backend.ID) error {
return errors.New("LoadBlob returned wrong data, len() doesn't match")
}
_, err = dst.SaveAndEncrypt(blob.Type, buf, &id)
_, err = dst.SaveAndEncrypt(blob.Type, buf, &id, true)
if err != nil {
return err
}

6
crypto/crypto_test.go
View file

@ -133,7 +133,6 @@ func TestLargeEncrypt(t *testing.T) {
func BenchmarkEncryptWriter(b *testing.B) {
size := 8 << 20 // 8MiB
rd := RandomReader(23, size)
k := crypto.NewRandomKey()
@ -141,7 +140,7 @@ func BenchmarkEncryptWriter(b *testing.B) {
b.SetBytes(int64(size))
for i := 0; i < b.N; i++ {
rd.Seek(0, 0)
rd := RandomReader(23, size)
wr := crypto.EncryptTo(k, ioutil.Discard)
n, err := io.Copy(wr, rd)
OK(b, err)
@ -195,14 +194,13 @@ func BenchmarkEncryptDecryptReader(b *testing.B) {
k := crypto.NewRandomKey()
size := 8 << 20 // 8MiB
rd := RandomReader(23, size)
b.ResetTimer()
b.SetBytes(int64(size))
buf := bytes.NewBuffer(nil)
for i := 0; i < b.N; i++ {
rd.Seek(0, 0)
rd := RandomReader(23, size)
buf.Reset()
wr := crypto.EncryptTo(k, buf)
_, err := io.Copy(wr, rd)

70
repository/master_index.go
View file

@ -155,49 +155,55 @@ func (mi *MasterIndex) Current() *Index {
}
// AddInFlight add the given ID to the list of in-flight IDs. An error is
// returned when the ID is already in the list.
func (mi *MasterIndex) AddInFlight(id backend.ID) error {
// The index + inFlight store must be searched for a matching id in one
// atomic operation. This requires locking the inFlight store and the
// index together!
mi.inFlight.Lock()
defer mi.inFlight.Unlock()
// returned when the ID is already in the list. Setting ignoreDuplicates to
// true only checks the in flight list, otherwise the index itself is also
// tested.
func (mi *MasterIndex) AddInFlight(id backend.ID, ignoreDuplicates bool) error {
// The index + inFlight store must be searched for a matching id in one
// atomic operation. This requires locking the inFlight store and the
// index together!
mi.inFlight.Lock()
defer mi.inFlight.Unlock()
// Note: mi.Has read locks the index again.
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
if !ignoreDuplicates {
// Note: mi.Has read locks the index again.
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
}
debug.Log("MasterIndex.AddInFlight", "adding %v", id)
if mi.inFlight.Has(id) {
return fmt.Errorf("%v is already in flight", id)
}
if mi.Has(id) {
return fmt.Errorf("%v is already indexed (fully processed)", id)
}
debug.Log("MasterIndex.AddInFlight", "adding %v", id.Str())
if mi.inFlight.Has(id) {
return fmt.Errorf("%v is already in flight", id.Str())
}
mi.inFlight.Insert(id)
return nil
if !ignoreDuplicates {
if mi.Has(id) {
return fmt.Errorf("%v is already indexed (fully processed)", id)
}
}
mi.inFlight.Insert(id)
return nil
}
// IsInFlight returns true iff the id is contained in the list of in-flight IDs.
func (mi *MasterIndex) IsInFlight(id backend.ID) bool {
// The index + inFlight store must be searched for a matching id in one
// atomic operation. This requires locking the inFlight store and the
// index together!
mi.inFlight.RLock()
defer mi.inFlight.RUnlock()
// The index + inFlight store must be searched for a matching id in one
// atomic operation. This requires locking the inFlight store and the
// index together!
mi.inFlight.RLock()
defer mi.inFlight.RUnlock()
// Note: mi.Has read locks the index again.
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
inFlight := mi.inFlight.Has(id)
debug.Log("MasterIndex.IsInFlight", "testing whether %v is in flight: %v", id.Str(), inFlight)
inFlight := mi.inFlight.Has(id)
debug.Log("MasterIndex.IsInFlight", "testing whether %v is in flight: %v", id.Str(), inFlight)
indexed := mi.Has(id)
debug.Log("MasterIndex.IsInFlight", "testing whether %v is indexed (fully processed): %v", id.Str(), indexed)
indexed := mi.Has(id)
debug.Log("MasterIndex.IsInFlight", "testing whether %v is indexed (fully processed): %v", id.Str(), indexed)
return inFlight
return inFlight
}
// RemoveFromInFlight deletes the given ID from the liste of in-flight IDs.

11
repository/repository.go
View file

@ -219,8 +219,9 @@ func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) {
}
// SaveAndEncrypt encrypts data and stores it to the backend as type t. If data is small
// enough, it will be packed together with other small blobs.
func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID) (backend.ID, error) {
// enough, it will be packed together with other small blobs. When
// ignoreDuplicates is true, blobs already in the index will be saved again.
func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID, ignoreDuplicates bool) (backend.ID, error) {
if id == nil {
// compute plaintext hash
hashedID := backend.Hash(data)
@ -241,7 +242,7 @@ func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID
// add this id to the list of in-flight chunk ids.
debug.Log("Repo.Save", "add %v to list of in-flight IDs", id.Str())
err = r.idx.AddInFlight(*id)
err = r.idx.AddInFlight(*id, ignoreDuplicates)
if err != nil {
debug.Log("Repo.Save", "another goroutine is already working on %v (%v) does already exist", id.Str, t)
return *id, nil
@ -284,7 +285,7 @@ func (r *Repository) SaveFrom(t pack.BlobType, id *backend.ID, length uint, rd i
return err
}
_, err = r.SaveAndEncrypt(t, buf, id)
_, err = r.SaveAndEncrypt(t, buf, id, false)
if err != nil {
return err
}
@ -308,7 +309,7 @@ func (r *Repository) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, er
}
buf = wr.Bytes()
return r.SaveAndEncrypt(t, buf, nil)
return r.SaveAndEncrypt(t, buf, nil, false)
}
// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the

4
repository/repository_test.go
View file

@ -83,7 +83,7 @@ func TestSave(t *testing.T) {
id := backend.Hash(data)
// save
sid, err := repo.SaveAndEncrypt(pack.Data, data, nil)
sid, err := repo.SaveAndEncrypt(pack.Data, data, nil, false)
OK(t, err)
Equals(t, id, sid)
@ -253,7 +253,7 @@ func saveRandomDataBlobs(t testing.TB, repo *repository.Repository, num int, siz
_, err := io.ReadFull(rand.Reader, buf)
OK(t, err)
_, err = repo.SaveAndEncrypt(pack.Data, buf, nil)
_, err = repo.SaveAndEncrypt(pack.Data, buf, nil, false)
OK(t, err)
}
}

1
test/helpers.go
View file

@ -90,7 +90,6 @@ type rndReader struct {
}
func (r *rndReader) Read(p []byte) (int, error) {
fmt.Printf("Read(%v)\n", len(p))
for i := range p {
p[i] = byte(r.src.Uint32())
}