repack: Use Get() instead of Load()

In addition, use a tempfile instead of a buffer.

src/restic/repository/repack.go

@@ -1,11 +1,14 @@
 package repository
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"io"
+	"io/ioutil"
+	"os"
 	"restic"
 	"restic/crypto"
 	"restic/debug"
+	"restic/hashing"
 	"restic/pack"
 
 	"restic/errors"
@@ -18,30 +21,47 @@ import (
 func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
 
-	buf := make([]byte, 0, maxPackSize)
 	for packID := range packs {
-		// load the complete pack
+		// load the complete pack into a temp file
 		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
 
-		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
+		tempfile, err := ioutil.TempFile("", "restic-temp-repack-")
-		if errors.Cause(err) == io.ErrUnexpectedEOF {
+		if err != nil {
-			err = nil
+			return errors.Wrap(err, "TempFile")
-			buf = buf[:l]
 		}
 
+		beRd, err := repo.Backend().Get(h, 0, 0)
 		if err != nil {
 			return err
 		}
 
-		debug.Log("pack %v loaded (%d bytes)", packID.Str(), len(buf))
+		defer beRd.Close()
 
-		blobs, err := pack.List(repo.Key(), bytes.NewReader(buf), int64(len(buf)))
+		hrd := hashing.NewReader(beRd, sha256.New())
+		packLength, err := io.Copy(tempfile, hrd)
+		if err != nil {
+			return errors.Wrap(err, "Copy")
+		}
+
+		hash := restic.IDFromHash(hrd.Sum(nil))
+		debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
+
+		if !packID.Equal(hash) {
+			return errors.Errorf("hash does not match id: want %v, got %v", packID, hash)
+		}
+
+		_, err = tempfile.Seek(0, 0)
+		if err != nil {
+			return errors.Wrap(err, "Seek")
+		}
+
+		blobs, err := pack.List(repo.Key(), tempfile, packLength)
 		if err != nil {
 			return err
 		}
 
 		debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
-		var plaintext []byte
+		var buf []byte
 		for _, entry := range blobs {
 			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
 			if !keepBlobs.Has(h) {
@@ -50,21 +70,36 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			debug.Log("  process blob %v", h)
 
-			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]
+			buf = buf[:len(buf)]
-			plaintext = plaintext[:len(plaintext)]
+			if uint(len(buf)) < entry.Length {
-			if len(plaintext) < len(ciphertext) {
+				buf = make([]byte, entry.Length)
-				plaintext = make([]byte, len(ciphertext))
+			}
+			buf = buf[:entry.Length]
+
+			n, err := tempfile.ReadAt(buf, int64(entry.Offset))
+			if err != nil {
+				return errors.Wrap(err, "ReadAt")
 			}
 
-			debug.Log("  ciphertext %d, plaintext %d", len(plaintext), len(ciphertext))
+			if n != len(buf) {
+				return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v",
+					h, tempfile.Name(), len(buf), n)
+			}
 
-			n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext)
+			n, err = crypto.Decrypt(repo.Key(), buf, buf)
 			if err != nil {
 				return err
 			}
-			plaintext = plaintext[:n]
 
-			_, err = repo.SaveBlob(entry.Type, plaintext, entry.ID)
+			buf = buf[:n]
+
+			id := restic.Hash(buf)
+			if !id.Equal(entry.ID) {
+				return errors.Errorf("read blob %v from %v: wrong data returned, hash is %v",
+					h, tempfile.Name(), id)
+			}
+
+			_, err = repo.SaveBlob(entry.Type, buf, entry.ID)
 			if err != nil {
 				return err
 			}
@@ -73,6 +108,14 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			keepBlobs.Delete(h)
 		}
+
+		if err = tempfile.Close(); err != nil {
+			return errors.Wrap(err, "Close")
+		}
+
+		if err = os.Remove(tempfile.Name()); err != nil {
+			return errors.Wrap(err, "Remove")
+		}
 	}
 
 	if err := repo.Flush(); err != nil {