Merge pull request #4486 from MichaelEischer/separate-large-blobs

repository: store oversized blobs in separate pack files

internal/repository/packer_manager.go

@@ -70,14 +70,19 @@ func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id rest
 
 	var err error
 	packer := r.packer
-	if r.packer == nil {
+	// use separate packer if compressed length is larger than the packsize
+	// this speeds up the garbage collection of oversized blobs and reduces the cache size
+	// as the oversize blobs are only downloaded if necessary
+	if len(ciphertext) >= int(r.packSize) || r.packer == nil {
 		packer, err = r.newPacker()
 		if err != nil {
 			return 0, err
 		}
-	}
-	// remember packer
+		// don't store packer for oversized blob
+		if r.packer == nil {
 			r.packer = packer
+		}
+	}
 
 	// save ciphertext
 	// Add only appends bytes in memory to avoid being a scaling bottleneck
@@ -91,8 +96,10 @@ func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id rest
 		debug.Log("pack is not full enough (%d bytes)", packer.Size())
 		return size, nil
 	}
+	if packer == r.packer {
 		// forget full packer
 		r.packer = nil
+	}
 
 	// call while holding lock to prevent findPacker from creating new packers if the uploaders are busy
 	// else write the pack to the backend

internal/repository/packer_manager_test.go

@@ -89,6 +89,24 @@ func testPackerManager(t testing.TB) int64 {
 	return int64(bytes)
 }
 
+func TestPackerManagerWithOversizeBlob(t *testing.T) {
+	packFiles := int(0)
+	sizeLimit := uint(512 * 1024)
+	pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, sizeLimit, func(ctx context.Context, tp restic.BlobType, p *Packer) error {
+		packFiles++
+		return nil
+	})
+
+	for _, i := range []uint{sizeLimit / 2, sizeLimit, sizeLimit / 3} {
+		_, err := pm.SaveBlob(context.TODO(), restic.DataBlob, restic.ID{}, make([]byte, i), 0)
+		test.OK(t, err)
+	}
+	test.OK(t, pm.Flush(context.TODO()))
+
+	// oversized blob must be stored in a separate packfile
+	test.Equals(t, packFiles, 2)
+}
+
 func BenchmarkPackerManager(t *testing.B) {
 	// Run testPackerManager if it hasn't run already, to set totalSize.
 	once.Do(func() {