Merge pull request #1265 from restic/improve-packers

Improve packers, prepare for cache

internal/repository/packer_manager.go

@@ -38,8 +38,6 @@ type packerManager struct {
 }
 
 const minPackSize = 4 * 1024 * 1024
-const maxPackSize = 16 * 1024 * 1024
-const maxPackers = 200
 
 // newPackerManager returns an new packer manager which writes temporary files
 // to a temporary directory
@@ -52,25 +50,19 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
+func (r *packerManager) findPacker() (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
 	// search for a suitable packer
 	if len(r.packers) > 0 {
-		debug.Log("searching packer for %d bytes\n", size)
+		p := r.packers[0]
-		for i, p := range r.packers {
+		r.packers = r.packers[1:]
-			if p.Packer.Size()+size < maxPackSize {
+		return p, nil
-				debug.Log("found packer %v", p)
-				// remove from list
-				r.packers = append(r.packers[:i], r.packers[i+1:]...)
-				return p, nil
-			}
-		}
 	}
 
 	// no suitable packer found, return new
-	debug.Log("create new pack for %d bytes", size)
+	debug.Log("create new pack")
 	tmpfile, err := fs.TempFile("", "restic-temp-pack-")
 	if err != nil {
 		return nil, errors.Wrap(err, "fs.TempFile")

internal/repository/packer_manager_test.go

@@ -72,7 +72,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 		l := rnd.rand.Intn(1 << 20)
 		seed := rnd.rand.Int63()
 
-		packer, err := pm.findPacker(uint(l))
+		packer, err := pm.findPacker()
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -91,7 +91,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 		}
 		bytes += l
 
-		if packer.Size() < minPackSize && pm.countPacker() < maxPackers {
+		if packer.Size() < minPackSize {
 			pm.insertPacker(packer)
 			continue
 		}

internal/repository/repository.go

@@ -24,15 +24,17 @@ type Repository struct {
 	keyName string
 	idx     *MasterIndex
 
-	*packerManager
+	treePM *packerManager
+	dataPM *packerManager
 }
 
 // New returns a new repository with backend be.
 func New(be restic.Backend) *Repository {
 	repo := &Repository{
-		be:            be,
+		be:     be,
-		idx:           NewMasterIndex(),
+		idx:    NewMasterIndex(),
-		packerManager: newPackerManager(be, nil),
+		dataPM: newPackerManager(be, nil),
+		treePM: newPackerManager(be, nil),
 	}
 
 	return repo
@@ -180,7 +182,18 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	}
 
 	// find suitable packer and add blob
-	packer, err := r.findPacker(uint(len(ciphertext)))
+	var pm *packerManager
+
+	switch t {
+	case restic.TreeBlob:
+		pm = r.treePM
+	case restic.DataBlob:
+		pm = r.dataPM
+	default:
+		panic(fmt.Sprintf("invalid type: %v", t))
+	}
+
+	packer, err := pm.findPacker()
 	if err != nil {
 		return restic.ID{}, err
 	}
@@ -191,11 +204,10 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 		return restic.ID{}, err
 	}
 
-	// if the pack is not full enough and there are less than maxPackers
+	// if the pack is not full enough, put back to the list
-	// packers, put back to the list
+	if packer.Size() < minPackSize {
-	if packer.Size() < minPackSize && r.countPacker() < maxPackers {
 		debug.Log("pack is not full enough (%d bytes)", packer.Size())
-		r.insertPacker(packer)
+		pm.insertPacker(packer)
 		return *id, nil
 	}
 
@@ -239,18 +251,22 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 
 // Flush saves all remaining packs.
 func (r *Repository) Flush() error {
-	r.pm.Lock()
+	for _, pm := range []*packerManager{r.dataPM, r.treePM} {
-	defer r.pm.Unlock()
+		pm.pm.Lock()
 
-	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
+		debug.Log("manually flushing %d packs", len(pm.packers))
-
+		for _, p := range pm.packers {
-	for _, p := range r.packerManager.packers {
+			err := r.savePacker(p)
-		err := r.savePacker(p)
+			if err != nil {
-		if err != nil {
+				pm.pm.Unlock()
-			return err
+				return err
+			}
 		}
+		pm.packers = pm.packers[:0]
+
+		pm.pm.Unlock()
 	}
-	r.packerManager.packers = r.packerManager.packers[:0]
+
 	return nil
 }
 
@@ -372,7 +388,8 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg, err = restic.LoadConfig(ctx, r)
 	return err
@@ -406,7 +423,8 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg = cfg
 	_, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg)