Add 'optimize' command that repacks blobs

cmd/restic/cmd_optimize.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/checker"
+)
+
+type CmdOptimize struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("optimize",
+		"optimize the repository",
+		"The optimize command reorganizes the repository and removes uneeded data",
+		&CmdOptimize{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdOptimize) Usage() string {
+	return "[optimize-options]"
+}
+
+func (cmd CmdOptimize) Execute(args []string) error {
+	if len(args) != 0 {
+		return errors.New("optimize has no arguments")
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("Create exclusive lock for repository\n")
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	chkr := checker.New(repo)
+
+	cmd.global.Verbosef("Load indexes\n")
+	_, errs := chkr.LoadIndex()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+		return fmt.Errorf("LoadIndex returned errors")
+	}
+
+	done := make(chan struct{})
+	errChan := make(chan error)
+	go chkr.Structure(errChan, done)
+
+	for err := range errChan {
+		if e, ok := err.(checker.TreeError); ok {
+			cmd.global.Warnf("error for tree %v:\n", e.ID.Str())
+			for _, treeErr := range e.Errors {
+				cmd.global.Warnf("  %v\n", treeErr)
+			}
+		} else {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+	}
+
+	unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...)
+	cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs))
+
+	repacker := checker.NewRepacker(repo, unusedBlobs)
+	err = repacker.Repack()
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("repacking done\n")
+	return nil
+}

cmd/restic/integration_test.go

@@ -61,7 +61,7 @@ func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, pare
 	OK(t, cmd.Execute(target))
 }
 
-func cmdList(t testing.TB, global GlobalOptions, tpe string) []backend.ID {
+func cmdList(t testing.TB, global GlobalOptions, tpe string) backend.IDs {
 	var buf bytes.Buffer
 	global.stdout = &buf
 	cmd := &CmdList{global: &global}
@@ -87,7 +87,11 @@ func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshot
 }
 
 func cmdCheck(t testing.TB, global GlobalOptions) {
-	cmd := &CmdCheck{global: &global, ReadData: true}
+	cmd := &CmdCheck{
+		global:      &global,
+		ReadData:    true,
+		CheckUnused: true,
+	}
 	OK(t, cmd.Execute(nil))
 }
 
@@ -105,6 +109,11 @@ func cmdRebuildIndex(t testing.TB, global GlobalOptions) {
 	OK(t, cmd.Execute(nil))
 }
 
+func cmdOptimize(t testing.TB, global GlobalOptions) {
+	cmd := &CmdOptimize{global: &global}
+	OK(t, cmd.Execute(nil))
+}
+
 func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string {
 	var buf bytes.Buffer
 	global.stdout = &buf
@@ -689,3 +698,17 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
 	repository.IndexFull = func(*repository.Index) bool { return true }
 	TestRebuildIndex(t)
 }
+
+func TestOptimizeRemoveUnusedBlobs(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("..", "..", "checker", "testdata", "checker-test-repo.tar.gz")
+		SetupTarTestFixture(t, env.base, datafile)
+
+		// snapshotIDs := cmdList(t, global, "snapshots")
+		// t.Logf("snapshots: %v", snapshotIDs)
+
+		OK(t, os.Remove(filepath.Join(env.repo, "snapshots", "a13c11e582b77a693dd75ab4e3a3ba96538a056594a4b9076e4cacebe6e06d43")))
+		cmdOptimize(t, global)
+		cmdCheck(t, global)
+	})
+}