TrueCloudLab/restic
16
0
Fork 2
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Merge pull request #4641 from MichaelEischer/reduce-restic-repository-usage

Browse source 
Misc cleanups
This commit is contained in:
Michael Eischer 2024-01-27 13:18:20 +01:00 committed by GitHub
commit c90f24a06c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 42 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/restic/cmd_backup.go
View file

@ -411,7 +411,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
// parent returns the ID of the parent snapshot. If there is none, nil is // parent returns the ID of the parent snapshot. If there is none, nil is
// returned. // returned.
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
if opts.Force { if opts.Force {
return nil, nil return nil, nil
} }

2
cmd/restic/cmd_debug.go
View file

@ -137,7 +137,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}) })
} }
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id) Printf("index_id: %v\n", id)
if err != nil { if err != nil {

6
cmd/restic/cmd_diff.go
View file

@ -61,7 +61,7 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
} }
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil { if err != nil {
return nil, "", errors.Fatal(err.Error()) return nil, "", errors.Fatal(err.Error())
@ -71,7 +71,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository,
// Comparer collects all things needed to compare two snapshots. // Comparer collects all things needed to compare two snapshots.
type Comparer struct { type Comparer struct {
repo restic.Repository repo restic.BlobLoader
opts DiffOptions opts DiffOptions
printChange func(change *Change) printChange func(change *Change)
} }
@ -147,7 +147,7 @@ type DiffStatsContainer struct {
} }
// updateBlobs updates the blob counters in the stats struct. // updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) { func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
for h := range blobs { for h := range blobs {
switch h.Type { switch h.Type {
case restic.DataBlob: case restic.DataBlob:

2
cmd/restic/cmd_dump.go
View file

@ -67,7 +67,7 @@ func splitPath(p string) []string {
return append(s, f) return append(s, f)
} }
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error { func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper) error {
// If we print / we need to assume that there are multiple nodes at that // If we print / we need to assume that there are multiple nodes at that
// level in the tree. // level in the tree.
if pathComponents[0] == "" { if pathComponents[0] == "" {

2
cmd/restic/cmd_recover.go
View file

@ -158,7 +158,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
} }
func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error { func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error {
sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now()) sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
if err != nil { if err != nil {
return errors.Fatalf("unable to save snapshot: %v", err) return errors.Fatalf("unable to save snapshot: %v", err)

6
cmd/restic/cmd_stats.go
View file

@ -189,7 +189,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
return nil return nil
} }
func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, opts StatsOptions, stats *statsContainer) error { func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error {
if snapshot.Tree == nil { if snapshot.Tree == nil {
return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
} }
@ -211,7 +211,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
return nil return nil
} }
func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc {
return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error { return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error {
if nodeErr != nil { if nodeErr != nil {
return nodeErr return nodeErr
@ -363,7 +363,7 @@ func statsDebug(ctx context.Context, repo restic.Repository) error {
return nil return nil
} }
func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic.FileType) (*sizeHistogram, error) { func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.FileType) (*sizeHistogram, error) {
hist := newSizeHistogram(2 * repository.MaxPackSize) hist := newSizeHistogram(2 * repository.MaxPackSize)
err := repo.List(ctx, tpe, func(id restic.ID, size int64) error { err := repo.List(ctx, tpe, func(id restic.ID, size int64) error {
hist.Add(uint64(size)) hist.Add(uint64(size))

4
internal/archiver/testing.go
View file

@ -209,7 +209,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
} }
// TestEnsureFileContent checks if the file in the repo is the same as file. // TestEnsureFileContent checks if the file in the repo is the same as file.
func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Repository, filename string, node *restic.Node, file TestFile) { func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLoader, filename string, node *restic.Node, file TestFile) {
if int(node.Size) != len(file.Content) { if int(node.Size) != len(file.Content) {
t.Fatalf("%v: wrong node size: want %d, got %d", filename, node.Size, len(file.Content)) t.Fatalf("%v: wrong node size: want %d, got %d", filename, node.Size, len(file.Content))
return return
@ -237,7 +237,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi
// TestEnsureTree checks that the tree ID in the repo matches dir. On Windows, // TestEnsureTree checks that the tree ID in the repo matches dir. On Windows,
// Symlinks are ignored. // Symlinks are ignored.
func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.Repository, treeID restic.ID, dir TestDir) { func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.BlobLoader, treeID restic.ID, dir TestDir) {
t.Helper() t.Helper()
tree, err := restic.LoadTree(ctx, repo, treeID) tree, err := restic.LoadTree(ctx, repo, treeID)

16
internal/checker/checker.go
View file

@ -14,7 +14,6 @@ import (
"github.com/minio/sha256-simd" "github.com/minio/sha256-simd"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/hashing"
@ -241,17 +240,8 @@ func IsOrphanedPack(err error) bool {
} }
func isS3Legacy(b backend.Backend) bool { func isS3Legacy(b backend.Backend) bool {
// unwrap cache be := backend.AsBackend[*s3.Backend](b)
if be, ok := b.(*cache.Backend); ok { return be != nil && be.Layout.Name() == "s3legacy"
b = be.Backend
}
be, ok := b.(*s3.Backend)
if !ok {
return false
}
return be.Layout.Name() == "s3legacy"
} }
// Packs checks that all packs referenced in the index are still available and // Packs checks that all packs referenced in the index are still available and
@ -362,7 +352,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI
} }
} }
func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) {
err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
errs = append(errs, err) errs = append(errs, err)

8
internal/dump/common.go
View file

@ -16,11 +16,11 @@ import (
type Dumper struct { type Dumper struct {
cache *bloblru.Cache cache *bloblru.Cache
format string format string
repo restic.Repository repo restic.BlobLoader
w io.Writer w io.Writer
} }
func New(format string, repo restic.Repository, w io.Writer) *Dumper { func New(format string, repo restic.BlobLoader, w io.Writer) *Dumper {
return &Dumper{ return &Dumper{
cache: bloblru.New(64 << 20), cache: bloblru.New(64 << 20),
format: format, format: format,
@ -47,7 +47,7 @@ func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath strin
} }
} }
func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, ch chan *restic.Node) { func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *restic.Tree, rootPath string, ch chan *restic.Node) {
defer close(ch) defer close(ch)
for _, root := range tree.Nodes { for _, root := range tree.Nodes {
@ -58,7 +58,7 @@ func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, r
} }
} }
func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, ch chan *restic.Node) error { func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, ch chan *restic.Node) error {
select { select {
case ch <- root: case ch <- root:
case <-ctx.Done(): case <-ctx.Done():

2
internal/fuse/dir.go
View file

@ -58,7 +58,7 @@ func unwrapCtxCanceled(err error) error {
// replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // replaceSpecialNodes replaces nodes with name "." and "/" by their contents.
// Otherwise, the node is returned. // Otherwise, the node is returned.
func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *restic.Node) ([]*restic.Node, error) { func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *restic.Node) ([]*restic.Node, error) {
if node.Type != "dir" || node.Subtree == nil { if node.Type != "dir" || node.Subtree == nil {
return []*restic.Node{node}, nil return []*restic.Node{node}, nil
} }

6
internal/fuse/fuse_test.go
View file

@ -37,7 +37,7 @@ func testRead(t testing.TB, f fs.Handle, offset, length int, data []byte) {
rtest.OK(t, fr.Read(ctx, req, resp)) rtest.OK(t, fr.Read(ctx, req, resp))
} }
func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { func firstSnapshotID(t testing.TB, repo restic.Lister) (first restic.ID) {
err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error { err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error {
if first.IsNull() { if first.IsNull() {
first = id first = id
@ -52,14 +52,14 @@ func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
return first return first
} }
func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *restic.Snapshot {
id := firstSnapshotID(t, repo) id := firstSnapshotID(t, repo)
sn, err := restic.LoadSnapshot(context.TODO(), repo, id) sn, err := restic.LoadSnapshot(context.TODO(), repo, id)
rtest.OK(t, err) rtest.OK(t, err)
return sn return sn
} }
func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *restic.Tree {
tree, err := restic.LoadTree(context.TODO(), repo, id) tree, err := restic.LoadTree(context.TODO(), repo, id)
rtest.OK(t, err) rtest.OK(t, err)
return tree return tree

2
internal/index/index_parallel.go
View file

@ -11,7 +11,7 @@ import (
// ForAllIndexes loads all index files in parallel and calls the given callback. // ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback // It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error. // returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.ListerLoaderUnpacked,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound // decoding an index can take quite some time such that this can be both CPU- or IO-bound

2
internal/repository/repack_test.go
View file

@ -120,7 +120,7 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2
return list1, list2 return list1, list2
} }
func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { func listPacks(t *testing.T, repo restic.Lister) restic.IDSet {
list := restic.NewIDSet() list := restic.NewIDSet()
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
list.Insert(id) list.Insert(id)

2
internal/repository/repository_test.go
View file

@ -229,7 +229,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
} }
// loadIndex loads the index id from backend and returns it. // loadIndex loads the index id from backend and returns it.
func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) { func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*index.Index, error) {
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id) buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
if err != nil { if err != nil {
return nil, err return nil, err

8
internal/restic/lock.go
View file

@ -341,8 +341,8 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
exists := false exists := false
err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error { err := l.repo.List(ctx, LockFile, func(id ID, size int64) error {
if fi.Name == l.lockID.String() { if id.Equal(*l.lockID) {
exists = true exists = true
} }
return nil return nil
@ -379,7 +379,7 @@ func init() {
} }
// LoadLock loads and unserializes a lock from a repository. // LoadLock loads and unserializes a lock from a repository.
func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) { func LoadLock(ctx context.Context, repo LoaderUnpacked, id ID) (*Lock, error) {
lock := &Lock{} lock := &Lock{}
if err := LoadJSONUnpacked(ctx, repo, LockFile, id, lock); err != nil { if err := LoadJSONUnpacked(ctx, repo, LockFile, id, lock); err != nil {
return nil, err return nil, err
@ -429,7 +429,7 @@ func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
// It is guaranteed that the function is not run concurrently. If the // It is guaranteed that the function is not run concurrently. If the
// callback returns an error, this function is cancelled and also returns that error. // callback returns an error, this function is cancelled and also returns that error.
// If a lock ID is passed via excludeID, it will be ignored. // If a lock ID is passed via excludeID, it will be ignored.
func ForAllLocks(ctx context.Context, repo Repository, excludeIDs IDSet, fn func(ID, *Lock, error) error) error { func ForAllLocks(ctx context.Context, repo ListerLoaderUnpacked, excludeIDs IDSet, fn func(ID, *Lock, error) error) error {
var m sync.Mutex var m sync.Mutex
// For locks decoding is nearly for free, thus just assume were only limited by IO // For locks decoding is nearly for free, thus just assume were only limited by IO

4
internal/restic/lock_test.go
View file

@ -120,7 +120,7 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) {
rtest.OK(t, elock.Unlock()) rtest.OK(t, elock.Unlock())
} }
func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, error) { func createFakeLock(repo restic.SaverUnpacked, t time.Time, pid int) (restic.ID, error) {
hostname, err := os.Hostname() hostname, err := os.Hostname()
if err != nil { if err != nil {
return restic.ID{}, err return restic.ID{}, err
@ -254,7 +254,7 @@ func TestRemoveAllLocks(t *testing.T) {
3, processed) 3, processed)
} }
func checkSingleLock(t *testing.T, repo restic.Repository) restic.ID { func checkSingleLock(t *testing.T, repo restic.Lister) restic.ID {
t.Helper() t.Helper()
var lockID *restic.ID var lockID *restic.ID
err := repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { err := repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error {

6
internal/restic/node.go
View file

@ -142,7 +142,7 @@ func (node Node) GetExtendedAttribute(a string) []byte {
} }
// CreateAt creates the node at the given path but does NOT restore node meta data. // CreateAt creates the node at the given path but does NOT restore node meta data.
func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) error { func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error {
debug.Log("create node %v at %v", node.Name, path) debug.Log("create node %v at %v", node.Name, path)
switch node.Type { switch node.Type {
@ -264,7 +264,7 @@ func (node Node) createDirAt(path string) error {
return nil return nil
} }
func (node Node) createFileAt(ctx context.Context, path string, repo Repository) error { func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error {
f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
@ -284,7 +284,7 @@ func (node Node) createFileAt(ctx context.Context, path string, repo Repository)
return nil return nil
} }
func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.File) error { func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error {
var buf []byte var buf []byte
for _, id := range node.Content { for _, id := range node.Content {
buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) buf, err := repo.LoadBlob(ctx, DataBlob, id, buf)

5
internal/restic/repository.go
View file

@ -113,3 +113,8 @@ type MasterIndex interface {
type Lister interface { type Lister interface {
List(ctx context.Context, t FileType, fn func(ID, int64) error) error List(ctx context.Context, t FileType, fn func(ID, int64) error) error
} }
type ListerLoaderUnpacked interface {
Lister
LoaderUnpacked
}

4
internal/restorer/restorer_test.go
View file

@ -40,7 +40,7 @@ type Dir struct {
ModTime time.Time ModTime time.Time
} }
func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@ -52,7 +52,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID {
return id return id
} }
func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode uint64) restic.ID { func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64) restic.ID {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()