KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

convert MemorizeList to be repository based

Browse source 
Ideally, code that uses a repository shouldn't directly interact with
the underlying backend. Thus, move MemorizeList one layer up.
This commit is contained in:
Michael Eischer 2023-10-01 13:05:56 +02:00
parent 1b8a67fe76
commit c7b770eb1f
42 changed files with 209 additions and 223 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/restic/cmd_backup.go
View file

@ -453,7 +453,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
f.Tags = []restic.TagList{opts.Tags.Flatten()}
}
sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
sn, _, err := f.FindLatest(ctx, repo, repo, snName)
// Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil

4
cmd/restic/cmd_cat.go
View file

@ -106,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "snapshot":
sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
@ -193,7 +193,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return errors.Fatal("blob not found")
case "tree":
sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}

5
cmd/restic/cmd_copy.go
View file

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@ -88,12 +87,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
return err
}
srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
if err != nil {
return err
}
dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile)
if err != nil {
return err
}

8
cmd/restic/cmd_debug.go
View file

@ -78,7 +78,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
}
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
if err != nil {
return err
}
@ -107,7 +107,7 @@ type Blob struct {
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
Warnf("error for pack %v: %v\n", id.Str(), err)
@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id)
if err != nil {
return err
@ -447,7 +447,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
for _, name := range args {
id, err := restic.ParseID(name)
if err != nil {
id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
id, err = restic.Find(ctx, repo, restic.PackFile, name)
if err != nil {
Warnf("error: %v\n", err)
continue

5
cmd/restic/cmd_diff.go
View file

@ -7,7 +7,6 @@ import (
"reflect"
"sort"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -58,7 +57,7 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
}
func loadSnapshot(ctx context.Context, be backend.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil {
return nil, "", errors.Fatal(err.Error())
@ -346,7 +345,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
}
// cache snapshots listing
be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_dump.go
View file

@ -147,7 +147,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}

3
cmd/restic/cmd_find.go
View file

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
@ -584,7 +583,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
}
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_forget.go
View file

@ -183,7 +183,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn)
}

4
cmd/restic/cmd_key.go
View file

@ -60,7 +60,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
var m sync.Mutex
var keys []keyInfo
err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
k, err := repository.LoadKey(ctx, s, id)
if err != nil {
Warnf("LoadKey() failed: %v\n", err)
@ -238,7 +238,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
return err
}
id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1])
id, err := restic.Find(ctx, repo, restic.KeyFile, args[1])
if err != nil {
return err
}

2
cmd/restic/cmd_list.go
View file

@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
case "locks":
t = restic.LockFile
case "blobs":
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}

3
cmd/restic/cmd_ls.go
View file

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@ -170,7 +169,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_prune.go
View file

@ -810,7 +810,7 @@ func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Rep
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n")
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err)

3
cmd/restic/cmd_recover.go
View file

@ -5,7 +5,6 @@ import (
"os"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
@ -52,7 +51,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
return err
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_repair_index.go
View file

@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
} else {
Verbosef("loading indexes...\n")
mi := index.NewMasterIndex()
err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id)

3
cmd/restic/cmd_repair_snapshots.go
View file

@ -3,7 +3,6 @@ package main
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
@ -84,7 +83,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
repo.SetDryRun()
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_restore.go
View file

@ -168,7 +168,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}

2
cmd/restic/cmd_rewrite.go
View file

@ -207,7 +207,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
repo.SetDryRun()
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_snapshots.go
View file

@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
}
var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn)
}
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)

3
cmd/restic/cmd_stats.go
View file

@ -8,7 +8,6 @@ import (
"strings"
"github.com/restic/chunker"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -94,7 +93,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
}
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_tag.go
View file

@ -120,7 +120,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
}
changeCnt := 0
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil {
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)

5
cmd/restic/find.go
View file

@ -3,7 +3,6 @@ package main
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
"github.com/spf13/pflag"
)
@ -29,11 +28,11 @@ func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter)
}
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be backend.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot)
go func() {
defer close(out)
be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile)
if err != nil {
Warnf("could not load snapshots: %v\n", err)
return

2
cmd/restic/integration_test.go
View file

@ -159,7 +159,7 @@ func TestFindListOnce(t *testing.T) {
snapshotIDs := restic.NewIDSet()
// specify the two oldest snapshots explicitly and use "latest" to reference the newest one
for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{
for sn := range FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{
secondSnapshot[0].String(),
secondSnapshot[1].String()[:8],
"latest",

7
internal/backend/backend.go
View file

@ -27,7 +27,7 @@ type Backend interface {
// HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool
// Remove removes a File described by h.
// Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error
// Close the backend
@ -110,8 +110,3 @@ type FileInfo struct {
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string)
}
// Lister allows listing files in a backend.
type Lister interface {
List(context.Context, FileType, func(FileInfo) error) error
}

41
internal/backend/utils.go
View file

@ -74,44 +74,3 @@ type LimitedReadCloser struct {
func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}}
}
type memorizedLister struct {
fileInfos []FileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
for _, fi := range m.fileInfos {
if ctx.Err() != nil {
break
}
err := fn(fi)
if err != nil {
return err
}
}
return ctx.Err()
}
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []FileInfo
err := be.List(ctx, t, func(fi FileInfo) error {
fileInfos = append(fileInfos, fi)
return nil
})
if err != nil {
return nil, err
}
return &memorizedLister{
fileInfos: fileInfos,
tpe: t,
}, nil
}

45
internal/backend/utils_test.go
View file

@ -3,7 +3,6 @@ package backend_test
import (
"bytes"
"context"
"fmt"
"io"
"math/rand"
"testing"
@ -148,47 +147,3 @@ func TestLoadAllAppend(t *testing.T) {
})
}
}
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
files := []backend.FileInfo{
{Size: 42, Name: restic.NewRandomID().String()},
{Size: 45, Name: restic.NewRandomID().String()},
}
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
for _, fi := range files {
if err := fn(fi); err != nil {
return err
}
}
return nil
}
mem, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), backend.IndexFile, func(fi backend.FileInfo) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []backend.FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(fi backend.FileInfo) error {
memFiles = append(memFiles, fi)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, files, memFiles)
}
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return fmt.Errorf("list error")
}
_, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

16
internal/checker/checker.go
View file

@ -39,7 +39,7 @@ type Checker struct {
trackUnused bool
masterIndex *index.MasterIndex
snapshots backend.Lister
snapshots restic.Lister
repo restic.Repository
}
@ -102,7 +102,7 @@ func (e *ErrPackData) Error() string {
func (c *Checker) LoadSnapshots(ctx context.Context) error {
var err error
c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile)
c.snapshots, err = restic.MemorizeList(ctx, c.repo, restic.SnapshotFile)
return err
}
@ -126,7 +126,7 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID
func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) {
debug.Log("Start")
indexList, err := backend.MemorizeList(ctx, c.repo.Backend(), restic.IndexFile)
indexList, err := restic.MemorizeList(ctx, c.repo, restic.IndexFile)
if err != nil {
// abort if an error occurs while listing the indexes
return hints, append(errs, err)
@ -134,13 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
if p != nil {
var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
numIndexFiles++
return nil
})
@ -367,7 +361,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI
}
}
func loadSnapshotTreeIDs(ctx context.Context, lister backend.Lister, repo restic.Repository) (ids restic.IDs, errs []error) {
func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) {
err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
errs = append(errs, err)

2
internal/fuse/snapshots_dirstruct.go
View file

@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
}
var snapshots restic.Snapshots
err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error {
err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error {
if sn != nil {
snapshots = append(snapshots, sn)
}

3
internal/index/index_parallel.go
View file

@ -5,14 +5,13 @@ import (
"runtime"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
)
// ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, lister backend.Lister, repo restic.Repository,
func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound

4
internal/index/index_parallel_test.go
View file

@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// check that all expected indexes are loaded without errors
indexIDs := restic.NewIDSet()
var indexErr error
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
if err != nil {
indexErr = err
}
@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// must failed with the returned error
iterErr := errors.New("error to pass upwards")
err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
err := index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
return iterErr
})

2
internal/repository/key.go
View file

@ -116,7 +116,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
checked := 0
if len(keyHint) > 0 {
id, err := restic.Find(ctx, s.Backend(), restic.KeyFile, keyHint)
id, err := restic.Find(ctx, s, restic.KeyFile, keyHint)
if err == nil {
key, err := OpenKey(ctx, s, id, password)

10
internal/repository/repository.go
View file

@ -584,20 +584,14 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
debug.Log("Loading index")
indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile)
indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
if err != nil {
return err
}
if p != nil {
var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
numIndexFiles++
return nil
})

17
internal/restic/backend_find.go
View file

@ -3,9 +3,6 @@ package restic
import (
"context"
"fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
)
// A MultipleIDMatchesError is returned by Find() when multiple IDs with a
@ -27,21 +24,15 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(ctx context.Context, be backend.Lister, t FileType, prefix string) (ID, error) {
func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
match := ID{}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := be.List(ctx, t, func(fi backend.FileInfo) error {
// ignore filename which are not an id
id, err := ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
if len(fi.Name) >= len(prefix) && prefix == fi.Name[:len(prefix)] {
err := be.List(ctx, t, func(id ID, size int64) error {
name := id.String()
if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
if match.IsNull() {
match = id
} else {

52
internal/restic/backend_find_test.go
View file

@ -1,39 +1,31 @@
package restic
package restic_test
import (
"context"
"strings"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
)
type mockBackend struct {
list func(context.Context, FileType, func(backend.FileInfo) error) error
}
func (m mockBackend) List(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error {
return m.list(ctx, t, fn)
}
var samples = IDs{
TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
var samples = restic.IDs{
restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
restic.TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
restic.TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
restic.TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
restic.TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
restic.TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
restic.TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
restic.TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
}
func TestFind(t *testing.T) {
list := samples
m := mockBackend{}
m.list = func(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error {
m := &ListHelper{}
m.ListFn = func(ctx context.Context, t restic.FileType, fn func(id restic.ID, size int64) error) error {
for _, id := range list {
err := fn(backend.FileInfo{Name: id.String()})
err := fn(id, 0)
if err != nil {
return err
}
@ -41,17 +33,17 @@ func TestFind(t *testing.T) {
return nil
}
f, err := Find(context.TODO(), m, SnapshotFile, "20bdc1402a6fc9b633aa")
f, err := restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc1402a6fc9b633aa")
if err != nil {
t.Error(err)
}
expectedMatch := TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff")
expectedMatch := restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff")
if f != expectedMatch {
t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f)
}
f, err = Find(context.TODO(), m, SnapshotFile, "NotAPrefix")
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") {
f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "NotAPrefix")
if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") {
t.Error("Expected no snapshots to be found.")
}
if !f.IsNull() {
@ -60,8 +52,8 @@ func TestFind(t *testing.T) {
// Try to match with a prefix longer than any ID.
extraLengthID := samples[0].String() + "f"
f, err = Find(context.TODO(), m, SnapshotFile, extraLengthID)
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) {
f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, extraLengthID)
if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) {
t.Errorf("Wrong error %v for no snapshots matched", err)
}
if !f.IsNull() {
@ -69,8 +61,8 @@ func TestFind(t *testing.T) {
}
// Use a prefix that will match the prefix of multiple Ids in `samples`.
f, err = Find(context.TODO(), m, SnapshotFile, "20bdc140")
if _, ok := err.(*MultipleIDMatchesError); !ok {
f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc140")
if _, ok := err.(*restic.MultipleIDMatchesError); !ok {
t.Errorf("Wrong error %v for multiple snapshots", err)
}
if !f.IsNull() {

52
internal/restic/lister.go Normal file
View file

@ -0,0 +1,52 @@
package restic
import (
"context"
"fmt"
)
type fileInfo struct {
id ID
size int64
}
type memorizedLister struct {
fileInfos []fileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(ID, int64) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
for _, fi := range m.fileInfos {
if ctx.Err() != nil {
break
}
err := fn(fi.id, fi.size)
if err != nil {
return err
}
}
return ctx.Err()
}
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []fileInfo
err := be.List(ctx, t, func(id ID, size int64) error {
fileInfos = append(fileInfos, fileInfo{id, size})
return nil
})
if err != nil {
return nil, err
}
return &memorizedLister{
fileInfos: fileInfos,
tpe: t,
}, nil
}

68
internal/restic/lister_test.go Normal file
View file

@ -0,0 +1,68 @@
package restic_test
import (
"context"
"fmt"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type ListHelper struct {
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error
}
func (l *ListHelper) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
return l.ListFn(ctx, t, fn)
}
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := &ListHelper{}
type FileInfo struct {
ID restic.ID
Size int64
}
files := []FileInfo{
{ID: restic.NewRandomID(), Size: 42},
{ID: restic.NewRandomID(), Size: 45},
}
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
for _, fi := range files {
if err := fn(fi.ID, fi.Size); err != nil {
return err
}
}
return nil
}
mem, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), backend.IndexFile, func(id restic.ID, size int64) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(id restic.ID, size int64) error {
memFiles = append(memFiles, FileInfo{ID: id, Size: size})
return nil
})
rtest.OK(t, err)
rtest.Equals(t, files, memFiles)
}
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := &ListHelper{}
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(restic.ID, int64) error) error {
return fmt.Errorf("list error")
}
_, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

4
internal/restic/lock.go
View file

@ -403,7 +403,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
// RemoveAllLocks removes all locks forcefully.
func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
var processed uint32
err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil {
atomic.AddUint32(&processed, 1)
@ -421,7 +421,7 @@ func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID
var m sync.Mutex
// For locks decoding is nearly for free, thus just assume were only limited by IO
return ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
if excludeID != nil && id.Equal(*excludeID) {
return nil
}

14
internal/restic/parallel.go
View file

@ -3,13 +3,11 @@ package restic
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"golang.org/x/sync/errgroup"
)
func ParallelList(ctx context.Context, r backend.Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error {
func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error {
type FileIDInfo struct {
ID
Size int64
@ -23,17 +21,11 @@ func ParallelList(ctx context.Context, r backend.Lister, t FileType, parallelism
// send list of index files through ch, which is closed afterwards
wg.Go(func() error {
defer close(ch)
return r.List(ctx, t, func(fi backend.FileInfo) error {
id, err := ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
return r.List(ctx, t, func(id ID, size int64) error {
select {
case <-ctx.Done():
return nil
case ch <- FileIDInfo{id, fi.Size}:
case ch <- FileIDInfo{id, size}:
}
return nil
})

5
internal/restic/repository.go
View file

@ -100,3 +100,8 @@ type MasterIndex interface {
Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error)
}
// Lister allows listing files in a backend.
type Lister interface {
List(ctx context.Context, t FileType, fn func(ID, int64) error) error
}

3
internal/restic/snapshot.go
View file

@ -8,7 +8,6 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
)
@ -80,7 +79,7 @@ func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, er
// If the called function returns an error, this function is cancelled and
// also returns this error.
// If a snapshot ID is in excludeIDs, it will be ignored.
func ForAllSnapshots(ctx context.Context, be backend.Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error {
func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error {
var m sync.Mutex
// For most snapshots decoding is nearly for free, thus just assume were only limited by IO

9
internal/restic/snapshot_find.go
View file

@ -7,7 +7,6 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
)
@ -35,7 +34,7 @@ func (f *SnapshotFilter) matches(sn *Snapshot) bool {
// findLatest finds the latest snapshot with optional target/directory,
// tags, hostname, and timestamp filters.
func (f *SnapshotFilter) findLatest(ctx context.Context, be backend.Lister, loader LoaderUnpacked) (*Snapshot, error) {
func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) {
var err error
absTargets := make([]string, 0, len(f.Paths))
@ -91,7 +90,7 @@ func splitSnapshotID(s string) (id, subfolder string) {
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func FindSnapshot(ctx context.Context, be backend.Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) {
func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) {
s, subfolder := splitSnapshotID(s)
// no need to list snapshots if `s` is already a full id
@ -109,7 +108,7 @@ func FindSnapshot(ctx context.Context, be backend.Lister, loader LoaderUnpacked,
// FindLatest returns either the latest of a filtered list of all snapshots
// or a snapshot specified by `snapshotID`.
func (f *SnapshotFilter) FindLatest(ctx context.Context, be backend.Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) {
func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) {
id, subfolder := splitSnapshotID(snapshotID)
if id == "latest" {
sn, err := f.findLatest(ctx, be, loader)
@ -127,7 +126,7 @@ type SnapshotFindCb func(string, *Snapshot, error) error
var ErrInvalidSnapshotSyntax = errors.New("<snapshot>:<subfolder> syntax not allowed")
// FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func (f *SnapshotFilter) FindAll(ctx context.Context, be backend.Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error {
func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error {
if len(snapshotIDs) != 0 {
var err error
usedFilter := false

8
internal/restic/snapshot_find_test.go
View file

@ -16,7 +16,7 @@ func TestFindLatestSnapshot(t *testing.T) {
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
f := restic.SnapshotFilter{Hosts: []string{"foo"}}
sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
sn, _, err := f.FindLatest(context.TODO(), repo, repo, "latest")
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@ -35,7 +35,7 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
sn, _, err := (&restic.SnapshotFilter{
Hosts: []string{"foo"},
TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"),
}).FindLatest(context.TODO(), repo.Backend(), repo, "latest")
}).FindLatest(context.TODO(), repo, repo, "latest")
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@ -62,7 +62,7 @@ func TestFindLatestWithSubpath(t *testing.T) {
{desiredSnapshot.ID().String() + ":subfolder", "subfolder"},
} {
t.Run("", func(t *testing.T) {
sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query)
sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query)
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@ -78,7 +78,7 @@ func TestFindAllSubpathError(t *testing.T) {
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
count := 0
test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo,
test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo, repo,
[]string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"},
func(id string, sn *restic.Snapshot, err error) error {
if err == restic.ErrInvalidSnapshotSyntax {

2
internal/restic/testing_test.go
View file

@ -20,7 +20,7 @@ const (
// LoadAllSnapshots returns a list of all snapshots in the repo.
// If a snapshot ID is in excludeIDs, it will not be included in the result.
func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) {
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
err = restic.ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
return err
}