alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #3569 from MichaelEischer/strict-locking

Browse source 
Strict repository lock handling
This commit is contained in:
Michael Eischer 2022-10-03 00:44:44 +02:00 committed by GitHub
commit a61fbd287a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 638 additions and 397 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
changelog/unreleased/issue-2715 Normal file
View file

@ -0,0 +1,17 @@
Enhancement: Stricter repository lock handling
Restic commands kept running even if they failed to refresh their locks in
time. This can be a problem if a concurrent call to `unlock` and `prune`
removes data from the repository. Not refreshing a lock in time can for example
be caused by a client switching to standby while running a backup.
Lock handling is now much stricter. Commands requiring a lock are canceled if
the lock is not refreshed successfully in time.
In addition, if a lock file is not readable restic will not allow starting a
command. It may be necessary to remove invalid lock file manually or using
`unlock --remove-all`. Please make sure that no other restic processes are
running concurrently.
https://github.com/restic/restic/issues/2715
https://github.com/restic/restic/pull/3569

19
cmd/restic/cmd_backup.go
View file

@ -57,8 +57,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
}, },
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
var wg sync.WaitGroup var wg sync.WaitGroup
cancelCtx, cancel := context.WithCancel(globalOptions.ctx) cancelCtx, cancel := context.WithCancel(ctx)
defer func() { defer func() {
// shutdown termstatus // shutdown termstatus
cancel() cancel()
@ -72,7 +73,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
term.Run(cancelCtx) term.Run(cancelCtx)
}() }()
return runBackup(backupOptions, globalOptions, term, args) return runBackup(ctx, backupOptions, globalOptions, term, args)
}, },
} }
@ -527,7 +528,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
return parentID, nil return parentID, nil
} }
func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args) err := opts.Check(gopts, args)
if err != nil { if err != nil {
return err return err
@ -550,7 +551,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
Verbosef("open repository\n") Verbosef("open repository\n")
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
@ -577,7 +578,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON)) progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
wg, wgCtx := errgroup.WithContext(gopts.ctx) wg, wgCtx := errgroup.WithContext(ctx)
cancelCtx, cancel := context.WithCancel(wgCtx) cancelCtx, cancel := context.WithCancel(wgCtx)
defer cancel() defer cancel()
wg.Go(func() error { return progressReporter.Run(cancelCtx) }) wg.Go(func() error { return progressReporter.Run(cancelCtx) })
@ -585,7 +586,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON { if !gopts.JSON {
progressPrinter.V("lock repository") progressPrinter.V("lock repository")
} }
lock, err := lockRepo(gopts.ctx, repo) lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -605,7 +606,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
var parentSnapshotID *restic.ID var parentSnapshotID *restic.ID
if !opts.Stdin { if !opts.Stdin {
parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp) parentSnapshotID, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
if err != nil { if err != nil {
return err return err
} }
@ -622,7 +623,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON { if !gopts.JSON {
progressPrinter.V("load index files") progressPrinter.V("load index files")
} }
err = repo.LoadIndex(gopts.ctx) err = repo.LoadIndex(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -727,7 +728,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON { if !gopts.JSON {
progressPrinter.V("start backup on %v", targets) progressPrinter.V("start backup on %v", targets)
} }
_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts) _, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
// cleanly shutdown all running goroutines // cleanly shutdown all running goroutines
cancel() cancel()

29
cmd/restic/cmd_cat.go
View file

@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"encoding/json" "encoding/json"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runCat(globalOptions, args) return runCat(cmd.Context(), globalOptions, args)
}, },
} }
@ -32,23 +33,23 @@ func init() {
cmdRoot.AddCommand(cmdCat) cmdRoot.AddCommand(cmdCat)
} }
func runCat(gopts GlobalOptions, args []string) error { func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) { if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
return errors.Fatal("type or ID not specified") return errors.Fatal("type or ID not specified")
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
defer unlockRepo(lock)
} }
tpe := args[0] tpe := args[0]
@ -62,7 +63,7 @@ func runCat(gopts GlobalOptions, args []string) error {
} }
// find snapshot id with prefix // find snapshot id with prefix
id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1]) id, err = restic.FindSnapshot(ctx, repo.Backend(), args[1])
if err != nil { if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err) return errors.Fatalf("could not find snapshot: %v\n", err)
} }
@ -79,7 +80,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf)) Println(string(buf))
return nil return nil
case "index": case "index":
buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil) buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
if err != nil { if err != nil {
return err return err
} }
@ -87,7 +88,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf)) Println(string(buf))
return nil return nil
case "snapshot": case "snapshot":
sn, err := restic.LoadSnapshot(gopts.ctx, repo, id) sn, err := restic.LoadSnapshot(ctx, repo, id)
if err != nil { if err != nil {
return err return err
} }
@ -100,7 +101,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf)) Println(string(buf))
return nil return nil
case "key": case "key":
key, err := repository.LoadKey(gopts.ctx, repo, id.String()) key, err := repository.LoadKey(ctx, repo, id.String())
if err != nil { if err != nil {
return err return err
} }
@ -121,7 +122,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf)) Println(string(buf))
return nil return nil
case "lock": case "lock":
lock, err := restic.LoadLock(gopts.ctx, repo, id) lock, err := restic.LoadLock(ctx, repo, id)
if err != nil { if err != nil {
return err return err
} }
@ -136,7 +137,7 @@ func runCat(gopts GlobalOptions, args []string) error {
case "pack": case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := restic.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h) buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil { if err != nil {
return err return err
} }
@ -150,7 +151,7 @@ func runCat(gopts GlobalOptions, args []string) error {
return err return err
case "blob": case "blob":
err = repo.LoadIndex(gopts.ctx) err = repo.LoadIndex(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -161,7 +162,7 @@ func runCat(gopts GlobalOptions, args []string) error {
continue continue
} }
buf, err := repo.LoadBlob(gopts.ctx, t, id, nil) buf, err := repo.LoadBlob(ctx, t, id, nil)
if err != nil { if err != nil {
return err return err
} }

22
cmd/restic/cmd_check.go
View file

@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
"strconv" "strconv"
@ -34,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runCheck(checkOptions, globalOptions, args) return runCheck(cmd.Context(), checkOptions, globalOptions, args)
}, },
PreRunE: func(cmd *cobra.Command, args []string) error { PreRunE: func(cmd *cobra.Command, args []string) error {
return checkFlags(checkOptions) return checkFlags(checkOptions)
@ -191,7 +192,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
return cleanup return cleanup
} }
func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 { if len(args) != 0 {
return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
} }
@ -202,14 +203,15 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
return code, nil return code, nil
}) })
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n") Verbosef("create exclusive lock for repository\n")
lock, err := lockRepoExclusive(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -217,13 +219,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
} }
chkr := checker.New(repo, opts.CheckUnused) chkr := checker.New(repo, opts.CheckUnused)
err = chkr.LoadSnapshots(gopts.ctx) err = chkr.LoadSnapshots(ctx)
if err != nil { if err != nil {
return err return err
} }
Verbosef("load indexes\n") Verbosef("load indexes\n")
hints, errs := chkr.LoadIndex(gopts.ctx) hints, errs := chkr.LoadIndex(ctx)
errorsFound := false errorsFound := false
suggestIndexRebuild := false suggestIndexRebuild := false
@ -260,7 +262,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
errChan := make(chan error) errChan := make(chan error)
Verbosef("check all packs\n") Verbosef("check all packs\n")
go chkr.Packs(gopts.ctx, errChan) go chkr.Packs(ctx, errChan)
for err := range errChan { for err := range errChan {
if checker.IsOrphanedPack(err) { if checker.IsOrphanedPack(err) {
@ -287,7 +289,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
defer wg.Done() defer wg.Done()
bar := newProgressMax(!gopts.Quiet, 0, "snapshots") bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
defer bar.Done() defer bar.Done()
chkr.Structure(gopts.ctx, bar, errChan) chkr.Structure(ctx, bar, errChan)
}() }()
for err := range errChan { for err := range errChan {
@ -308,7 +310,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
wg.Wait() wg.Wait()
if opts.CheckUnused { if opts.CheckUnused {
for _, id := range chkr.UnusedBlobs(gopts.ctx) { for _, id := range chkr.UnusedBlobs(ctx) {
Verbosef("unused blob %v\n", id) Verbosef("unused blob %v\n", id)
errorsFound = true errorsFound = true
} }
@ -320,7 +322,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
p := newProgressMax(!gopts.Quiet, packCount, "packs") p := newProgressMax(!gopts.Quiet, packCount, "packs")
errChan := make(chan error) errChan := make(chan error)
go chkr.ReadPacks(gopts.ctx, packs, p, errChan) go chkr.ReadPacks(ctx, packs, p, errChan)
for err := range errChan { for err := range errChan {
errorsFound = true errorsFound = true

16
cmd/restic/cmd_copy.go
View file

@ -32,7 +32,7 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a
new destination repository using the "init" command. new destination repository using the "init" command.
`, `,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runCopy(copyOptions, globalOptions, args) return runCopy(cmd.Context(), copyOptions, globalOptions, args)
}, },
} }
@ -52,7 +52,7 @@ func init() {
initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true) initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true)
} }
func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination") secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
if err != nil { if err != nil {
return err return err
@ -62,28 +62,26 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
gopts, secondaryGopts = secondaryGopts, gopts gopts, secondaryGopts = secondaryGopts, gopts
} }
ctx, cancel := context.WithCancel(gopts.ctx) srcRepo, err := OpenRepository(ctx, gopts)
defer cancel()
srcRepo, err := OpenRepository(gopts)
if err != nil { if err != nil {
return err return err
} }
dstRepo, err := OpenRepository(secondaryGopts) dstRepo, err := OpenRepository(ctx, secondaryGopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
srcLock, err := lockRepo(ctx, srcRepo) var srcLock *restic.Lock
srcLock, ctx, err = lockRepo(ctx, srcRepo)
defer unlockRepo(srcLock) defer unlockRepo(srcLock)
if err != nil { if err != nil {
return err return err
} }
} }
dstLock, err := lockRepo(ctx, dstRepo) dstLock, ctx, err := lockRepo(ctx, dstRepo)
defer unlockRepo(dstLock) defer unlockRepo(dstLock)
if err != nil { if err != nil {
return err return err

34
cmd/restic/cmd_debug.go
View file

@ -46,7 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDebugDump(globalOptions, args) return runDebugDump(cmd.Context(), globalOptions, args)
}, },
} }
@ -141,18 +141,19 @@ func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) erro
}) })
} }
func runDebugDump(gopts GlobalOptions, args []string) error { func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 { if len(args) != 1 {
return errors.Fatal("type not specified") return errors.Fatal("type not specified")
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -163,20 +164,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
switch tpe { switch tpe {
case "indexes": case "indexes":
return dumpIndexes(gopts.ctx, repo, gopts.stdout) return dumpIndexes(ctx, repo, gopts.stdout)
case "snapshots": case "snapshots":
return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout) return debugPrintSnapshots(ctx, repo, gopts.stdout)
case "packs": case "packs":
return printPacks(gopts.ctx, repo, gopts.stdout) return printPacks(ctx, repo, gopts.stdout)
case "all": case "all":
Printf("snapshots:\n") Printf("snapshots:\n")
err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout) err := debugPrintSnapshots(ctx, repo, gopts.stdout)
if err != nil { if err != nil {
return err return err
} }
Printf("\nindexes:\n") Printf("\nindexes:\n")
err = dumpIndexes(gopts.ctx, repo, gopts.stdout) err = dumpIndexes(ctx, repo, gopts.stdout)
if err != nil { if err != nil {
return err return err
} }
@ -192,7 +193,7 @@ var cmdDebugExamine = &cobra.Command{
Short: "Examine a pack file", Short: "Examine a pack file",
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDebugExamine(globalOptions, args) return runDebugExamine(cmd.Context(), globalOptions, args)
}, },
} }
@ -426,8 +427,8 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
return nil return nil
} }
func runDebugExamine(gopts GlobalOptions, args []string) error { func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
@ -436,7 +437,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
for _, name := range args { for _, name := range args {
id, err := restic.ParseID(name) id, err := restic.ParseID(name)
if err != nil { if err != nil {
name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name) name, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
if err == nil { if err == nil {
id, err = restic.ParseID(name) id, err = restic.ParseID(name)
} }
@ -453,20 +454,21 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
} }
err = repo.LoadIndex(gopts.ctx) err = repo.LoadIndex(ctx)
if err != nil { if err != nil {
return err return err
} }
for _, id := range ids { for _, id := range ids {
err := examinePack(gopts.ctx, repo, id) err := examinePack(ctx, repo, id)
if err != nil { if err != nil {
Warnf("error: %v\n", err) Warnf("error: %v\n", err)
} }

12
cmd/restic/cmd_diff.go
View file

@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDiff(diffOptions, globalOptions, args) return runDiff(cmd.Context(), diffOptions, globalOptions, args)
}, },
} }
@ -321,21 +321,19 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return nil return nil
} }
func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 { if len(args) != 2 {
return errors.Fatalf("specify two snapshot IDs") return errors.Fatalf("specify two snapshot IDs")
} }
ctx, cancel := context.WithCancel(gopts.ctx) repo, err := OpenRepository(ctx, gopts)
defer cancel()
repo, err := OpenRepository(gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err

13
cmd/restic/cmd_dump.go
View file

@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDump(dumpOptions, globalOptions, args) return runDump(cmd.Context(), dumpOptions, globalOptions, args)
}, },
} }
@ -107,9 +107,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
return fmt.Errorf("path %q not found in snapshot", item) return fmt.Errorf("path %q not found in snapshot", item)
} }
func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
ctx := gopts.ctx
if len(args) != 2 { if len(args) != 2 {
return errors.Fatal("no file and no snapshot ID specified") return errors.Fatal("no file and no snapshot ID specified")
} }
@ -127,13 +125,14 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
splittedPath := splitPath(path.Clean(pathToPrint)) splittedPath := splitPath(path.Clean(pathToPrint))
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -154,7 +153,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
} }
} }
sn, err := restic.LoadSnapshot(gopts.ctx, repo, id) sn, err := restic.LoadSnapshot(ctx, repo, id)
if err != nil { if err != nil {
Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err) Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
} }

16
cmd/restic/cmd_find.go
View file

@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runFind(findOptions, globalOptions, args) return runFind(cmd.Context(), findOptions, globalOptions, args)
}, },
} }
@ -534,7 +534,7 @@ func (f *Finder) findObjectsPacks(ctx context.Context) {
} }
} }
func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 { if len(args) == 0 {
return errors.Fatal("wrong number of arguments") return errors.Fatal("wrong number of arguments")
} }
@ -568,31 +568,29 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("cannot have several ID types") return errors.Fatal("cannot have several ID types")
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
} }
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }
if err = repo.LoadIndex(gopts.ctx); err != nil { if err = repo.LoadIndex(ctx); err != nil {
return err return err
} }
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
f := &Finder{ f := &Finder{
repo: repo, repo: repo,
pat: pat, pat: pat,

16
cmd/restic/cmd_forget.go
View file

@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runForget(forgetOptions, globalOptions, args) return runForget(cmd.Context(), forgetOptions, globalOptions, args)
}, },
} }
@ -99,13 +99,13 @@ func init() {
addPruneOptions(cmdForget) addPruneOptions(cmdForget)
} }
func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
err := verifyPruneOptions(&pruneOptions) err := verifyPruneOptions(&pruneOptions)
if err != nil { if err != nil {
return err return err
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
@ -115,16 +115,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
} }
if !opts.DryRun || !gopts.NoLock { if !opts.DryRun || !gopts.NoLock {
lock, err := lockRepoExclusive(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
} }
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var snapshots restic.Snapshots var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet() removeSnIDs := restic.NewIDSet()
@ -219,7 +217,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
if len(removeSnIDs) > 0 { if len(removeSnIDs) > 0 {
if !opts.DryRun { if !opts.DryRun {
err := DeleteFilesChecked(gopts, repo, removeSnIDs, restic.SnapshotFile) err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }
@ -242,7 +240,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs)) Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
} }
pruneOptions.DryRun = opts.DryRun pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs) return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
} }
return nil return nil

15
cmd/restic/cmd_init.go
View file

@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"strconv" "strconv"
"github.com/restic/chunker" "github.com/restic/chunker"
@ -25,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runInit(initOptions, globalOptions, args) return runInit(cmd.Context(), initOptions, globalOptions, args)
}, },
} }
@ -47,7 +48,7 @@ func init() {
f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'") f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
} }
func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
var version uint var version uint
if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" { if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
version = restic.MaxRepoVersion version = restic.MaxRepoVersion
@ -64,7 +65,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion) return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
} }
chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts) chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts)
if err != nil { if err != nil {
return err return err
} }
@ -81,7 +82,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err return err
} }
be, err := create(repo, gopts.extended) be, err := create(ctx, repo, gopts.extended)
if err != nil { if err != nil {
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
} }
@ -94,7 +95,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err return err
} }
err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial) err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
if err != nil { if err != nil {
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
} }
@ -108,14 +109,14 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return nil return nil
} }
func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) { func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
if opts.CopyChunkerParameters { if opts.CopyChunkerParameters {
otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary") otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
if err != nil { if err != nil {
return nil, err return nil, err
} }
otherRepo, err := OpenRepository(otherGopts) otherRepo, err := OpenRepository(ctx, otherGopts)
if err != nil { if err != nil {
return nil, err return nil, err
} }

37
cmd/restic/cmd_key.go
View file

@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runKey(globalOptions, args) return runKey(cmd.Context(), globalOptions, args)
}, },
} }
@ -120,18 +120,18 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
"enter password again: ") "enter password again: ")
} }
func addKey(gopts GlobalOptions, repo *repository.Repository) error { func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts) pw, err := getNewPassword(gopts)
if err != nil { if err != nil {
return err return err
} }
id, err := repository.AddKey(gopts.ctx, repo, pw, keyUsername, keyHostname, repo.Key()) id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
if err != nil { if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err) return errors.Fatalf("creating new key failed: %v\n", err)
} }
err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw) err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil { if err != nil {
return err return err
} }
@ -156,25 +156,25 @@ func deleteKey(ctx context.Context, repo *repository.Repository, name string) er
return nil return nil
} }
func changePassword(gopts GlobalOptions, repo *repository.Repository) error { func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts) pw, err := getNewPassword(gopts)
if err != nil { if err != nil {
return err return err
} }
id, err := repository.AddKey(gopts.ctx, repo, pw, "", "", repo.Key()) id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
if err != nil { if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err) return errors.Fatalf("creating new key failed: %v\n", err)
} }
oldID := repo.KeyName() oldID := repo.KeyName()
err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw) err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil { if err != nil {
return err return err
} }
h := restic.Handle{Type: restic.KeyFile, Name: oldID} h := restic.Handle{Type: restic.KeyFile, Name: oldID}
err = repo.Backend().Remove(gopts.ctx, h) err = repo.Backend().Remove(ctx, h)
if err != nil { if err != nil {
return err return err
} }
@ -197,22 +197,19 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos
return nil return nil
} }
func runKey(gopts GlobalOptions, args []string) error { func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
return errors.Fatal("wrong number of arguments") return errors.Fatal("wrong number of arguments")
} }
ctx, cancel := context.WithCancel(gopts.ctx) repo, err := OpenRepository(ctx, gopts)
defer cancel()
repo, err := OpenRepository(gopts)
if err != nil { if err != nil {
return err return err
} }
switch args[0] { switch args[0] {
case "list": case "list":
lock, err := lockRepo(ctx, repo) lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -220,15 +217,15 @@ func runKey(gopts GlobalOptions, args []string) error {
return listKeys(ctx, repo, gopts) return listKeys(ctx, repo, gopts)
case "add": case "add":
lock, err := lockRepo(ctx, repo) lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
return addKey(gopts, repo) return addKey(ctx, repo, gopts)
case "remove": case "remove":
lock, err := lockRepoExclusive(ctx, repo) lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -239,15 +236,15 @@ func runKey(gopts GlobalOptions, args []string) error {
return err return err
} }
return deleteKey(gopts.ctx, repo, id) return deleteKey(ctx, repo, id)
case "passwd": case "passwd":
lock, err := lockRepoExclusive(ctx, repo) lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
return changePassword(gopts, repo) return changePassword(ctx, repo, gopts)
} }
return nil return nil

17
cmd/restic/cmd_list.go
View file

@ -1,6 +1,8 @@
package main package main
import ( import (
"context"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -21,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runList(cmd, globalOptions, args) return runList(cmd.Context(), cmd, globalOptions, args)
}, },
} }
@ -29,18 +31,19 @@ func init() {
cmdRoot.AddCommand(cmdList) cmdRoot.AddCommand(cmdList)
} }
func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error { func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error {
if len(args) != 1 { if len(args) != 1 {
return errors.Fatal("type not specified, usage: " + cmd.Use) return errors.Fatal("type not specified, usage: " + cmd.Use)
} }
repo, err := OpenRepository(opts) repo, err := OpenRepository(ctx, opts)
if err != nil { if err != nil {
return err return err
} }
if !opts.NoLock && args[0] != "locks" { if !opts.NoLock && args[0] != "locks" {
lock, err := lockRepo(opts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -60,11 +63,11 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
case "locks": case "locks":
t = restic.LockFile t = restic.LockFile
case "blobs": case "blobs":
return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
return err return err
} }
idx.Each(opts.ctx, func(blobs restic.PackedBlob) { idx.Each(ctx, func(blobs restic.PackedBlob) {
Printf("%v %v\n", blobs.Type, blobs.ID) Printf("%v %v\n", blobs.Type, blobs.ID)
}) })
return nil return nil
@ -73,7 +76,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
return errors.Fatal("invalid type") return errors.Fatal("invalid type")
} }
return repo.List(opts.ctx, t, func(id restic.ID, size int64) error { return repo.List(ctx, t, func(id restic.ID, size int64) error {
Printf("%s\n", id) Printf("%s\n", id)
return nil return nil
}) })

13
cmd/restic/cmd_ls.go
View file

@ -42,7 +42,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runLs(lsOptions, globalOptions, args) return runLs(cmd.Context(), lsOptions, globalOptions, args)
}, },
} }
@ -111,7 +111,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
return enc.Encode(n) return enc.Encode(n)
} }
func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 { if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
} }
@ -161,23 +161,20 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
return false return false
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }
if err = repo.LoadIndex(gopts.ctx); err != nil { if err = repo.LoadIndex(ctx); err != nil {
return err return err
} }
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var ( var (
printSnapshot func(sn *restic.Snapshot) printSnapshot func(sn *restic.Snapshot)
printNode func(path string, node *restic.Node) printNode func(path string, node *restic.Node)

23
cmd/restic/cmd_migrate.go
View file

@ -1,6 +1,8 @@
package main package main
import ( import (
"context"
"github.com/restic/restic/internal/migrations" "github.com/restic/restic/internal/migrations"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runMigrate(migrateOptions, globalOptions, args) return runMigrate(cmd.Context(), migrateOptions, globalOptions, args)
}, },
} }
@ -39,8 +41,7 @@ func init() {
f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`) f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
} }
func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error { func checkMigrations(ctx context.Context, repo restic.Repository) error {
ctx := gopts.ctx
Printf("available migrations:\n") Printf("available migrations:\n")
found := false found := false
@ -63,9 +64,7 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
return nil return nil
} }
func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error { func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
ctx := gopts.ctx
var firsterr error var firsterr error
for _, name := range args { for _, name := range args {
for _, m := range migrations.All { for _, m := range migrations.All {
@ -94,7 +93,7 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
checkGopts := gopts checkGopts := gopts
// the repository is already locked // the repository is already locked
checkGopts.NoLock = true checkGopts.NoLock = true
err = runCheck(checkOptions, checkGopts, []string{}) err = runCheck(ctx, checkOptions, checkGopts, []string{})
if err != nil { if err != nil {
return err return err
} }
@ -117,21 +116,21 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
return firsterr return firsterr
} }
func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error { func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
lock, err := lockRepoExclusive(gopts.ctx, repo) lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
if len(args) == 0 { if len(args) == 0 {
return checkMigrations(opts, gopts, repo) return checkMigrations(ctx, repo)
} }
return applyMigrations(opts, gopts, repo, args) return applyMigrations(ctx, opts, gopts, repo, args)
} }

13
cmd/restic/cmd_mount.go
View file

@ -4,6 +4,7 @@
package main package main
import ( import (
"context"
"os" "os"
"strings" "strings"
"time" "time"
@ -12,6 +13,7 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
resticfs "github.com/restic/restic/internal/fs" resticfs "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/fuse" "github.com/restic/restic/internal/fuse"
@ -66,7 +68,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runMount(mountOptions, globalOptions, args) return runMount(cmd.Context(), mountOptions, globalOptions, args)
}, },
} }
@ -98,7 +100,7 @@ func init() {
_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template") _ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
} }
func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
if opts.TimeTemplate == "" { if opts.TimeTemplate == "" {
return errors.Fatal("time template string cannot be empty") return errors.Fatal("time template string cannot be empty")
} }
@ -114,20 +116,21 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
debug.Log("start mount") debug.Log("start mount")
defer debug.Log("finish mount") defer debug.Log("finish mount")
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
} }
err = repo.LoadIndex(gopts.ctx) err = repo.LoadIndex(ctx)
if err != nil { if err != nil {
return err return err
} }

53
cmd/restic/cmd_prune.go
View file

@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runPrune(pruneOptions, globalOptions) return runPrune(cmd.Context(), pruneOptions, globalOptions)
}, },
} }
@ -134,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
return nil return nil
} }
func runPrune(opts PruneOptions, gopts GlobalOptions) error { func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
err := verifyPruneOptions(&opts) err := verifyPruneOptions(&opts)
if err != nil { if err != nil {
return err return err
@ -144,7 +144,7 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive") return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
@ -165,16 +165,16 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
opts.unsafeRecovery = true opts.unsafeRecovery = true
} }
lock, err := lockRepoExclusive(gopts.ctx, repo) lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
return runPruneWithRepo(opts, gopts, repo, restic.NewIDSet()) return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
} }
func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error { func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
// we do not need index updates while pruning! // we do not need index updates while pruning!
repo.DisableAutoIndexUpdate() repo.DisableAutoIndexUpdate()
@ -184,12 +184,12 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
Verbosef("loading indexes...\n") Verbosef("loading indexes...\n")
// loading the index before the snapshots is ok, as we use an exclusive lock here // loading the index before the snapshots is ok, as we use an exclusive lock here
err := repo.LoadIndex(gopts.ctx) err := repo.LoadIndex(ctx)
if err != nil { if err != nil {
return err return err
} }
plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots) plan, stats, err := planPrune(ctx, opts, gopts, repo, ignoreSnapshots)
if err != nil { if err != nil {
return err return err
} }
@ -199,7 +199,7 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
return err return err
} }
return doPrune(opts, gopts, repo, plan) return doPrune(ctx, opts, gopts, repo, plan)
} }
type pruneStats struct { type pruneStats struct {
@ -255,11 +255,10 @@ type packInfoWithID struct {
// planPrune selects which files to rewrite and which to delete and which blobs to keep. // planPrune selects which files to rewrite and which to delete and which blobs to keep.
// Also some summary statistics are returned. // Also some summary statistics are returned.
func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) { func planPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
ctx := gopts.ctx
var stats pruneStats var stats pruneStats
usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots) usedBlobs, err := getUsedBlobs(ctx, gopts, repo, ignoreSnapshots)
if err != nil { if err != nil {
return prunePlan{}, stats, err return prunePlan{}, stats, err
} }
@ -652,9 +651,7 @@ func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
// - rebuild the index while ignoring all files that will be deleted // - rebuild the index while ignoring all files that will be deleted
// - delete the files // - delete the files
// plan.removePacks and plan.ignorePacks are modified in this function. // plan.removePacks and plan.ignorePacks are modified in this function.
func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) { func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
ctx := gopts.ctx
if opts.DryRun { if opts.DryRun {
if !gopts.JSON && gopts.verbosity >= 2 { if !gopts.JSON && gopts.verbosity >= 2 {
if len(plan.removePacksFirst) > 0 { if len(plan.removePacksFirst) > 0 {
@ -670,7 +667,7 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
// unreferenced packs can be safely deleted first // unreferenced packs can be safely deleted first
if len(plan.removePacksFirst) != 0 { if len(plan.removePacksFirst) != 0 {
Verbosef("deleting unreferenced packs\n") Verbosef("deleting unreferenced packs\n")
DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile) DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
} }
if len(plan.repackPacks) != 0 { if len(plan.repackPacks) != 0 {
@ -703,12 +700,12 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
if opts.unsafeRecovery { if opts.unsafeRecovery {
Verbosef("deleting index files\n") Verbosef("deleting index files\n")
indexFiles := repo.Index().(*repository.MasterIndex).IDs() indexFiles := repo.Index().(*repository.MasterIndex).IDs()
err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile) err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
if err != nil { if err != nil {
return errors.Fatalf("%s", err) return errors.Fatalf("%s", err)
} }
} else if len(plan.ignorePacks) != 0 { } else if len(plan.ignorePacks) != 0 {
err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil) err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
if err != nil { if err != nil {
return errors.Fatalf("%s", err) return errors.Fatalf("%s", err)
} }
@ -716,11 +713,11 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
if len(plan.removePacks) != 0 { if len(plan.removePacks) != 0 {
Verbosef("removing %d old packs\n", len(plan.removePacks)) Verbosef("removing %d old packs\n", len(plan.removePacks))
DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile) DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
} }
if opts.unsafeRecovery { if opts.unsafeRecovery {
_, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil) _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
if err != nil { if err != nil {
return errors.Fatalf("%s", err) return errors.Fatalf("%s", err)
} }
@ -730,31 +727,29 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
return nil return nil
} }
func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
Verbosef("rebuilding index\n") Verbosef("rebuilding index\n")
bar := newProgressMax(!gopts.Quiet, 0, "packs processed") bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar) obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
bar.Done() bar.Done()
return obsoleteIndexes, err return obsoleteIndexes, err
} }
func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete) obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
if err != nil { if err != nil {
return err return err
} }
Verbosef("deleting obsolete index files\n") Verbosef("deleting obsolete index files\n")
return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile) return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
} }
func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) { func getUsedBlobs(ctx context.Context, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) {
ctx := gopts.ctx
var snapshotTrees restic.IDs var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n") Verbosef("loading all snapshots...\n")
err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots, err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error { func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err) debug.Log("failed to load snapshot %v (error %v)", id, err)

18
cmd/restic/cmd_rebuild_index.go
View file

@ -1,6 +1,8 @@
package main package main
import ( import (
"context"
"github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runRebuildIndex(rebuildIndexOptions, globalOptions) return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
}, },
} }
@ -40,24 +42,22 @@ func init() {
} }
func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error { func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
lock, err := lockRepoExclusive(gopts.ctx, repo) lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
return rebuildIndex(opts, gopts, repo, restic.NewIDSet()) return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
} }
func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
ctx := gopts.ctx
var obsoleteIndexes restic.IDs var obsoleteIndexes restic.IDs
packSizeFromList := make(map[restic.ID]int64) packSizeFromList := make(map[restic.ID]int64)
packSizeFromIndex := make(map[restic.ID]int64) packSizeFromIndex := make(map[restic.ID]int64)
@ -141,7 +141,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
} }
} }
err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes) err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes)
if err != nil { if err != nil {
return err return err
} }

28
cmd/restic/cmd_recover.go
View file

@ -27,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runRecover(globalOptions) return runRecover(cmd.Context(), globalOptions)
}, },
} }
@ -35,30 +35,30 @@ func init() {
cmdRoot.AddCommand(cmdRecover) cmdRoot.AddCommand(cmdRecover)
} }
func runRecover(gopts GlobalOptions) error { func runRecover(ctx context.Context, gopts GlobalOptions) error {
hostname, err := os.Hostname() hostname, err := os.Hostname()
if err != nil { if err != nil {
return err return err
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
lock, err := lockRepo(gopts.ctx, repo) lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }
Verbosef("load index files\n") Verbosef("load index files\n")
if err = repo.LoadIndex(gopts.ctx); err != nil { if err = repo.LoadIndex(ctx); err != nil {
return err return err
} }
@ -66,7 +66,7 @@ func runRecover(gopts GlobalOptions) error {
// tree. If it is not referenced, we have a root tree. // tree. If it is not referenced, we have a root tree.
trees := make(map[restic.ID]bool) trees := make(map[restic.ID]bool)
repo.Index().Each(gopts.ctx, func(blob restic.PackedBlob) { repo.Index().Each(ctx, func(blob restic.PackedBlob) {
if blob.Type == restic.TreeBlob { if blob.Type == restic.TreeBlob {
trees[blob.Blob.ID] = false trees[blob.Blob.ID] = false
} }
@ -75,7 +75,7 @@ func runRecover(gopts GlobalOptions) error {
Verbosef("load %d trees\n", len(trees)) Verbosef("load %d trees\n", len(trees))
bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
for id := range trees { for id := range trees {
tree, err := restic.LoadTree(gopts.ctx, repo, id) tree, err := restic.LoadTree(ctx, repo, id)
if err != nil { if err != nil {
Warnf("unable to load tree %v: %v\n", id.Str(), err) Warnf("unable to load tree %v: %v\n", id.Str(), err)
continue continue
@ -91,7 +91,7 @@ func runRecover(gopts GlobalOptions) error {
bar.Done() bar.Done()
Verbosef("load snapshots\n") Verbosef("load snapshots\n")
err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
trees[*sn.Tree] = true trees[*sn.Tree] = true
return nil return nil
}) })
@ -132,18 +132,18 @@ func runRecover(gopts GlobalOptions) error {
} }
} }
wg, ctx := errgroup.WithContext(gopts.ctx) wg, wgCtx := errgroup.WithContext(ctx)
repo.StartPackUploader(ctx, wg) repo.StartPackUploader(wgCtx, wg)
var treeID restic.ID var treeID restic.ID
wg.Go(func() error { wg.Go(func() error {
var err error var err error
treeID, err = restic.SaveTree(ctx, repo, tree) treeID, err = restic.SaveTree(wgCtx, repo, tree)
if err != nil { if err != nil {
return errors.Fatalf("unable to save new tree to the repository: %v", err) return errors.Fatalf("unable to save new tree to the repository: %v", err)
} }
err = repo.Flush(ctx) err = repo.Flush(wgCtx)
if err != nil { if err != nil {
return errors.Fatalf("unable to save blobs to the repository: %v", err) return errors.Fatalf("unable to save blobs to the repository: %v", err)
} }
@ -154,7 +154,7 @@ func runRecover(gopts GlobalOptions) error {
return err return err
} }
return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID) return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
} }

11
cmd/restic/cmd_restore.go
View file

@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"strings" "strings"
"time" "time"
@ -30,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runRestore(restoreOptions, globalOptions, args) return runRestore(cmd.Context(), restoreOptions, globalOptions, args)
}, },
} }
@ -63,8 +64,7 @@ func init() {
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
} }
func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error {
ctx := gopts.ctx
hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
@ -117,13 +117,14 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
debug.Log("restore %v to %v", snapshotIDString, opts.Target) debug.Log("restore %v to %v", snapshotIDString, opts.Target)
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err

7
cmd/restic/cmd_self_update.go
View file

@ -3,6 +3,7 @@
package main package main
import ( import (
"context"
"os" "os"
"path/filepath" "path/filepath"
@ -27,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runSelfUpdate(selfUpdateOptions, globalOptions, args) return runSelfUpdate(cmd.Context(), selfUpdateOptions, globalOptions, args)
}, },
} }
@ -45,7 +46,7 @@ func init() {
flags.StringVar(&selfUpdateOptions.Output, "output", "", "Save the downloaded file as `filename` (default: running binary itself)") flags.StringVar(&selfUpdateOptions.Output, "output", "", "Save the downloaded file as `filename` (default: running binary itself)")
} }
func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error { func runSelfUpdate(ctx context.Context, opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
if opts.Output == "" { if opts.Output == "" {
file, err := os.Executable() file, err := os.Executable()
if err != nil { if err != nil {
@ -73,7 +74,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
Verbosef("writing restic to %v\n", opts.Output) Verbosef("writing restic to %v\n", opts.Output)
v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef) v, err := selfupdate.DownloadLatestStableRelease(ctx, opts.Output, version, Verbosef)
if err != nil { if err != nil {
return errors.Fatalf("unable to update restic: %v", err) return errors.Fatalf("unable to update restic: %v", err)
} }

12
cmd/restic/cmd_snapshots.go
View file

@ -26,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runSnapshots(snapshotOptions, globalOptions, args) return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args)
}, },
} }
@ -57,23 +57,21 @@ func init() {
f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma") f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma")
} }
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
} }
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var snapshots restic.Snapshots var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)

14
cmd/restic/cmd_stats.go
View file

@ -47,7 +47,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runStats(globalOptions, args) return runStats(cmd.Context(), globalOptions, args)
}, },
} }
@ -68,29 +68,27 @@ func init() {
initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true) initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true)
} }
func runStats(gopts GlobalOptions, args []string) error { func runStats(ctx context.Context, gopts GlobalOptions, args []string) error {
err := verifyStatsInput(gopts, args) err := verifyStatsInput(gopts, args)
if err != nil { if err != nil {
return err return err
} }
ctx, cancel := context.WithCancel(gopts.ctx) repo, err := OpenRepository(ctx, gopts)
defer cancel()
repo, err := OpenRepository(gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
lock, err := lockRepo(ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
} }
} }
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

11
cmd/restic/cmd_tag.go
View file

@ -29,7 +29,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runTag(tagOptions, globalOptions, args) return runTag(cmd.Context(), tagOptions, globalOptions, args)
}, },
} }
@ -95,7 +95,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
return changed, nil return changed, nil
} }
func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []string) error {
if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 { if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
return errors.Fatal("nothing to do!") return errors.Fatal("nothing to do!")
} }
@ -103,14 +103,15 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("--set and --add/--remove cannot be given at the same time") return errors.Fatal("--set and --add/--remove cannot be given at the same time")
} }
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
if !gopts.NoLock { if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n") Verbosef("create exclusive lock for repository\n")
lock, err := lockRepoExclusive(gopts.ctx, repo) var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock) defer unlockRepo(lock)
if err != nil { if err != nil {
return err return err
@ -118,8 +119,6 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
} }
changeCnt := 0 changeCnt := 0
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil { if err != nil {

10
cmd/restic/cmd_unlock.go
View file

@ -1,6 +1,8 @@
package main package main
import ( import (
"context"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -18,7 +20,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runUnlock(unlockOptions, globalOptions) return runUnlock(cmd.Context(), unlockOptions, globalOptions)
}, },
} }
@ -35,8 +37,8 @@ func init() {
unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones") unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
} }
func runUnlock(opts UnlockOptions, gopts GlobalOptions) error { func runUnlock(ctx context.Context, opts UnlockOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(gopts) repo, err := OpenRepository(ctx, gopts)
if err != nil { if err != nil {
return err return err
} }
@ -46,7 +48,7 @@ func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
fn = restic.RemoveAllLocks fn = restic.RemoveAllLocks
} }
processed, err := fn(gopts.ctx, repo) processed, err := fn(ctx, repo)
if err != nil { if err != nil {
return err return err
} }

14
cmd/restic/delete.go
View file

@ -1,6 +1,8 @@
package main package main
import ( import (
"context"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -8,22 +10,22 @@ import (
// DeleteFiles deletes the given fileList of fileType in parallel // DeleteFiles deletes the given fileList of fileType in parallel
// it will print a warning if there is an error, but continue deleting the remaining files // it will print a warning if there is an error, but continue deleting the remaining files
func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) { func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
_ = deleteFiles(gopts, true, repo, fileList, fileType) _ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
} }
// DeleteFilesChecked deletes the given fileList of fileType in parallel // DeleteFilesChecked deletes the given fileList of fileType in parallel
// if an error occurs, it will cancel and return this error // if an error occurs, it will cancel and return this error
func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
return deleteFiles(gopts, false, repo, fileList, fileType) return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
} }
// deleteFiles deletes the given fileList of fileType in parallel // deleteFiles deletes the given fileList of fileType in parallel
// if ignoreError=true, it will print a warning if there was an error, else it will abort. // if ignoreError=true, it will print a warning if there was an error, else it will abort.
func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
totalCount := len(fileList) totalCount := len(fileList)
fileChan := make(chan restic.ID) fileChan := make(chan restic.ID)
wg, ctx := errgroup.WithContext(gopts.ctx) wg, ctx := errgroup.WithContext(ctx)
wg.Go(func() error { wg.Go(func() error {
defer close(fileChan) defer close(fileChan)
for id := range fileList { for id := range fileList {

40
cmd/restic/global.go
View file

@ -68,7 +68,6 @@ type GlobalOptions struct {
backend.TransportOptions backend.TransportOptions
limiter.Limits limiter.Limits
ctx context.Context
password string password string
stdout io.Writer stdout io.Writer
stderr io.Writer stderr io.Writer
@ -93,10 +92,11 @@ var globalOptions = GlobalOptions{
} }
var isReadingPassword bool var isReadingPassword bool
var internalGlobalCtx context.Context
func init() { func init() {
var cancel context.CancelFunc var cancel context.CancelFunc
globalOptions.ctx, cancel = context.WithCancel(context.Background()) internalGlobalCtx, cancel = context.WithCancel(context.Background())
AddCleanupHandler(func(code int) (int, error) { AddCleanupHandler(func(code int) (int, error) {
// Must be called before the unlock cleanup handler to ensure that the latter is // Must be called before the unlock cleanup handler to ensure that the latter is
// not blocked due to limited number of backend connections, see #1434 // not blocked due to limited number of backend connections, see #1434
@ -428,13 +428,13 @@ func ReadRepo(opts GlobalOptions) (string, error) {
const maxKeys = 20 const maxKeys = 20
// OpenRepository reads the password and opens the repository. // OpenRepository reads the password and opens the repository.
func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Repository, error) {
repo, err := ReadRepo(opts) repo, err := ReadRepo(opts)
if err != nil { if err != nil {
return nil, err return nil, err
} }
be, err := open(repo, opts, opts.extended) be, err := open(ctx, repo, opts, opts.extended)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -478,7 +478,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
continue continue
} }
err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint) err = s.SearchKey(ctx, opts.password, maxKeys, opts.KeyHint)
if err != nil && passwordTriesLeft > 1 { if err != nil && passwordTriesLeft > 1 {
opts.password = "" opts.password = ""
fmt.Fprintf(os.Stderr, "%s. Try again\n", err) fmt.Fprintf(os.Stderr, "%s. Try again\n", err)
@ -695,7 +695,7 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
} }
// Open the backend specified by a location config. // Open the backend specified by a location config.
func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(s)) debug.Log("parsing location %v", location.StripPassword(s))
loc, err := location.Parse(s) loc, err := location.Parse(s)
if err != nil { if err != nil {
@ -720,19 +720,19 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
switch loc.Scheme { switch loc.Scheme {
case "local": case "local":
be, err = local.Open(globalOptions.ctx, cfg.(local.Config)) be, err = local.Open(ctx, cfg.(local.Config))
case "sftp": case "sftp":
be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config)) be, err = sftp.Open(ctx, cfg.(sftp.Config))
case "s3": case "s3":
be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt) be, err = s3.Open(ctx, cfg.(s3.Config), rt)
case "gs": case "gs":
be, err = gs.Open(cfg.(gs.Config), rt) be, err = gs.Open(cfg.(gs.Config), rt)
case "azure": case "azure":
be, err = azure.Open(cfg.(azure.Config), rt) be, err = azure.Open(cfg.(azure.Config), rt)
case "swift": case "swift":
be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt) be, err = swift.Open(ctx, cfg.(swift.Config), rt)
case "b2": case "b2":
be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt) be, err = b2.Open(ctx, cfg.(b2.Config), rt)
case "rest": case "rest":
be, err = rest.Open(cfg.(rest.Config), rt) be, err = rest.Open(cfg.(rest.Config), rt)
case "rclone": case "rclone":
@ -760,7 +760,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
} }
// check if config is there // check if config is there
fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile}) fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil { if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s)) return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s))
} }
@ -773,7 +773,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
} }
// Create the backend specified by URI. // Create the backend specified by URI.
func create(s string, opts options.Options) (restic.Backend, error) { func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", s) debug.Log("parsing location %v", s)
loc, err := location.Parse(s) loc, err := location.Parse(s)
if err != nil { if err != nil {
@ -792,23 +792,23 @@ func create(s string, opts options.Options) (restic.Backend, error) {
switch loc.Scheme { switch loc.Scheme {
case "local": case "local":
return local.Create(globalOptions.ctx, cfg.(local.Config)) return local.Create(ctx, cfg.(local.Config))
case "sftp": case "sftp":
return sftp.Create(globalOptions.ctx, cfg.(sftp.Config)) return sftp.Create(ctx, cfg.(sftp.Config))
case "s3": case "s3":
return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt) return s3.Create(ctx, cfg.(s3.Config), rt)
case "gs": case "gs":
return gs.Create(cfg.(gs.Config), rt) return gs.Create(cfg.(gs.Config), rt)
case "azure": case "azure":
return azure.Create(cfg.(azure.Config), rt) return azure.Create(cfg.(azure.Config), rt)
case "swift": case "swift":
return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt) return swift.Open(ctx, cfg.(swift.Config), rt)
case "b2": case "b2":
return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt) return b2.Create(ctx, cfg.(b2.Config), rt)
case "rest": case "rest":
return rest.Create(globalOptions.ctx, cfg.(rest.Config), rt) return rest.Create(ctx, cfg.(rest.Config), rt)
case "rclone": case "rclone":
return rclone.Create(globalOptions.ctx, cfg.(rclone.Config)) return rclone.Create(ctx, cfg.(rclone.Config))
} }
debug.Log("invalid repository scheme: %v", s) debug.Log("invalid repository scheme: %v", s)

9
cmd/restic/integration_fuse_test.go
View file

@ -4,6 +4,7 @@
package main package main
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@ -57,7 +58,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
opts := MountOptions{ opts := MountOptions{
TimeTemplate: time.RFC3339, TimeTemplate: time.RFC3339,
} }
rtest.OK(t, runMount(opts, gopts, []string{dir})) rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir}))
} }
func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) { func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) {
@ -119,7 +120,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
} }
for _, id := range snapshotIDs { for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(global.ctx, repo, id) snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
rtest.OK(t, err) rtest.OK(t, err)
ts := snapshot.Time.Format(time.RFC3339) ts := snapshot.Time.Format(time.RFC3339)
@ -160,7 +161,7 @@ func TestMount(t *testing.T) {
testRunInit(t, env.gopts) testRunInit(t, env.gopts)
repo, err := OpenRepository(env.gopts) repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err) rtest.OK(t, err)
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0) checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
@ -205,7 +206,7 @@ func TestMountSameTimestamps(t *testing.T) {
rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz")) rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
repo, err := OpenRepository(env.gopts) repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err) rtest.OK(t, err)
ids := []restic.ID{ ids := []restic.ID{

2
cmd/restic/integration_helpers_test.go
View file

@ -2,7 +2,6 @@ package main
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
@ -193,7 +192,6 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
Repo: env.repo, Repo: env.repo,
Quiet: true, Quiet: true,
CacheDir: env.cache, CacheDir: env.cache,
ctx: context.Background(),
password: rtest.TestPassword, password: rtest.TestPassword,
stdout: os.Stdout, stdout: os.Stdout,
stderr: os.Stderr, stderr: os.Stderr,

112
cmd/restic/integration_test.go
View file

@ -52,12 +52,12 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
restic.TestDisableCheckPolynomial(t) restic.TestDisableCheckPolynomial(t)
restic.TestSetLockTimeout(t, 0) restic.TestSetLockTimeout(t, 0)
rtest.OK(t, runInit(InitOptions{}, opts, nil)) rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
t.Logf("repository initialized at %v", opts.Repo) t.Logf("repository initialized at %v", opts.Repo)
} }
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error { func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
ctx, cancel := context.WithCancel(gopts.ctx) ctx, cancel := context.WithCancel(context.TODO())
defer cancel() defer cancel()
var wg errgroup.Group var wg errgroup.Group
@ -71,7 +71,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts
defer cleanup() defer cleanup()
} }
backupErr := runBackup(opts, gopts, term, target) backupErr := runBackup(ctx, opts, gopts, term, target)
cancel() cancel()
@ -95,7 +95,7 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
globalOptions.stdout = os.Stdout globalOptions.stdout = os.Stdout
}() }()
rtest.OK(t, runList(cmdList, opts, []string{tpe})) rtest.OK(t, runList(context.TODO(), cmdList, opts, []string{tpe}))
return parseIDsFromReader(t, buf) return parseIDsFromReader(t, buf)
} }
@ -112,7 +112,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [
}, },
} }
rtest.OK(t, runRestore(opts, gopts, []string{"latest"})) rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"}))
} }
func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) { func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
@ -121,7 +121,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Exclude: excludes, Exclude: excludes,
} }
rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()})) rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
} }
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) { func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
@ -130,11 +130,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Include: includes, Include: includes,
} }
rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()})) rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
} }
func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error { func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
err := runRestore(opts, gopts, []string{snapshotID}) err := runRestore(context.TODO(), opts, gopts, []string{snapshotID})
return err return err
} }
@ -144,7 +144,7 @@ func testRunCheck(t testing.TB, gopts GlobalOptions) {
ReadData: true, ReadData: true,
CheckUnused: true, CheckUnused: true,
} }
rtest.OK(t, runCheck(opts, gopts, nil)) rtest.OK(t, runCheck(context.TODO(), opts, gopts, nil))
} }
func testRunCheckOutput(gopts GlobalOptions) (string, error) { func testRunCheckOutput(gopts GlobalOptions) (string, error) {
@ -159,7 +159,7 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) {
ReadData: true, ReadData: true,
} }
err := runCheck(opts, gopts, nil) err := runCheck(context.TODO(), opts, gopts, nil)
return buf.String(), err return buf.String(), err
} }
@ -177,7 +177,7 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
opts := DiffOptions{ opts := DiffOptions{
ShowMetadata: false, ShowMetadata: false,
} }
err := runDiff(opts, gopts, []string{firstSnapshotID, secondSnapshotID}) err := runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
return buf.String(), err return buf.String(), err
} }
@ -187,7 +187,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
globalOptions.stdout = os.Stdout globalOptions.stdout = os.Stdout
}() }()
rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, gopts)) rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts))
} }
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
@ -202,7 +202,7 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
opts := LsOptions{} opts := LsOptions{}
rtest.OK(t, runLs(opts, gopts, []string{snapshotID})) rtest.OK(t, runLs(context.TODO(), opts, gopts, []string{snapshotID}))
return strings.Split(buf.String(), "\n") return strings.Split(buf.String(), "\n")
} }
@ -218,7 +218,7 @@ func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern strin
opts := FindOptions{} opts := FindOptions{}
rtest.OK(t, runFind(opts, gopts, []string{pattern})) rtest.OK(t, runFind(context.TODO(), opts, gopts, []string{pattern}))
return buf.Bytes() return buf.Bytes()
} }
@ -234,7 +234,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
opts := SnapshotOptions{} opts := SnapshotOptions{}
rtest.OK(t, runSnapshots(opts, globalOptions, []string{})) rtest.OK(t, runSnapshots(context.TODO(), opts, globalOptions, []string{}))
snapshots := []Snapshot{} snapshots := []Snapshot{}
rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots)) rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
@ -251,7 +251,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) { func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
opts := ForgetOptions{} opts := ForgetOptions{}
rtest.OK(t, runForget(opts, gopts, args)) rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
} }
func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
@ -269,7 +269,7 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
Last: 1, Last: 1,
} }
rtest.OK(t, runForget(opts, gopts, args)) rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
var forgets []*ForgetGroup var forgets []*ForgetGroup
rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets)) rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets))
@ -288,7 +288,7 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
defer func() { defer func() {
gopts.backendTestHook = oldHook gopts.backendTestHook = oldHook
}() }()
rtest.OK(t, runPrune(opts, gopts)) rtest.OK(t, runPrune(context.TODO(), opts, gopts))
} }
func testSetupBackupData(t testing.TB, env *testEnvironment) string { func testSetupBackupData(t testing.TB, env *testEnvironment) string {
@ -437,11 +437,11 @@ func TestBackupNonExistingFile(t *testing.T) {
} }
func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) { func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) {
r, err := OpenRepository(gopts) r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err) rtest.OK(t, err)
// Get all tree packs // Get all tree packs
rtest.OK(t, r.LoadIndex(gopts.ctx)) rtest.OK(t, r.LoadIndex(context.TODO()))
treePacks := restic.NewIDSet() treePacks := restic.NewIDSet()
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
@ -451,11 +451,11 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem
}) })
// remove all packs containing data blobs // remove all packs containing data blobs
rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error { rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
if treePacks.Has(id) != removeTreePacks || keep.Has(id) { if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
return nil return nil
} }
return r.Backend().Remove(gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}) return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
})) }))
} }
@ -479,7 +479,7 @@ func TestBackupSelfHealing(t *testing.T) {
testRunRebuildIndex(t, env.gopts) testRunRebuildIndex(t, env.gopts)
// now the repo is also missing the data blob in the index; check should report this // now the repo is also missing the data blob in the index; check should report this
rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil,
"check should have reported an error") "check should have reported an error")
// second backup should report an error but "heal" this situation // second backup should report an error but "heal" this situation
@ -502,9 +502,9 @@ func TestBackupTreeLoadError(t *testing.T) {
// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory // Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts) testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
r, err := OpenRepository(env.gopts) r, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err) rtest.OK(t, err)
rtest.OK(t, r.LoadIndex(env.gopts.ctx)) rtest.OK(t, r.LoadIndex(context.TODO()))
treePacks := restic.NewIDSet() treePacks := restic.NewIDSet()
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob { if pb.Type == restic.TreeBlob {
@ -517,11 +517,11 @@ func TestBackupTreeLoadError(t *testing.T) {
// delete the subdirectory pack first // delete the subdirectory pack first
for id := range treePacks { for id := range treePacks {
rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()})) rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
} }
testRunRebuildIndex(t, env.gopts) testRunRebuildIndex(t, env.gopts)
// now the repo is missing the tree blob in the index; check should report this // now the repo is missing the tree blob in the index; check should report this
rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
// second backup should report an error but "heal" this situation // second backup should report an error but "heal" this situation
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory") rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
@ -531,7 +531,7 @@ func TestBackupTreeLoadError(t *testing.T) {
removePacksExcept(env.gopts, t, restic.NewIDSet(), true) removePacksExcept(env.gopts, t, restic.NewIDSet(), true)
testRunRebuildIndex(t, env.gopts) testRunRebuildIndex(t, env.gopts)
// now the repo is also missing the data blob in the index; check should report this // now the repo is also missing the data blob in the index; check should report this
rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
// second backup should report an error but "heal" this situation // second backup should report an error but "heal" this situation
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
rtest.Assert(t, err != nil, "backup should have reported an error") rtest.Assert(t, err != nil, "backup should have reported an error")
@ -761,7 +761,7 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
}, },
} }
rtest.OK(t, runCopy(copyOpts, gopts, nil)) rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
} }
func TestCopy(t *testing.T) { func TestCopy(t *testing.T) {
@ -903,15 +903,15 @@ func TestInitCopyChunkerParams(t *testing.T) {
password: env2.gopts.password, password: env2.gopts.password,
}, },
} }
rtest.Assert(t, runInit(initOpts, env.gopts, nil) != nil, "expected invalid init options to fail") rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
initOpts.CopyChunkerParameters = true initOpts.CopyChunkerParameters = true
rtest.OK(t, runInit(initOpts, env.gopts, nil)) rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil))
repo, err := OpenRepository(env.gopts) repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err) rtest.OK(t, err)
otherRepo, err := OpenRepository(env2.gopts) otherRepo, err := OpenRepository(context.TODO(), env2.gopts)
rtest.OK(t, err) rtest.OK(t, err)
rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial, rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,
@ -920,7 +920,7 @@ func TestInitCopyChunkerParams(t *testing.T) {
} }
func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) { func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
rtest.OK(t, runTag(opts, gopts, []string{})) rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{}))
} }
func TestTag(t *testing.T) { func TestTag(t *testing.T) {
@ -1012,7 +1012,7 @@ func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
globalOptions.stdout = os.Stdout globalOptions.stdout = os.Stdout
}() }()
rtest.OK(t, runKey(gopts, []string{"list"})) rtest.OK(t, runKey(context.TODO(), gopts, []string{"list"}))
scanner := bufio.NewScanner(buf) scanner := bufio.NewScanner(buf)
exp := regexp.MustCompile(`^ ([a-f0-9]+) `) exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
@ -1033,7 +1033,7 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
testKeyNewPassword = "" testKeyNewPassword = ""
}() }()
rtest.OK(t, runKey(gopts, []string{"add"})) rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
} }
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
@ -1047,11 +1047,11 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"})) rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
t.Log("adding key for john@example.com") t.Log("adding key for john@example.com")
rtest.OK(t, runKey(gopts, []string{"add"})) rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
repo, err := OpenRepository(gopts) repo, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err) rtest.OK(t, err)
key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "") key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "")
rtest.OK(t, err) rtest.OK(t, err)
rtest.Equals(t, "john", key.Username) rtest.Equals(t, "john", key.Username)
@ -1064,13 +1064,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
testKeyNewPassword = "" testKeyNewPassword = ""
}() }()
rtest.OK(t, runKey(gopts, []string{"passwd"})) rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
} }
func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) { func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
t.Logf("remove %d keys: %q\n", len(IDs), IDs) t.Logf("remove %d keys: %q\n", len(IDs), IDs)
for _, id := range IDs { for _, id := range IDs {
rtest.OK(t, runKey(gopts, []string{"remove", id})) rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
} }
} }
@ -1100,7 +1100,7 @@ func TestKeyAddRemove(t *testing.T) {
env.gopts.password = passwordList[len(passwordList)-1] env.gopts.password = passwordList[len(passwordList)-1]
t.Logf("testing access with last password %q\n", env.gopts.password) t.Logf("testing access with last password %q\n", env.gopts.password)
rtest.OK(t, runKey(env.gopts, []string{"list"})) rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
testRunKeyAddNewKeyUserHost(t, env.gopts) testRunKeyAddNewKeyUserHost(t, env.gopts)
@ -1128,16 +1128,16 @@ func TestKeyProblems(t *testing.T) {
testKeyNewPassword = "" testKeyNewPassword = ""
}() }()
err := runKey(env.gopts, []string{"passwd"}) err := runKey(context.TODO(), env.gopts, []string{"passwd"})
t.Log(err) t.Log(err)
rtest.Assert(t, err != nil, "expected passwd change to fail") rtest.Assert(t, err != nil, "expected passwd change to fail")
err = runKey(env.gopts, []string{"add"}) err = runKey(context.TODO(), env.gopts, []string{"add"})
t.Log(err) t.Log(err)
rtest.Assert(t, err != nil, "expected key adding to fail") rtest.Assert(t, err != nil, "expected key adding to fail")
t.Logf("testing access with initial password %q\n", env.gopts.password) t.Logf("testing access with initial password %q\n", env.gopts.password)
rtest.OK(t, runKey(env.gopts, []string{"list"})) rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
} }
@ -1549,7 +1549,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
return &appendOnlyBackend{r}, nil return &appendOnlyBackend{r}, nil
} }
err := runRebuildIndex(RebuildIndexOptions{}, env.gopts) err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)
if err == nil { if err == nil {
t.Error("expected rebuildIndex to fail") t.Error("expected rebuildIndex to fail")
} }
@ -1645,18 +1645,18 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
testRunForgetJSON(t, env.gopts) testRunForgetJSON(t, env.gopts)
testRunForget(t, env.gopts, firstSnapshot[0].String()) testRunForget(t, env.gopts, firstSnapshot[0].String())
testRunPrune(t, env.gopts, pruneOpts) testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, runCheck(checkOpts, env.gopts, nil)) rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
} }
var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"} var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet { func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
r, err := OpenRepository(gopts) r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err) rtest.OK(t, err)
packs := restic.NewIDSet() packs := restic.NewIDSet()
rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error { rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
packs.Insert(id) packs.Insert(id)
return nil return nil
})) }))
@ -1697,7 +1697,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
env.gopts.backendTestHook = oldHook env.gopts.backendTestHook = oldHook
}() }()
// prune should fail // prune should fail
rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing, rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
"prune should have reported index not complete error") "prune should have reported index not complete error")
} }
@ -1769,7 +1769,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
if checkOK { if checkOK {
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
} else { } else {
rtest.Assert(t, runCheck(optionsCheck, env.gopts, nil) != nil, rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
"check should have reported an error") "check should have reported an error")
} }
@ -1777,7 +1777,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
testRunPrune(t, env.gopts, optionsPrune) testRunPrune(t, env.gopts, optionsPrune)
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
} else { } else {
rtest.Assert(t, runPrune(optionsPrune, env.gopts) != nil, rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
"prune should have reported an error") "prune should have reported an error")
} }
} }
@ -1848,10 +1848,10 @@ func TestListOnce(t *testing.T) {
testRunForgetJSON(t, env.gopts) testRunForgetJSON(t, env.gopts)
testRunForget(t, env.gopts, firstSnapshot[0].String()) testRunForget(t, env.gopts, firstSnapshot[0].String())
testRunPrune(t, env.gopts, pruneOpts) testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, runCheck(checkOpts, env.gopts, nil)) rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, env.gopts)) rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts))
rtest.OK(t, runRebuildIndex(RebuildIndexOptions{ReadAllPacks: true}, env.gopts)) rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
} }
func TestHardLink(t *testing.T) { func TestHardLink(t *testing.T) {
@ -2204,7 +2204,7 @@ func TestFindListOnce(t *testing.T) {
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts) testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...) thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...)
repo, err := OpenRepository(env.gopts) repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err) rtest.OK(t, err)
snapshotIDs := restic.NewIDSet() snapshotIDs := restic.NewIDSet()

168
cmd/restic/lock.go
View file

@ -7,27 +7,31 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
var globalLocks struct { type lockContext struct {
locks []*restic.Lock cancel context.CancelFunc
cancelRefresh chan struct{}
refreshWG sync.WaitGroup refreshWG sync.WaitGroup
}
var globalLocks struct {
locks map[*restic.Lock]*lockContext
sync.Mutex sync.Mutex
sync.Once sync.Once
} }
func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) { func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
return lockRepository(ctx, repo, false) return lockRepository(ctx, repo, false)
} }
func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) { func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
return lockRepository(ctx, repo, true) return lockRepository(ctx, repo, true)
} }
func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) { // lockRepository wraps the ctx such that it is cancelled when the repository is unlocked
// cancelling the original context also stops the lock refresh
func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) {
// make sure that a repository is unlocked properly and after cancel() was // make sure that a repository is unlocked properly and after cancel() was
// called by the cleanup handler in global.go // called by the cleanup handler in global.go
globalLocks.Do(func() { globalLocks.Do(func() {
@ -41,53 +45,114 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive
lock, err := lockFn(ctx, repo) lock, err := lockFn(ctx, repo)
if err != nil { if err != nil {
return nil, errors.WithMessage(err, "unable to create lock in backend") return nil, ctx, errors.WithMessage(err, "unable to create lock in backend")
} }
debug.Log("create lock %p (exclusive %v)", lock, exclusive) debug.Log("create lock %p (exclusive %v)", lock, exclusive)
globalLocks.Lock() ctx, cancel := context.WithCancel(ctx)
if globalLocks.cancelRefresh == nil { lockInfo := &lockContext{
debug.Log("start goroutine for lock refresh") cancel: cancel,
globalLocks.cancelRefresh = make(chan struct{})
globalLocks.refreshWG = sync.WaitGroup{}
globalLocks.refreshWG.Add(1)
go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh)
} }
lockInfo.refreshWG.Add(2)
refreshChan := make(chan struct{})
globalLocks.locks = append(globalLocks.locks, lock) globalLocks.Lock()
globalLocks.locks[lock] = lockInfo
go refreshLocks(ctx, lock, lockInfo, refreshChan)
go monitorLockRefresh(ctx, lock, lockInfo, refreshChan)
globalLocks.Unlock() globalLocks.Unlock()
return lock, err return lock, ctx, err
} }
var refreshInterval = 5 * time.Minute var refreshInterval = 5 * time.Minute
func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) { // consider a lock refresh failed a bit before the lock actually becomes stale
debug.Log("start") // the difference allows to compensate for a small time drift between clients.
defer func() { var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2
wg.Done()
globalLocks.Lock()
globalLocks.cancelRefresh = nil
globalLocks.Unlock()
}()
func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed chan<- struct{}) {
debug.Log("start")
ticker := time.NewTicker(refreshInterval) ticker := time.NewTicker(refreshInterval)
lastRefresh := lock.Time
defer func() {
ticker.Stop()
// ensure that the context was cancelled before removing the lock
lockInfo.cancel()
// remove the lock from the repo
debug.Log("unlocking repository with lock %v", lock)
if err := lock.Unlock(); err != nil {
debug.Log("error while unlocking: %v", err)
Warnf("error while unlocking: %v", err)
}
lockInfo.refreshWG.Done()
}()
for { for {
select { select {
case <-done: case <-ctx.Done():
debug.Log("terminate") debug.Log("terminate")
return return
case <-ticker.C: case <-ticker.C:
if time.Since(lastRefresh) > refreshabilityTimeout {
// the lock is too old, wait until the expiry monitor cancels the context
continue
}
debug.Log("refreshing locks") debug.Log("refreshing locks")
globalLocks.Lock()
for _, lock := range globalLocks.locks {
err := lock.Refresh(context.TODO()) err := lock.Refresh(context.TODO())
if err != nil { if err != nil {
Warnf("unable to refresh lock: %v\n", err) Warnf("unable to refresh lock: %v\n", err)
} else {
lastRefresh = lock.Time
// inform monitor gorountine about successful refresh
select {
case <-ctx.Done():
case refreshed <- struct{}{}:
} }
} }
globalLocks.Unlock() }
}
}
func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed <-chan struct{}) {
// time.Now() might use a monotonic timer which is paused during standby
// convert to unix time to ensure we compare real time values
lastRefresh := time.Now().Unix()
pollDuration := 1 * time.Second
if refreshInterval < pollDuration {
// require for TestLockFailedRefresh
pollDuration = refreshInterval / 5
}
// timers are paused during standby, which is a problem as the refresh timeout
// _must_ expire if the host was too long in standby. Thus fall back to periodic checks
// https://github.com/golang/go/issues/35012
timer := time.NewTimer(pollDuration)
defer func() {
timer.Stop()
lockInfo.cancel()
lockInfo.refreshWG.Done()
}()
for {
select {
case <-ctx.Done():
debug.Log("terminate expiry monitoring")
return
case <-refreshed:
lastRefresh = time.Now().Unix()
case <-timer.C:
if float64(time.Now().Unix()-lastRefresh) < refreshInterval.Seconds() {
// restart timer
timer.Reset(pollDuration)
continue
}
Warnf("Fatal: failed to refresh lock in time\n")
return
} }
} }
} }
@ -98,40 +163,35 @@ func unlockRepo(lock *restic.Lock) {
} }
globalLocks.Lock() globalLocks.Lock()
defer globalLocks.Unlock() lockInfo, exists := globalLocks.locks[lock]
delete(globalLocks.locks, lock)
for i := 0; i < len(globalLocks.locks); i++ { globalLocks.Unlock()
if lock == globalLocks.locks[i] {
// remove the lock from the repo
debug.Log("unlocking repository with lock %v", lock)
if err := lock.Unlock(); err != nil {
debug.Log("error while unlocking: %v", err)
Warnf("error while unlocking: %v", err)
return
}
// remove the lock from the list of locks
globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...)
return
}
}
if !exists {
debug.Log("unable to find lock %v in the global list of locks, ignoring", lock) debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
return
}
lockInfo.cancel()
lockInfo.refreshWG.Wait()
} }
func unlockAll(code int) (int, error) { func unlockAll(code int) (int, error) {
globalLocks.Lock() globalLocks.Lock()
defer globalLocks.Unlock() locks := globalLocks.locks
debug.Log("unlocking %d locks", len(globalLocks.locks)) debug.Log("unlocking %d locks", len(globalLocks.locks))
for _, lock := range globalLocks.locks { for _, lockInfo := range globalLocks.locks {
if err := lock.Unlock(); err != nil { lockInfo.cancel()
debug.Log("error while unlocking: %v", err)
return code, err
} }
debug.Log("successfully removed lock") globalLocks.locks = make(map[*restic.Lock]*lockContext)
globalLocks.Unlock()
for _, lockInfo := range locks {
lockInfo.refreshWG.Wait()
} }
globalLocks.locks = globalLocks.locks[:0]
return code, nil return code, nil
} }
func init() {
globalLocks.locks = make(map[*restic.Lock]*lockContext)
}

130
cmd/restic/lock_test.go Normal file
View file

@ -0,0 +1,130 @@
package main
import (
"context"
"fmt"
"testing"
"time"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) {
env, cleanup := withTestEnvironment(t)
if wrapper != nil {
env.gopts.backendTestHook = wrapper
}
testRunInit(t, env.gopts)
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
return repo, cleanup, env
}
func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) {
lock, wrappedCtx, err := lockRepo(ctx, repo)
rtest.OK(t, err)
rtest.OK(t, wrappedCtx.Err())
if lock.Stale() {
t.Fatal("lock returned stale lock")
}
return lock, wrappedCtx
}
func TestLock(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, nil)
defer cleanup()
lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
unlockRepo(lock)
if wrappedCtx.Err() == nil {
t.Fatal("unlock did not cancel context")
}
}
func TestLockCancel(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, nil)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
lock, wrappedCtx := checkedLockRepo(ctx, t, repo)
cancel()
if wrappedCtx.Err() == nil {
t.Fatal("canceled parent context did not cancel context")
}
// unlockRepo should not crash
unlockRepo(lock)
}
func TestLockUnlockAll(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, nil)
defer cleanup()
lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
_, err := unlockAll(0)
rtest.OK(t, err)
if wrappedCtx.Err() == nil {
t.Fatal("canceled parent context did not cancel context")
}
// unlockRepo should not crash
unlockRepo(lock)
}
func TestLockConflict(t *testing.T) {
repo, cleanup, env := openTestRepo(t, nil)
defer cleanup()
repo2, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
lock, _, err := lockRepoExclusive(context.Background(), repo)
rtest.OK(t, err)
defer unlockRepo(lock)
_, _, err = lockRepo(context.Background(), repo2)
if err == nil {
t.Fatal("second lock should have failed")
}
}
type writeOnceBackend struct {
restic.Backend
written bool
}
func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if b.written {
return fmt.Errorf("fail after first write")
}
b.written = true
return b.Backend.Save(ctx, h, rd)
}
func TestLockFailedRefresh(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
return &writeOnceBackend{Backend: r}, nil
})
defer cleanup()
// reduce locking intervals to be suitable for testing
ri, rt := refreshInterval, refreshabilityTimeout
refreshInterval = 20 * time.Millisecond
refreshabilityTimeout = 100 * time.Millisecond
defer func() {
refreshInterval, refreshabilityTimeout = ri, rt
}()
lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
select {
case <-wrappedCtx.Done():
// expected lock refresh failure
case <-time.After(time.Second):
t.Fatal("failed lock refresh did not cause context cancellation")
}
// unlockRepo should not crash
unlockRepo(lock)
}

2
cmd/restic/main.go
View file

@ -95,7 +95,7 @@ func main() {
debug.Log("main %#v", os.Args) debug.Log("main %#v", os.Args)
debug.Log("restic %s compiled with %v on %v/%v", debug.Log("restic %s compiled with %v on %v/%v",
version, runtime.Version(), runtime.GOOS, runtime.GOARCH) version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
err := cmdRoot.Execute() err := cmdRoot.ExecuteContext(internalGlobalCtx)
switch { switch {
case restic.IsAlreadyLocked(err): case restic.IsAlreadyLocked(err):

24
internal/restic/lock.go
View file

@ -137,11 +137,15 @@ func (l *Lock) fillUserInfo() error {
// non-exclusive lock is to be created, an error is only returned when an // non-exclusive lock is to be created, an error is only returned when an
// exclusive lock is found. // exclusive lock is found.
func (l *Lock) checkForOtherLocks(ctx context.Context) error { func (l *Lock) checkForOtherLocks(ctx context.Context) error {
return ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error { var err error
// retry locking a few times
for i := 0; i < 3; i++ {
err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error {
if err != nil { if err != nil {
// ignore locks that cannot be loaded // if we cannot load a lock then it is unclear whether it can be ignored
// it could either be invalid or just unreadable due to network/permission problems
debug.Log("ignore lock %v: %v", id, err) debug.Log("ignore lock %v: %v", id, err)
return nil return errors.Fatal(err.Error())
} }
if l.Exclusive { if l.Exclusive {
@ -154,6 +158,16 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error {
return nil return nil
}) })
// no lock detected
if err == nil {
return nil
}
// lock conflicts are permanent
if _, ok := err.(*alreadyLockedError); ok {
return err
}
}
return err
} }
// createLock acquires the lock by creating a file in the repository. // createLock acquires the lock by creating a file in the repository.
@ -175,14 +189,14 @@ func (l *Lock) Unlock() error {
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()})
} }
var staleTimeout = 30 * time.Minute var StaleLockTimeout = 30 * time.Minute
// Stale returns true if the lock is stale. A lock is stale if the timestamp is // Stale returns true if the lock is stale. A lock is stale if the timestamp is
// older than 30 minutes or if it was created on the current machine and the // older than 30 minutes or if it was created on the current machine and the
// process isn't alive any more. // process isn't alive any more.
func (l *Lock) Stale() bool { func (l *Lock) Stale() bool {
debug.Log("testing if lock %v for process %d is stale", l, l.PID) debug.Log("testing if lock %v for process %d is stale", l, l.PID)
if time.Since(l.Time) > staleTimeout { if time.Since(l.Time) > StaleLockTimeout {
debug.Log("lock is stale, timestamp is too old: %v\n", l.Time) debug.Log("lock is stale, timestamp is too old: %v\n", l.Time)
return true return true
} }

28
internal/restic/lock_test.go
View file

@ -2,10 +2,13 @@ package restic_test
import ( import (
"context" "context"
"fmt"
"io"
"os" "os"
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
@ -49,6 +52,31 @@ func TestMultipleLock(t *testing.T) {
rtest.OK(t, lock2.Unlock()) rtest.OK(t, lock2.Unlock())
} }
type failLockLoadingBackend struct {
restic.Backend
}
func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if h.Type == restic.LockFile {
return fmt.Errorf("error loading lock")
}
return be.Backend.Load(ctx, h, length, offset, fn)
}
func TestMultipleLockFailure(t *testing.T) {
be := &failLockLoadingBackend{Backend: mem.New()}
repo, cleanup := repository.TestRepositoryWithBackend(t, be, 0)
defer cleanup()
lock1, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
_, err = restic.NewLock(context.TODO(), repo)
rtest.Assert(t, err != nil, "unreadable lock file did not result in an error")
rtest.OK(t, lock1.Unlock())
}
func TestLockExclusive(t *testing.T) { func TestLockExclusive(t *testing.T) {
repo, cleanup := repository.TestRepository(t) repo, cleanup := repository.TestRepository(t)
defer cleanup() defer cleanup()