From d15e693045918bd92dbf74a958127242224378bb Mon Sep 17 00:00:00 2001 From: Shivashis Padhi Date: Sun, 11 Aug 2024 22:25:21 +0200 Subject: [PATCH 01/52] restore: Add progress bar to 'restore --verify' --- changelog/unreleased/issue-4795 | 7 +++ cmd/restic/cmd_restore.go | 5 +- internal/restorer/restorer.go | 25 +++++--- internal/restorer/restorer_test.go | 71 ++++++++++++++-------- internal/restorer/restorer_unix_test.go | 10 +-- internal/restorer/restorer_windows_test.go | 6 +- 6 files changed, 83 insertions(+), 41 deletions(-) create mode 100644 changelog/unreleased/issue-4795 diff --git a/changelog/unreleased/issue-4795 b/changelog/unreleased/issue-4795 new file mode 100644 index 000000000..084335f51 --- /dev/null +++ b/changelog/unreleased/issue-4795 @@ -0,0 +1,7 @@ +Enhancement: `restore --verify` shows progress with a progress bar + +If restore command was run with `--verify` restic didn't show any progress indication, now it shows a progress bar while 'verification' is running. +The progress bar is text only for now and doesn't respect `--json` flag. + +https://github.com/restic/restic/issues/4795 +https://github.com/restic/restic/pull/4989 diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index eda608802..d71cb7683 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -220,7 +220,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target) } - err = res.RestoreTo(ctx, opts.Target) + countRestoredFiles, err := res.RestoreTo(ctx, opts.Target) if err != nil { return err } @@ -237,7 +237,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } var count int t0 := time.Now() - count, err = res.VerifyFiles(ctx, opts.Target) + bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON && stdoutIsTerminal(), 0, "files verified", term) + count, err = res.VerifyFiles(ctx, opts.Target, countRestoredFiles, bar) if err != nil { return err } diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 00da4e18e..0e30b82f8 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -12,6 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" @@ -333,12 +334,13 @@ func (res *Restorer) ensureDir(target string) error { // RestoreTo creates the directories and files in the snapshot below dst. // Before an item is created, res.Filter is called. -func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { +func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) { + restoredFileCount := uint64(0) var err error if !filepath.IsAbs(dst) { dst, err = filepath.Abs(dst) if err != nil { - return errors.Wrap(err, "Abs") + return restoredFileCount, errors.Wrap(err, "Abs") } } @@ -346,7 +348,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { // ensure that the target directory exists and is actually a directory // Using ensureDir is too aggressive here as it also removes unexpected files if err := fs.MkdirAll(dst, 0700); err != nil { - return fmt.Errorf("cannot create target directory: %w", err) + return restoredFileCount, fmt.Errorf("cannot create target directory: %w", err) } } @@ -406,19 +408,22 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } } res.trackFile(location, updateMetadataOnly) + if !updateMetadataOnly { + restoredFileCount++ + } return nil }) return err }, }) if err != nil { - return err + return 0, err } if !res.opts.DryRun { err = filerestorer.restoreFiles(ctx) if err != nil { - return err + return 0, err } } @@ -466,7 +471,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return err }, }) - return err + return restoredFileCount, err } func (res *Restorer) removeUnexpectedFiles(ctx context.Context, target, location string, expectedFilenames []string) error { @@ -587,7 +592,7 @@ const nVerifyWorkers = 8 // have been successfully written to dst. It stops when it encounters an // error. It returns that error and the number of files it has successfully // verified. -func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { +func (res *Restorer) VerifyFiles(ctx context.Context, dst string, countRestoredFiles uint64, p *progress.Counter) (int, error) { type mustCheck struct { node *restic.Node path string @@ -598,6 +603,11 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { work = make(chan mustCheck, 2*nVerifyWorkers) ) + if p != nil { + p.SetMax(countRestoredFiles) + defer p.Done() + } + g, ctx := errgroup.WithContext(ctx) // Traverse tree and send jobs to work. @@ -632,6 +642,7 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { if err != nil || ctx.Err() != nil { break } + p.Add(1) atomic.AddUint64(&nchecked, 1) } return err diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index a6de50556..7d4895068 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -22,6 +22,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" ) @@ -403,13 +404,13 @@ func TestRestorer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) if err != nil { t.Fatal(err) } if len(test.ErrorsMust)+len(test.ErrorsMay) == 0 { - _, err = res.VerifyFiles(ctx, tempdir) + _, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.OK(t, err) } @@ -501,13 +502,18 @@ func TestRestorerRelative(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, "restore") + countRestoredFiles, err := res.RestoreTo(ctx, "restore") if err != nil { t.Fatal(err) } - nverified, err := res.VerifyFiles(ctx, "restore") + p := progress.NewCounter(time.Second, countRestoredFiles, func(value uint64, total uint64, runtime time.Duration, final bool) {}) + defer p.Done() + nverified, err := res.VerifyFiles(ctx, "restore", countRestoredFiles, p) rtest.OK(t, err) rtest.Equals(t, len(test.Files), nverified) + counterValue, maxValue := p.Get() + rtest.Equals(t, counterValue, uint64(2)) + rtest.Equals(t, maxValue, uint64(2)) for filename, err := range errors { t.Errorf("unexpected error for %v found: %v", filename, err) @@ -524,6 +530,13 @@ func TestRestorerRelative(t *testing.T) { t.Errorf("file %v has wrong content: want %q, got %q", filename, content, data) } } + + // verify that restoring the same snapshot again results in countRestoredFiles == 0 + countRestoredFiles, err = res.RestoreTo(ctx, "restore") + if err != nil { + t.Fatal(err) + } + rtest.Equals(t, uint64(0), countRestoredFiles) }) } } @@ -835,7 +848,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) var testPatterns = []struct { @@ -872,9 +885,9 @@ func TestVerifyCancel(t *testing.T) { tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - rtest.OK(t, res.RestoreTo(ctx, tempdir)) - err := os.WriteFile(filepath.Join(tempdir, "foo"), []byte("bar"), 0644) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + err = os.WriteFile(filepath.Join(tempdir, "foo"), []byte("bar"), 0644) rtest.OK(t, err) var errs []error @@ -883,7 +896,7 @@ func TestVerifyCancel(t *testing.T) { return err } - nverified, err := res.VerifyFiles(ctx, tempdir) + nverified, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.Equals(t, 0, nverified) rtest.Assert(t, err != nil, "nil error from VerifyFiles") rtest.Equals(t, 1, len(errs)) @@ -915,7 +928,7 @@ func TestRestorerSparseFiles(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err = res.RestoreTo(ctx, tempdir) + _, err = res.RestoreTo(ctx, tempdir) rtest.OK(t, err) filename := filepath.Join(tempdir, "zeros") @@ -952,15 +965,17 @@ func saveSnapshotsAndOverwrite(t *testing.T, baseSnapshot Snapshot, overwriteSna t.Logf("base snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, baseOptions) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) // overwrite snapshot sn, id = saveSnapshot(t, repo, overwriteSnapshot, noopGetGenericAttributes) t.Logf("overwrite snapshot saved as %v", id.Str()) res = NewRestorer(repo, sn, overwriteOptions) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) - _, err := res.VerifyFiles(ctx, tempdir) + _, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.OK(t, err) return tempdir @@ -1241,8 +1256,9 @@ func TestRestoreModified(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{Overwrite: OverwriteIfChanged}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) - n, err := res.VerifyFiles(ctx, tempdir) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + n, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.OK(t, err) rtest.Equals(t, 2, n, "unexpected number of verified files") } @@ -1267,7 +1283,8 @@ func TestRestoreIfChanged(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) // modify file but maintain size and timestamp path := filepath.Join(tempdir, "foo") @@ -1286,7 +1303,8 @@ func TestRestoreIfChanged(t *testing.T) { for _, overwrite := range []OverwriteBehavior{OverwriteIfChanged, OverwriteAlways} { res = NewRestorer(repo, sn, Options{Overwrite: overwrite}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) data, err := os.ReadFile(path) rtest.OK(t, err) if overwrite == OverwriteAlways { @@ -1322,9 +1340,10 @@ func TestRestoreDryRun(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{DryRun: true}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) - _, err := os.Stat(tempdir) + _, err = os.Stat(tempdir) rtest.Assert(t, errors.Is(err, os.ErrNotExist), "expected no file to be created, got %v", err) } @@ -1348,7 +1367,8 @@ func TestRestoreDryRunDelete(t *testing.T) { sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, Options{DryRun: true, Delete: true}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err = res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) _, err = os.Stat(tempfile) rtest.Assert(t, err == nil, "expected file to still exist, got error %v", err) @@ -1466,14 +1486,14 @@ func TestRestoreDelete(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) res = NewRestorer(repo, deleteSn, Options{Delete: true}) if test.selectFilter != nil { res.SelectFilter = test.selectFilter } - err = res.RestoreTo(ctx, tempdir) + _, err = res.RestoreTo(ctx, tempdir) rtest.OK(t, err) for fn, shouldExist := range test.fileState { @@ -1506,7 +1526,7 @@ func TestRestoreToFile(t *testing.T) { sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, Options{}) - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.Assert(t, strings.Contains(err.Error(), "cannot create target directory"), "unexpected error %v", err) } @@ -1538,7 +1558,8 @@ func TestRestorerLongPath(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rtest.OK(t, res.RestoreTo(ctx, tmp)) - _, err = res.VerifyFiles(ctx, tmp) + countRestoredFiles, err := res.RestoreTo(ctx, tmp) + rtest.OK(t, err) + _, err = res.VerifyFiles(ctx, tmp, countRestoredFiles, nil) rtest.OK(t, err) } diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index 27d990af4..c4e8149b2 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -37,7 +37,7 @@ func TestRestorerRestoreEmptyHardlinkedFields(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) f1, err := os.Stat(filepath.Join(tempdir, "dirtest/file1")) @@ -96,7 +96,7 @@ func testRestorerProgressBar(t *testing.T, dryRun bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) progress.Finish() @@ -126,7 +126,8 @@ func TestRestorePermissions(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) for _, overwrite := range []OverwriteBehavior{OverwriteIfChanged, OverwriteAlways} { // tamper with permissions @@ -134,7 +135,8 @@ func TestRestorePermissions(t *testing.T) { rtest.OK(t, os.Chmod(path, 0o700)) res = NewRestorer(repo, sn, Options{Overwrite: overwrite}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) fi, err := os.Stat(path) rtest.OK(t, err) rtest.Equals(t, fs.FileMode(0o600), fi.Mode().Perm(), "unexpected permissions") diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 3f6c8472b..4764bed2d 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -181,7 +181,7 @@ func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAtt ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, testDir) + _, err := res.RestoreTo(ctx, testDir) rtest.OK(t, err) mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name) @@ -562,11 +562,11 @@ func TestRestoreDeleteCaseInsensitive(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) res = NewRestorer(repo, deleteSn, Options{Delete: true}) - err = res.RestoreTo(ctx, tempdir) + _, err = res.RestoreTo(ctx, tempdir) rtest.OK(t, err) // anotherfile must still exist From c83b529c47d7a32c6150be16b0be55c6be516edb Mon Sep 17 00:00:00 2001 From: Andreas Deininger Date: Sun, 11 Aug 2024 21:38:15 +0200 Subject: [PATCH 02/52] Fix typos --- doc/040_backup.rst | 2 +- internal/restorer/fileswriter_test.go | 2 +- internal/restorer/restorer_windows.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 25c5a8ad1..f1f355c53 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -585,7 +585,7 @@ Sometimes, it can be useful to directly save the output of a program, for exampl ``mysqldump`` so that the SQL can later be restored. Restic supports this mode of operation; just supply the option ``--stdin-from-command`` when using the ``backup`` action, and write the command in place of the files/directories. To prevent -restic from interpreting the arguments for the commmand, make sure to add ``--`` before +restic from interpreting the arguments for the command, make sure to add ``--`` before the command starts: .. code-block:: console diff --git a/internal/restorer/fileswriter_test.go b/internal/restorer/fileswriter_test.go index c69847927..9ea8767b8 100644 --- a/internal/restorer/fileswriter_test.go +++ b/internal/restorer/fileswriter_test.go @@ -49,7 +49,7 @@ func TestFilesWriterRecursiveOverwrite(t *testing.T) { // must error if recursive delete is not allowed w := newFilesWriter(1, false) err := w.writeToFile(path, []byte{1}, 0, 2, false) - rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexepected error got %v", err) + rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexpected error got %v", err) rtest.Equals(t, 0, len(w.buckets[0].files)) // must replace directory diff --git a/internal/restorer/restorer_windows.go b/internal/restorer/restorer_windows.go index 72337d8ae..9ddc0a932 100644 --- a/internal/restorer/restorer_windows.go +++ b/internal/restorer/restorer_windows.go @@ -8,6 +8,6 @@ import "strings" // toComparableFilename returns a filename suitable for equality checks. On Windows, it returns the // uppercase version of the string. On all other systems, it returns the unmodified filename. func toComparableFilename(path string) string { - // apparently NTFS internally uppercases filenames for comparision + // apparently NTFS internally uppercases filenames for comparison return strings.ToUpper(path) } From 8861421cd65d257a9ad03f09637787b8d80c9f53 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Sun, 11 Aug 2024 01:23:47 -0600 Subject: [PATCH 03/52] Fix extended attributes handling for VSS snapshots --- internal/fs/ea_windows.go | 18 +++++++ internal/restic/node_windows.go | 90 +++++++++++++++++++++++---------- 2 files changed, 80 insertions(+), 28 deletions(-) diff --git a/internal/fs/ea_windows.go b/internal/fs/ea_windows.go index d19a1ee6a..bf7b02fd4 100644 --- a/internal/fs/ea_windows.go +++ b/internal/fs/ea_windows.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "errors" "fmt" + "strings" "syscall" "unsafe" @@ -298,3 +299,20 @@ func PathSupportsExtendedAttributes(path string) (supported bool, err error) { supported = (fileSystemFlags & windows.FILE_SUPPORTS_EXTENDED_ATTRIBUTES) != 0 return supported, nil } + +// GetVolumePathName returns the volume path name for the given path. +func GetVolumePathName(path string) (volumeName string, err error) { + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", err + } + // Get the volume path (e.g., "D:") + var volumePath [windows.MAX_PATH + 1]uint16 + err = windows.GetVolumePathName(utf16Path, &volumePath[0], windows.MAX_PATH+1) + if err != nil { + return "", err + } + // Trim any trailing backslashes + volumeName = strings.TrimRight(windows.UTF16ToString(volumePath[:]), "\\") + return volumeName, nil +} diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index ceb304d0c..6adb51f0d 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -407,40 +407,74 @@ func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT // checkAndStoreEASupport checks if the volume of the path supports extended attributes and stores the result in a map // If the result is already in the map, it returns the result from the map. func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { - // Check if it's an extended length path - if strings.HasPrefix(path, uncPathPrefix) { - // Convert \\?\UNC\ extended path to standard path to get the volume name correctly - path = `\\` + path[len(uncPathPrefix):] - } else if strings.HasPrefix(path, extendedPathPrefix) { - //Extended length path prefix needs to be trimmed to get the volume name correctly - path = path[len(extendedPathPrefix):] - } else if strings.HasPrefix(path, globalRootPrefix) { - // EAs are not supported for \\?\GLOBALROOT i.e. VSS snapshots - return false, nil - } else { - // Use the absolute path - path, err = filepath.Abs(path) - if err != nil { - return false, fmt.Errorf("failed to get absolute path: %w", err) - } - } - volumeName := filepath.VolumeName(path) - if volumeName == "" { - return false, nil - } - eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeName) - if exists { - return eaSupportedValue.(bool), nil + var volumeName string + volumeName, err = prepareVolumeName(path) + if err != nil { + return false, err } - // Add backslash to the volume name to ensure it is a valid path - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) - if err == nil { - eaSupportedVolumesMap.Store(volumeName, isEASupportedVolume) + if volumeName != "" { + // First check if the manually prepared volume name is already in the map + eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeName) + if exists { + return eaSupportedValue.(bool), nil + } + // If not found, check if EA is supported with manually prepared volume name + isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) + if err != nil { + return false, err + } } + // If an entry is not found, get the actual volume name using the GetVolumePathName function + volumeNameActual, err := fs.GetVolumePathName(path) + if err != nil { + return false, err + } + if volumeNameActual != volumeName { + // If the actual volume name is different, check cache for the actual volume name + eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeNameActual) + if exists { + return eaSupportedValue.(bool), nil + } + // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name + isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeNameActual + `\`) + if err != nil { + return false, err + } + } + eaSupportedVolumesMap.Store(volumeNameActual, isEASupportedVolume) return isEASupportedVolume, err } +// prepareVolumeName prepares the volume name for different cases in Windows +func prepareVolumeName(path string) (volumeName string, err error) { + // Check if it's an extended length path + if strings.HasPrefix(path, globalRootPrefix) { + // Extract the VSS snapshot volume name eg. `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyXX` + if parts := strings.SplitN(path, `\`, 7); len(parts) >= 6 { + volumeName = strings.Join(parts[:6], `\`) + } else { + volumeName = filepath.VolumeName(path) + } + } else { + if strings.HasPrefix(path, uncPathPrefix) { + // Convert \\?\UNC\ extended path to standard path to get the volume name correctly + path = `\\` + path[len(uncPathPrefix):] + } else if strings.HasPrefix(path, extendedPathPrefix) { + //Extended length path prefix needs to be trimmed to get the volume name correctly + path = path[len(extendedPathPrefix):] + } else { + // Use the absolute path + path, err = filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } + } + volumeName = filepath.VolumeName(path) + } + return volumeName, nil +} + // windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { // Get the value of the WindowsAttributes From 111490b8bebac875d1e51926997dd95521f97e38 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Sun, 11 Aug 2024 01:32:55 -0600 Subject: [PATCH 04/52] Add changelog --- changelog/unreleased/pull-4998 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog/unreleased/pull-4998 diff --git a/changelog/unreleased/pull-4998 b/changelog/unreleased/pull-4998 new file mode 100644 index 000000000..23ff3dbd2 --- /dev/null +++ b/changelog/unreleased/pull-4998 @@ -0,0 +1,8 @@ +Bugfix: Fix extended attributes handling for VSS snapshots + +Restic was failing to backup extended attributes for VSS snapshots +after the fix for https://github.com/restic/restic/pull/4980. +Restic now correctly handles extended attributes for VSS snapshots. + +https://github.com/restic/restic/pull/4998 +https://github.com/restic/restic/pull/4980 From 51fad2eecb62209a39742c50087dc73635ad3f85 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Sun, 11 Aug 2024 01:48:25 -0600 Subject: [PATCH 05/52] Gracefully handle invalid prepared volume names --- internal/restic/node_windows.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index 6adb51f0d..2ca7e42e6 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -417,11 +417,13 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { // First check if the manually prepared volume name is already in the map eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeName) if exists { + // Cache hit, immediately return the cached value return eaSupportedValue.(bool), nil } // If not found, check if EA is supported with manually prepared volume name isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) - if err != nil { + // If the prepared volume name is not valid, we will next fetch the actual volume name. + if err != nil && !errors.Is(err, windows.DNS_ERROR_INVALID_NAME) { return false, err } } @@ -434,6 +436,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { // If the actual volume name is different, check cache for the actual volume name eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeNameActual) if exists { + // Cache hit, immediately return the cached value return eaSupportedValue.(bool), nil } // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name From 7642e05eed907d4d5508a7f0bf6d08d1492aea83 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Sun, 11 Aug 2024 19:25:58 -0600 Subject: [PATCH 06/52] Add test cases and handle volume GUID paths Gracefully handle errors while checking for EA and add debug logs. --- changelog/unreleased/pull-4980 | 1 + changelog/unreleased/pull-4998 | 8 -- internal/fs/ea_windows_test.go | 76 +++++++++++ internal/restic/node_windows.go | 49 ++++--- internal/restic/node_windows_test.go | 196 +++++++++++++++++++++++++++ 5 files changed, 306 insertions(+), 24 deletions(-) delete mode 100644 changelog/unreleased/pull-4998 diff --git a/changelog/unreleased/pull-4980 b/changelog/unreleased/pull-4980 index 264f347fa..5713db7a2 100644 --- a/changelog/unreleased/pull-4980 +++ b/changelog/unreleased/pull-4980 @@ -8,5 +8,6 @@ Restic now completely skips the attempt to fetch extended attributes for such volumes where it is not supported. https://github.com/restic/restic/pull/4980 +https://github.com/restic/restic/pull/4998 https://github.com/restic/restic/issues/4955 https://github.com/restic/restic/issues/4950 diff --git a/changelog/unreleased/pull-4998 b/changelog/unreleased/pull-4998 deleted file mode 100644 index 23ff3dbd2..000000000 --- a/changelog/unreleased/pull-4998 +++ /dev/null @@ -1,8 +0,0 @@ -Bugfix: Fix extended attributes handling for VSS snapshots - -Restic was failing to backup extended attributes for VSS snapshots -after the fix for https://github.com/restic/restic/pull/4980. -Restic now correctly handles extended attributes for VSS snapshots. - -https://github.com/restic/restic/pull/4998 -https://github.com/restic/restic/pull/4980 diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go index b249f43c4..74afd7aa5 100644 --- a/internal/fs/ea_windows_test.go +++ b/internal/fs/ea_windows_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "syscall" "testing" "unsafe" @@ -245,3 +246,78 @@ func testSetGetEA(t *testing.T, path string, handle windows.Handle, testEAs []Ex t.Fatalf("EAs read from path %s don't match", path) } } + +func TestPathSupportsExtendedAttributes(t *testing.T) { + testCases := []struct { + name string + path string + expected bool + }{ + { + name: "System drive", + path: os.Getenv("SystemDrive") + `\`, + expected: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + supported, err := PathSupportsExtendedAttributes(tc.path) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if supported != tc.expected { + t.Errorf("Expected %v, got %v for path %s", tc.expected, supported, tc.path) + } + }) + } + + // Test with an invalid path + _, err := PathSupportsExtendedAttributes("Z:\\NonExistentPath-UAS664da5s4dyu56das45f5as") + if err == nil { + t.Error("Expected an error for non-existent path, but got nil") + } +} + +func TestGetVolumePathName(t *testing.T) { + tempDirVolume := filepath.VolumeName(os.TempDir()) + testCases := []struct { + name string + path string + expectedPrefix string + }{ + { + name: "Root directory", + path: os.Getenv("SystemDrive") + `\`, + expectedPrefix: os.Getenv("SystemDrive"), + }, + { + name: "Nested directory", + path: os.Getenv("SystemDrive") + `\Windows\System32`, + expectedPrefix: os.Getenv("SystemDrive"), + }, + { + name: "Temp directory", + path: os.TempDir() + `\`, + expectedPrefix: tempDirVolume, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + volumeName, err := GetVolumePathName(tc.path) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !strings.HasPrefix(volumeName, tc.expectedPrefix) { + t.Errorf("Expected volume name to start with %s, but got %s", tc.expectedPrefix, volumeName) + } + }) + } + + // Test with an invalid path + _, err := GetVolumePathName("Z:\\NonExistentPath") + if err == nil { + t.Error("Expected an error for non-existent path, but got nil") + } +} diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index 2ca7e42e6..bce01ccad 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -42,6 +42,7 @@ const ( extendedPathPrefix = `\\?\` uncPathPrefix = `\\?\UNC\` globalRootPrefix = `\\?\GLOBALROOT\` + volumeGUIDPrefix = `\\?\Volume{` ) // mknod is not supported on Windows. @@ -422,15 +423,21 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { } // If not found, check if EA is supported with manually prepared volume name isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) - // If the prepared volume name is not valid, we will next fetch the actual volume name. + // If the prepared volume name is not valid, we will fetch the actual volume name next. if err != nil && !errors.Is(err, windows.DNS_ERROR_INVALID_NAME) { - return false, err + debug.Log("Error checking if extended attributes are supported for prepared volume name %s: %v", volumeName, err) + // There can be multiple errors like path does not exist, bad network path, etc. + // We just gracefully disallow extended attributes for cases. + return false, nil } } // If an entry is not found, get the actual volume name using the GetVolumePathName function volumeNameActual, err := fs.GetVolumePathName(path) if err != nil { - return false, err + debug.Log("Error getting actual volume name %s for path %s: %v", volumeName, path, err) + // There can be multiple errors like path does not exist, bad network path, etc. + // We just gracefully disallow extended attributes for cases. + return false, nil } if volumeNameActual != volumeName { // If the actual volume name is different, check cache for the actual volume name @@ -441,11 +448,19 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { } // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeNameActual + `\`) + // Debug log for cases where the prepared volume name is not valid if err != nil { - return false, err + debug.Log("Error checking if extended attributes are supported for actual volume name %s: %v", volumeNameActual, err) + // There can be multiple errors like path does not exist, bad network path, etc. + // We just gracefully disallow extended attributes for cases. + return false, nil + } else { + debug.Log("Checking extended attributes. Prepared volume name: %s, actual volume name: %s, isEASupportedVolume: %v, err: %v", volumeName, volumeNameActual, isEASupportedVolume, err) } } - eaSupportedVolumesMap.Store(volumeNameActual, isEASupportedVolume) + if volumeNameActual != "" { + eaSupportedVolumesMap.Store(volumeNameActual, isEASupportedVolume) + } return isEASupportedVolume, err } @@ -460,17 +475,19 @@ func prepareVolumeName(path string) (volumeName string, err error) { volumeName = filepath.VolumeName(path) } } else { - if strings.HasPrefix(path, uncPathPrefix) { - // Convert \\?\UNC\ extended path to standard path to get the volume name correctly - path = `\\` + path[len(uncPathPrefix):] - } else if strings.HasPrefix(path, extendedPathPrefix) { - //Extended length path prefix needs to be trimmed to get the volume name correctly - path = path[len(extendedPathPrefix):] - } else { - // Use the absolute path - path, err = filepath.Abs(path) - if err != nil { - return "", fmt.Errorf("failed to get absolute path: %w", err) + if !strings.HasPrefix(path, volumeGUIDPrefix) { // Handle volume GUID path + if strings.HasPrefix(path, uncPathPrefix) { + // Convert \\?\UNC\ extended path to standard path to get the volume name correctly + path = `\\` + path[len(uncPathPrefix):] + } else if strings.HasPrefix(path, extendedPathPrefix) { + //Extended length path prefix needs to be trimmed to get the volume name correctly + path = path[len(extendedPathPrefix):] + } else { + // Use the absolute path + path, err = filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } } } volumeName = filepath.VolumeName(path) diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go index 4fd57bbb7..6ba25559b 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/restic/node_windows_test.go @@ -12,6 +12,7 @@ import ( "strings" "syscall" "testing" + "time" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -329,3 +330,198 @@ func TestRestoreExtendedAttributes(t *testing.T) { } } } + +func TestPrepareVolumeName(t *testing.T) { + currentVolume := filepath.VolumeName(func() string { + // Get the current working directory + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get current working directory: %v", err) + } + return pwd + }()) + // Create a temporary directory for the test + tempDir, err := os.MkdirTemp("", "restic_test_"+time.Now().Format("20060102150405")) + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Create a long file name + longFileName := `\Very\Long\Path\That\Exceeds\260\Characters\` + strings.Repeat(`\VeryLongFolderName`, 20) + `\\LongFile.txt` + longFilePath := filepath.Join(tempDir, longFileName) + + tempDirVolume := filepath.VolumeName(tempDir) + // Create the file + content := []byte("This is a test file with a very long name.") + err = os.MkdirAll(filepath.Dir(longFilePath), 0755) + test.OK(t, err) + if err != nil { + t.Fatalf("Failed to create long folder: %v", err) + } + err = os.WriteFile(longFilePath, content, 0644) + test.OK(t, err) + if err != nil { + t.Fatalf("Failed to create long file: %v", err) + } + osVolumeGUIDPath := getOSVolumeGUIDPath(t) + osVolumeGUIDVolume := filepath.VolumeName(osVolumeGUIDPath) + + testCases := []struct { + name string + path string + expectedVolume string + expectError bool + expectedEASupported bool + isRealPath bool + }{ + { + name: "Network drive path", + path: `Z:\Shared\Documents`, + expectedVolume: `Z:`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Subst drive path", + path: `X:\Virtual\Folder`, + expectedVolume: `X:`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Windows reserved path", + path: `\\.\` + os.Getenv("SystemDrive") + `\System32\drivers\etc\hosts`, + expectedVolume: `\\.\` + os.Getenv("SystemDrive"), + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Long UNC path", + path: `\\?\UNC\LongServerName\VeryLongShareName\DeepPath\File.txt`, + expectedVolume: `\\LongServerName\VeryLongShareName`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Volume GUID path", + path: osVolumeGUIDPath, + expectedVolume: osVolumeGUIDVolume, + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Volume GUID path with subfolder", + path: osVolumeGUIDPath + `\Windows`, + expectedVolume: osVolumeGUIDVolume, + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Standard path", + path: os.Getenv("SystemDrive") + `\Users\`, + expectedVolume: os.Getenv("SystemDrive"), + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Extended length path", + path: longFilePath, + expectedVolume: tempDirVolume, + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "UNC path", + path: `\\server\share\folder`, + expectedVolume: `\\server\share`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Extended UNC path", + path: `\\?\UNC\server\share\folder`, + expectedVolume: `\\server\share`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Volume Shadow Copy path", + path: `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1\Users\test`, + expectedVolume: `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Relative path", + path: `folder\subfolder`, + + expectedVolume: currentVolume, // Get current volume + expectError: false, + expectedEASupported: true, + }, + { + name: "Empty path", + path: ``, + expectedVolume: currentVolume, + expectError: false, + expectedEASupported: true, + isRealPath: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + isEASupported, err := checkAndStoreEASupport(tc.path) + test.OK(t, err) + test.Equals(t, tc.expectedEASupported, isEASupported) + + volume, err := prepareVolumeName(tc.path) + + if tc.expectError { + test.Assert(t, err != nil, "Expected an error, but got none") + } else { + test.OK(t, err) + } + test.Equals(t, tc.expectedVolume, volume) + + if tc.isRealPath { + isEASupportedVolume, err := fs.PathSupportsExtendedAttributes(volume + `\`) + // If the prepared volume name is not valid, we will next fetch the actual volume name. + test.OK(t, err) + + test.Equals(t, tc.expectedEASupported, isEASupportedVolume) + + actualVolume, err := fs.GetVolumePathName(tc.path) + test.OK(t, err) + test.Equals(t, tc.expectedVolume, actualVolume) + } + }) + } +} + +func getOSVolumeGUIDPath(t *testing.T) string { + // Get the path of the OS drive (usually C:\) + osDrive := os.Getenv("SystemDrive") + "\\" + + // Convert to a volume GUID path + volumeName, err := windows.UTF16PtrFromString(osDrive) + test.OK(t, err) + if err != nil { + return "" + } + + var volumeGUID [windows.MAX_PATH]uint16 + err = windows.GetVolumeNameForVolumeMountPoint(volumeName, &volumeGUID[0], windows.MAX_PATH) + test.OK(t, err) + if err != nil { + return "" + } + + return windows.UTF16ToString(volumeGUID[:]) +} From 61e1f4a916e58c21d0b9bf4c250af81b63fc2480 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Aug 2024 18:06:24 +0200 Subject: [PATCH 07/52] backend: return correct error on upload/request timeout --- internal/backend/watchdog_roundtriper.go | 3 +++ internal/backend/watchdog_roundtriper_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/backend/watchdog_roundtriper.go b/internal/backend/watchdog_roundtriper.go index e3e10d7fe..dc270b974 100644 --- a/internal/backend/watchdog_roundtriper.go +++ b/internal/backend/watchdog_roundtriper.go @@ -65,6 +65,9 @@ func (w *watchdogRoundtripper) RoundTrip(req *http.Request) (*http.Response, err resp, err := w.rt.RoundTrip(req) if err != nil { + if isTimeout(err) { + err = errRequestTimeout + } return nil, err } diff --git a/internal/backend/watchdog_roundtriper_test.go b/internal/backend/watchdog_roundtriper_test.go index 723a311cb..f7f90259c 100644 --- a/internal/backend/watchdog_roundtriper_test.go +++ b/internal/backend/watchdog_roundtriper_test.go @@ -135,7 +135,7 @@ func TestUploadTimeout(t *testing.T) { rtest.OK(t, err) resp, err := rt.RoundTrip(req) - rtest.Equals(t, context.Canceled, err) + rtest.Equals(t, errRequestTimeout, err) // make linter happy if resp != nil { rtest.OK(t, resp.Body.Close()) @@ -162,7 +162,7 @@ func TestProcessingTimeout(t *testing.T) { rtest.OK(t, err) resp, err := rt.RoundTrip(req) - rtest.Equals(t, context.Canceled, err) + rtest.Equals(t, errRequestTimeout, err) // make linter happy if resp != nil { rtest.OK(t, resp.Body.Close()) From 48e3832322e8d03bd60a379d0c5ae1c48977a370 Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Tue, 30 Jul 2024 19:06:18 -0400 Subject: [PATCH 08/52] main: return an exit code (12) for "bad password" errors --- changelog/unreleased/pull-4959 | 6 ++++++ cmd/restic/cmd_backup.go | 1 + cmd/restic/cmd_cat.go | 1 + cmd/restic/cmd_check.go | 1 + cmd/restic/cmd_copy.go | 1 + cmd/restic/cmd_debug.go | 1 + cmd/restic/cmd_diff.go | 1 + cmd/restic/cmd_dump.go | 1 + cmd/restic/cmd_find.go | 1 + cmd/restic/cmd_forget.go | 1 + cmd/restic/cmd_key_add.go | 1 + cmd/restic/cmd_key_list.go | 1 + cmd/restic/cmd_key_passwd.go | 1 + cmd/restic/cmd_key_remove.go | 1 + cmd/restic/cmd_list.go | 1 + cmd/restic/cmd_ls.go | 1 + cmd/restic/cmd_migrate.go | 1 + cmd/restic/cmd_mount.go | 1 + cmd/restic/cmd_prune.go | 1 + cmd/restic/cmd_recover.go | 1 + cmd/restic/cmd_repair_index.go | 1 + cmd/restic/cmd_repair_packs.go | 1 + cmd/restic/cmd_repair_snapshots.go | 1 + cmd/restic/cmd_restore.go | 1 + cmd/restic/cmd_rewrite.go | 1 + cmd/restic/cmd_self_update.go | 1 + cmd/restic/cmd_snapshots.go | 1 + cmd/restic/cmd_stats.go | 1 + cmd/restic/cmd_tag.go | 1 + cmd/restic/global.go | 2 +- cmd/restic/main.go | 5 +++++ doc/075_scripting.rst | 2 ++ 32 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/pull-4959 diff --git a/changelog/unreleased/pull-4959 b/changelog/unreleased/pull-4959 new file mode 100644 index 000000000..120527e22 --- /dev/null +++ b/changelog/unreleased/pull-4959 @@ -0,0 +1,6 @@ +Enhancement: Return exit code 12 for "bad password" + +Restic now returns exit code 12 when it can't open the repository +because of a bad password. + +https://github.com/restic/restic/pull/4959 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 9957b5784..28b6c7feb 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -43,6 +43,7 @@ Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, PreRun: func(_ *cobra.Command, _ []string) { if backupOptions.Host == "" { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index d3e98b2ff..ac03798d2 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 9cccc0609..b0749e022 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -39,6 +39,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index d7761174a..40015b13c 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -38,6 +38,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, RunE: func(cmd *cobra.Command, args []string) error { return runCopy(cmd.Context(), copyOptions, globalOptions, args) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 74c21df24..18b4b7631 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -47,6 +47,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 6793184b1..24f445b64 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -43,6 +43,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 9c0fe535e..a5794ad30 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -38,6 +38,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index aebca594e..f84ad43c3 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -37,6 +37,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 27b8f4f74..01fe0e606 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -39,6 +39,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_key_add.go b/cmd/restic/cmd_key_add.go index c9f0ef233..2737410a0 100644 --- a/cmd/restic/cmd_key_add.go +++ b/cmd/restic/cmd_key_add.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, } diff --git a/cmd/restic/cmd_key_list.go b/cmd/restic/cmd_key_list.go index ae751a487..1c70cce8a 100644 --- a/cmd/restic/cmd_key_list.go +++ b/cmd/restic/cmd_key_list.go @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_key_passwd.go b/cmd/restic/cmd_key_passwd.go index 723acaaab..9bb141749 100644 --- a/cmd/restic/cmd_key_passwd.go +++ b/cmd/restic/cmd_key_passwd.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, } diff --git a/cmd/restic/cmd_key_remove.go b/cmd/restic/cmd_key_remove.go index c4c24fdb7..3cb2e0bd7 100644 --- a/cmd/restic/cmd_key_remove.go +++ b/cmd/restic/cmd_key_remove.go @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 060bca871..4aa9f43bb 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 76e192b6c..7c712e481 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -43,6 +43,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 7e472ff12..2cc44bff0 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -26,6 +26,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 3e0b159be..0b79afe45 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -68,6 +68,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 7e706ccf8..e19c2e04b 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -32,6 +32,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 4e8b8c077..0ff6e2d66 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -26,6 +26,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index e6b6e9fa5..83c1bfa7f 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index b0afefb2d..290c3734e 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index fc221ebea..385854312 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -41,6 +41,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index d71cb7683..eb437a11d 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -36,6 +36,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 463720ee1..d1088d00b 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -42,6 +42,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_self_update.go b/cmd/restic/cmd_self_update.go index 0fce41241..09c86bf2c 100644 --- a/cmd/restic/cmd_self_update.go +++ b/cmd/restic/cmd_self_update.go @@ -28,6 +28,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 826ab55ec..442c57375 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 60ab1e5bc..d26411783 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -53,6 +53,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index ea73955f0..47e3c02ad 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -29,6 +29,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 846f3339f..22aa8a290 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -493,7 +493,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } } if err != nil { - if errors.IsFatal(err) { + if errors.IsFatal(err) || errors.Is(err, repository.ErrNoKeyFound) { return nil, err } return nil, errors.Fatalf("%s", err) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 5818221a5..6661b4f5d 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -138,6 +139,8 @@ func main() { fmt.Fprintf(os.Stderr, "Warning: %v\n", err) case errors.IsFatal(err): fmt.Fprintf(os.Stderr, "%v\n", err) + case errors.Is(err, repository.ErrNoKeyFound): + fmt.Fprintf(os.Stderr, "Fatal: %v\n", err) case err != nil: fmt.Fprintf(os.Stderr, "%+v\n", err) @@ -160,6 +163,8 @@ func main() { exitCode = 10 case restic.IsAlreadyLocked(err): exitCode = 11 + case errors.Is(err, repository.ErrNoKeyFound): + exitCode = 12 case errors.Is(err, context.Canceled): exitCode = 130 default: diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index fa7fa1b6e..438eaa84f 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -63,6 +63,8 @@ a more specific description. +-----+----------------------------------------------------+ | 11 | Failed to lock repository (since restic 0.17.0) | +-----+----------------------------------------------------+ +| 12 | Wrong password (since restic 0.17.1) | ++-----+----------------------------------------------------+ | 130 | Restic was interrupted using SIGINT or SIGSTOP | +-----+----------------------------------------------------+ From fac1d9fea17d8ad23073d5d4523ebaca16e42a0e Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 11 Aug 2024 01:42:13 +0530 Subject: [PATCH 09/52] cache: backend add List method and a cache clear functionality * removes files which are no longer in the repository, including index files, snapshot files and pack files from the cache. cache: fix ids set initialisation with NewIDSet() --- internal/backend/cache/backend.go | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/internal/backend/cache/backend.go b/internal/backend/cache/backend.go index 94f648cf4..92cca4d0e 100644 --- a/internal/backend/cache/backend.go +++ b/internal/backend/cache/backend.go @@ -2,11 +2,14 @@ package cache import ( "context" + "fmt" "io" + "os" "sync" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" ) // Backend wraps a restic.Backend and adds a cache. @@ -215,3 +218,40 @@ func (b *Backend) IsNotExist(err error) bool { func (b *Backend) Unwrap() backend.Backend { return b.Backend } + +func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error { + if !b.Cache.canBeCached(t) { + return b.Backend.List(ctx, t, fn) + } + + // will contain the IDs of the files that are in the repository + ids := restic.NewIDSet() + + // wrap the original function to also add the file to the ids set + wrapFn := func(f backend.FileInfo) error { + id, err := restic.ParseID(f.Name) + if err != nil { + // returning error here since, if we cannot parse the ID, the file + // is invalid and the list must exit. + return err + } + + ids.Insert(id) + + // execute the original function + return fn(f) + } + + err := b.Backend.List(ctx, t, wrapFn) + if err != nil { + return err + } + + // clear the cache for files that are not in the repo anymore, ignore errors + err = b.Cache.Clear(t, ids) + if err != nil { + fmt.Fprintf(os.Stderr, "error clearing %s files in cache: %v\n", t.String(), err) + } + + return nil +} From 0cf17372894d15c9f3a3e04ffe8523971d0f4bd8 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 11 Aug 2024 15:43:03 +0530 Subject: [PATCH 10/52] cache: check for context cancellation before clearing cache --- internal/backend/cache/backend.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/backend/cache/backend.go b/internal/backend/cache/backend.go index 92cca4d0e..58b03dd38 100644 --- a/internal/backend/cache/backend.go +++ b/internal/backend/cache/backend.go @@ -247,6 +247,10 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen return err } + if ctx.Err() != nil { + return ctx.Err() + } + // clear the cache for files that are not in the repo anymore, ignore errors err = b.Cache.Clear(t, ids) if err != nil { From 1e68fbca90dff5ac02f7665eb9193d3914d0d284 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 11 Aug 2024 15:58:27 +0530 Subject: [PATCH 11/52] repository: removed redundant prepareCache method from Repository * remove the prepareCache method from the Repository * changed the signature of the SetIndex function to no longer return an error --- internal/checker/checker.go | 6 +---- internal/repository/repair_index.go | 6 ++--- internal/repository/repository.go | 37 ++--------------------------- internal/restic/repository.go | 2 +- 4 files changed, 6 insertions(+), 45 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 031e13807..d5e7fd1f8 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -146,11 +146,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e return hints, append(errs, err) } - err = c.repo.SetIndex(c.masterIndex) - if err != nil { - debug.Log("SetIndex returned error: %v", err) - errs = append(errs, err) - } + c.repo.SetIndex(c.masterIndex) // compute pack size using index entries c.packs, err = pack.Size(ctx, c.repo, false) diff --git a/internal/repository/repair_index.go b/internal/repository/repair_index.go index 770809254..c72dcfd00 100644 --- a/internal/repository/repair_index.go +++ b/internal/repository/repair_index.go @@ -52,10 +52,8 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions, return err } - err = repo.SetIndex(mi) - if err != nil { - return err - } + repo.SetIndex(mi) + packSizeFromIndex, err = pack.Size(ctx, repo, false) if err != nil { return err diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f7fd65c71..3dc248c5e 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "math" - "os" "runtime" "sort" "sync" @@ -586,9 +585,8 @@ func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) } // SetIndex instructs the repository to use the given index. -func (r *Repository) SetIndex(i restic.MasterIndex) error { +func (r *Repository) SetIndex(i restic.MasterIndex) { r.idx = i.(*index.MasterIndex) - return r.prepareCache() } func (r *Repository) clearIndex() { @@ -628,12 +626,8 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { return errors.New("index uses feature not supported by repository version 1") } } - if ctx.Err() != nil { - return ctx.Err() - } - // remove index files from the cache which have been removed in the repo - return r.prepareCache() + return ctx.Err() } // createIndexFromPacks creates a new index by reading all given pack files (with sizes). @@ -699,33 +693,6 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest return invalid, nil } -// prepareCache initializes the local cache. indexIDs is the list of IDs of -// index files still present in the repo. -func (r *Repository) prepareCache() error { - if r.Cache == nil { - return nil - } - - indexIDs := r.idx.IDs() - debug.Log("prepare cache with %d index files", len(indexIDs)) - - // clear old index files - err := r.Cache.Clear(restic.IndexFile, indexIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err) - } - - packs := r.idx.Packs(restic.NewIDSet()) - - // clear old packs - err = r.Cache.Clear(restic.PackFile, packs) - if err != nil { - fmt.Fprintf(os.Stderr, "error clearing pack files in cache: %v\n", err) - } - - return nil -} - // SearchKey finds a key with the supplied password, afterwards the config is // read and parsed. It tries at most maxKeys key files in the repo. func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int, keyHint string) error { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index b18b036a7..ce8401b37 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -22,7 +22,7 @@ type Repository interface { Key() *crypto.Key LoadIndex(ctx context.Context, p *progress.Counter) error - SetIndex(mi MasterIndex) error + SetIndex(mi MasterIndex) LookupBlob(t BlobType, id ID) []PackedBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) From b10d7ccddad646ddd35e29faa26a865d65cfbf20 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 11 Aug 2024 16:07:38 +0530 Subject: [PATCH 12/52] changelog: add unrelease changelog --- changelog/unreleased/issue-4934 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/unreleased/issue-4934 diff --git a/changelog/unreleased/issue-4934 b/changelog/unreleased/issue-4934 new file mode 100644 index 000000000..03194168e --- /dev/null +++ b/changelog/unreleased/issue-4934 @@ -0,0 +1,9 @@ +Enhancement: Clear removed snapshots, index and pack files from the local cache + +Restic did not clear removed snapshots from the cache after the `forget` +operation; only indexes and pack files were removed automatically. +Restic now automatically clears removed indexes, packs and snapshots from the +local cache. + +https://github.com/restic/restic/issues/4934 +https://github.com/restic/restic/pull/4981 \ No newline at end of file From 0ca9355bc034055fc1d126ce73c0e7ac507c0186 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 11 Aug 2024 16:44:43 +0530 Subject: [PATCH 13/52] cache: add test for the automated cache clear to cache backend --- internal/backend/cache/backend_test.go | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/internal/backend/cache/backend_test.go b/internal/backend/cache/backend_test.go index 7addc275d..dca51c2bf 100644 --- a/internal/backend/cache/backend_test.go +++ b/internal/backend/cache/backend_test.go @@ -57,6 +57,13 @@ func randomData(n int) (backend.Handle, []byte) { return h, data } +func list(t testing.TB, be backend.Backend, fn func(backend.FileInfo) error) { + err := be.List(context.TODO(), backend.IndexFile, fn) + if err != nil { + t.Fatal(err) + } +} + func TestBackend(t *testing.T) { be := mem.New() c := TestNewCache(t) @@ -238,3 +245,54 @@ func TestErrorBackend(t *testing.T) { wg.Wait() } + +func TestAutomaticCacheClear(t *testing.T) { + be := mem.New() + c := TestNewCache(t) + wbe := c.Wrap(be) + + // add two handles h1 and h2 + h1, data := randomData(2000) + // save h1 directly to the backend + save(t, be, h1, data) + if c.Has(h1) { + t.Errorf("cache has file1 too early") + } + + h2, data2 := randomData(3000) + + // save h2 directly to the backend + save(t, be, h2, data2) + if c.Has(h2) { + t.Errorf("cache has file2 too early") + } + + loadAndCompare(t, wbe, h1, data) + if !c.Has(h1) { + t.Errorf("cache doesn't have file1 after load") + } + + loadAndCompare(t, wbe, h2, data2) + if !c.Has(h2) { + t.Errorf("cache doesn't have file2 after load") + } + + // remove h1 directly from the backend + remove(t, be, h1) + if !c.Has(h1) { + t.Errorf("file1 not in cache any more, should be removed from cache only after list") + } + + // list all files in the backend + list(t, wbe, func(_ backend.FileInfo) error { return nil }) + + // h1 should be removed from the cache + if c.Has(h1) { + t.Errorf("cache has file1 after remove") + } + + // h2 should still be in the cache + if !c.Has(h2) { + t.Errorf("cache doesn't have file2 after list") + } +} From 8a7ae17d4dcd11f8b38e9f0be205baac216bb95f Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sat, 17 Aug 2024 00:18:13 +0530 Subject: [PATCH 14/52] Revert "repository: removed redundant prepareCache method from Repository" This reverts commit 720609f8ba6dcf44b7fe51cd9b543ee44bbbaf38. --- internal/checker/checker.go | 6 ++++- internal/repository/repair_index.go | 6 +++-- internal/repository/repository.go | 37 +++++++++++++++++++++++++++-- internal/restic/repository.go | 2 +- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index d5e7fd1f8..031e13807 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -146,7 +146,11 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e return hints, append(errs, err) } - c.repo.SetIndex(c.masterIndex) + err = c.repo.SetIndex(c.masterIndex) + if err != nil { + debug.Log("SetIndex returned error: %v", err) + errs = append(errs, err) + } // compute pack size using index entries c.packs, err = pack.Size(ctx, c.repo, false) diff --git a/internal/repository/repair_index.go b/internal/repository/repair_index.go index c72dcfd00..770809254 100644 --- a/internal/repository/repair_index.go +++ b/internal/repository/repair_index.go @@ -52,8 +52,10 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions, return err } - repo.SetIndex(mi) - + err = repo.SetIndex(mi) + if err != nil { + return err + } packSizeFromIndex, err = pack.Size(ctx, repo, false) if err != nil { return err diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 3dc248c5e..f7fd65c71 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "math" + "os" "runtime" "sort" "sync" @@ -585,8 +586,9 @@ func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) } // SetIndex instructs the repository to use the given index. -func (r *Repository) SetIndex(i restic.MasterIndex) { +func (r *Repository) SetIndex(i restic.MasterIndex) error { r.idx = i.(*index.MasterIndex) + return r.prepareCache() } func (r *Repository) clearIndex() { @@ -626,8 +628,12 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { return errors.New("index uses feature not supported by repository version 1") } } + if ctx.Err() != nil { + return ctx.Err() + } - return ctx.Err() + // remove index files from the cache which have been removed in the repo + return r.prepareCache() } // createIndexFromPacks creates a new index by reading all given pack files (with sizes). @@ -693,6 +699,33 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest return invalid, nil } +// prepareCache initializes the local cache. indexIDs is the list of IDs of +// index files still present in the repo. +func (r *Repository) prepareCache() error { + if r.Cache == nil { + return nil + } + + indexIDs := r.idx.IDs() + debug.Log("prepare cache with %d index files", len(indexIDs)) + + // clear old index files + err := r.Cache.Clear(restic.IndexFile, indexIDs) + if err != nil { + fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err) + } + + packs := r.idx.Packs(restic.NewIDSet()) + + // clear old packs + err = r.Cache.Clear(restic.PackFile, packs) + if err != nil { + fmt.Fprintf(os.Stderr, "error clearing pack files in cache: %v\n", err) + } + + return nil +} + // SearchKey finds a key with the supplied password, afterwards the config is // read and parsed. It tries at most maxKeys key files in the repo. func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int, keyHint string) error { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index ce8401b37..b18b036a7 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -22,7 +22,7 @@ type Repository interface { Key() *crypto.Key LoadIndex(ctx context.Context, p *progress.Counter) error - SetIndex(mi MasterIndex) + SetIndex(mi MasterIndex) error LookupBlob(t BlobType, id ID) []PackedBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) From a8032c932cf414ba14b51d3e9ad5d947855a35b5 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sat, 17 Aug 2024 00:21:49 +0530 Subject: [PATCH 15/52] cache: remove redundant index file cleanup addressing code review comments --- internal/repository/repository.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f7fd65c71..d408e3105 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -706,19 +706,10 @@ func (r *Repository) prepareCache() error { return nil } - indexIDs := r.idx.IDs() - debug.Log("prepare cache with %d index files", len(indexIDs)) - - // clear old index files - err := r.Cache.Clear(restic.IndexFile, indexIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err) - } - packs := r.idx.Packs(restic.NewIDSet()) // clear old packs - err = r.Cache.Clear(restic.PackFile, packs) + err := r.Cache.Clear(restic.PackFile, packs) if err != nil { fmt.Fprintf(os.Stderr, "error clearing pack files in cache: %v\n", err) } From f6e8d925902d1c968456b5629a99d50361d66113 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sat, 17 Aug 2024 00:24:19 +0530 Subject: [PATCH 16/52] changelog: update changelog --- changelog/unreleased/issue-4934 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/changelog/unreleased/issue-4934 b/changelog/unreleased/issue-4934 index 03194168e..6891ca204 100644 --- a/changelog/unreleased/issue-4934 +++ b/changelog/unreleased/issue-4934 @@ -1,9 +1,8 @@ -Enhancement: Clear removed snapshots, index and pack files from the local cache +Enhancement: Clear removed snapshots from local cache of the current host -Restic did not clear removed snapshots from the cache after the `forget` -operation; only indexes and pack files were removed automatically. -Restic now automatically clears removed indexes, packs and snapshots from the -local cache. +Restic only removed snapshots from the cache on the host that runs the `forget` command. +On other hosts that use the same repository, the old snapshots remained in the cache. +Restic now, automatically clears old snapshots from the local cache of the current host. https://github.com/restic/restic/issues/4934 https://github.com/restic/restic/pull/4981 \ No newline at end of file From 12089054d8cb17005261773227215320493e2ca0 Mon Sep 17 00:00:00 2001 From: Andreas Deininger Date: Sat, 17 Aug 2024 12:39:41 +0200 Subject: [PATCH 17/52] GitHub test actions: fix warnings 'Restore cache failed' --- .github/workflows/tests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ca7a9edb..2ffeb5ff2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,6 +66,9 @@ jobs: GOPROXY: https://proxy.golang.org steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v5 with: @@ -139,9 +142,6 @@ jobs: echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH if: matrix.os == 'windows-latest' - - name: Check out code - uses: actions/checkout@v4 - - name: Build with build.go run: | go run build.go @@ -230,14 +230,14 @@ jobs: name: Cross Compile for subset ${{ matrix.subset }} steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} - - name: Check out code - uses: actions/checkout@v4 - - name: Cross-compile for subset ${{ matrix.subset }} run: | mkdir build-output build-output-debug @@ -252,14 +252,14 @@ jobs: # allow annotating code in the PR checks: write steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} - - name: Check out code - uses: actions/checkout@v4 - - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: From 00f63d72fa3431d4c2a5471f5b2457e7e10c9ca8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 18 Aug 2024 19:41:58 +0200 Subject: [PATCH 18/52] Mention RESTIC_HOST environment variable in docs --- doc/040_backup.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index f1f355c53..b53ae8d09 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -704,6 +704,7 @@ environment variables. The following lists these environment variables: RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert) RESTIC_CACHE_DIR Location of the cache directory RESTIC_COMPRESSION Compression mode (only available for repository format version 2) + RESTIC_HOST Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host) RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated RESTIC_PACK_SIZE Target size for pack files RESTIC_READ_CONCURRENCY Concurrency for file reads From a5f2d0cf565ba48bb3ae1ea8f7050abc19d387bf Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 18 Aug 2024 19:45:54 +0200 Subject: [PATCH 19/52] Improve description for no password on secondary repo --- cmd/restic/secondary_repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/secondary_repo.go b/cmd/restic/secondary_repo.go index 9a3eb5fe2..44621afa1 100644 --- a/cmd/restic/secondary_repo.go +++ b/cmd/restic/secondary_repo.go @@ -50,7 +50,7 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)") f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)") f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)") - f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository, must be passed to every restic command (insecure)") + f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository (insecure)") opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY") opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE") From 61aaddac280087ab73d01d7ad2060bd603ab0ae8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 15 Aug 2024 21:17:49 +0200 Subject: [PATCH 20/52] restic: restore timestamps after extended attributes restoring the xattr containing resource forks on macOS apparently modifies the file modification timestamps. Thus, restore the timestamp after xattrs. --- changelog/unreleased/issue-4969 | 7 +++++++ internal/restic/node.go | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/issue-4969 diff --git a/changelog/unreleased/issue-4969 b/changelog/unreleased/issue-4969 new file mode 100644 index 000000000..ce76a7389 --- /dev/null +++ b/changelog/unreleased/issue-4969 @@ -0,0 +1,7 @@ +Bugfix: Correctly restore timestamp for files with resource forks on macOS + +On macOS, timestamps were incorrectly restored for files with resource forks. +This has been fixed. + +https://github.com/restic/restic/issues/4969 +https://github.com/restic/restic/pull/5006 diff --git a/internal/restic/node.go b/internal/restic/node.go index 7c1988227..6afdff64a 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -249,13 +249,6 @@ func (node Node) restoreMetadata(path string, warn func(msg string)) error { firsterr = errors.WithStack(err) } - if err := node.RestoreTimestamps(path); err != nil { - debug.Log("error restoring timestamps for dir %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - if err := node.restoreExtendedAttributes(path); err != nil { debug.Log("error restoring extended attributes for %v: %v", path, err) if firsterr == nil { @@ -270,6 +263,13 @@ func (node Node) restoreMetadata(path string, warn func(msg string)) error { } } + if err := node.RestoreTimestamps(path); err != nil { + debug.Log("error restoring timestamps for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows // calling Chmod below will no longer allow any modifications to be made on the file and the // calls above would fail. From bc1aecfb15da21d335409efa558de2224225fc65 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 25 Aug 2024 23:13:54 +0200 Subject: [PATCH 21/52] restore: test timestamps for macOS resource forks are restored correctly --- internal/restic/node_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 7991d33e0..642beadc5 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -197,6 +197,20 @@ var nodeTests = []Node{ {"user.foo", []byte("bar")}, }, }, + { + Name: "testXattrFileMacOSResourceFork", + Type: "file", + Content: IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []ExtendedAttribute{ + {"com.apple.ResourceFork", []byte("bar")}, + }, + }, } func TestNodeRestoreAt(t *testing.T) { @@ -216,6 +230,11 @@ func TestNodeRestoreAt(t *testing.T) { extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) } } + for _, attr := range test.ExtendedAttributes { + if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { + t.Skipf("attr %v only relevant on macOS", attr.Name) + } + } // tempdir might be backed by a filesystem that does not support // extended attributes From cb16add8c82ab4f5087bd044893ab382c250e89e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 25 Aug 2024 23:14:39 +0200 Subject: [PATCH 22/52] restic: cleanup redundant code in test case --- internal/restic/node_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 642beadc5..ab7f66e5b 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -248,10 +248,6 @@ func TestNodeRestoreAt(t *testing.T) { rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) - if test.Type == "dir" { - rtest.OK(t, test.RestoreTimestamps(nodePath)) - } - fi, err := os.Lstat(nodePath) rtest.OK(t, err) From e5a08e6808666c4929142722c35f9c9da1fe10bc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 23 Aug 2024 23:48:45 +0200 Subject: [PATCH 23/52] group commands and make features/options visible --- cmd/restic/cmd_backup.go | 1 + cmd/restic/cmd_cache.go | 1 + cmd/restic/cmd_cat.go | 1 + cmd/restic/cmd_check.go | 1 + cmd/restic/cmd_copy.go | 1 + cmd/restic/cmd_debug.go | 5 +++-- cmd/restic/cmd_diff.go | 1 + cmd/restic/cmd_dump.go | 1 + cmd/restic/cmd_features.go | 2 +- cmd/restic/cmd_find.go | 1 + cmd/restic/cmd_forget.go | 1 + cmd/restic/cmd_init.go | 1 + cmd/restic/cmd_key.go | 1 + cmd/restic/cmd_list.go | 1 + cmd/restic/cmd_ls.go | 1 + cmd/restic/cmd_migrate.go | 1 + cmd/restic/cmd_mount.go | 1 + cmd/restic/cmd_options.go | 2 +- cmd/restic/cmd_prune.go | 1 + cmd/restic/cmd_recover.go | 1 + cmd/restic/cmd_repair.go | 5 +++-- cmd/restic/cmd_restore.go | 1 + cmd/restic/cmd_rewrite.go | 1 + cmd/restic/cmd_snapshots.go | 1 + cmd/restic/cmd_stats.go | 1 + cmd/restic/cmd_tag.go | 1 + cmd/restic/cmd_unlock.go | 1 + cmd/restic/main.go | 16 ++++++++++++++++ 28 files changed, 47 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 28b6c7feb..562108a33 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -55,6 +55,7 @@ Exit status is 12 if the password is incorrect. backupOptions.Host = hostname } }, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index e71d38365..e54c73451 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -28,6 +28,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, args []string) error { return runCache(cacheOptions, globalOptions, args) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index ac03798d2..6160c54df 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -29,6 +29,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runCat(cmd.Context(), globalOptions, args) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index b0749e022..dcf7f27df 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -41,6 +41,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 40015b13c..cfe574d35 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -40,6 +40,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runCopy(cmd.Context(), copyOptions, globalOptions, args) }, diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 18b4b7631..2a48762d1 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -29,8 +29,9 @@ import ( ) var cmdDebug = &cobra.Command{ - Use: "debug", - Short: "Debug commands", + Use: "debug", + Short: "Debug commands", + GroupID: cmdGroupDefault, } var cmdDebugDump = &cobra.Command{ diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 24f445b64..594e387e8 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -45,6 +45,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runDiff(cmd.Context(), diffOptions, globalOptions, args) diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index a5794ad30..7d6652e17 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -40,6 +40,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runDump(cmd.Context(), dumpOptions, globalOptions, args) diff --git a/cmd/restic/cmd_features.go b/cmd/restic/cmd_features.go index 497013696..a2f04be31 100644 --- a/cmd/restic/cmd_features.go +++ b/cmd/restic/cmd_features.go @@ -31,7 +31,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, - Hidden: true, + GroupID: cmdGroupAdvanced, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, args []string) error { if len(args) != 0 { diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index f84ad43c3..cb5c0e5e0 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -39,6 +39,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runFind(cmd.Context(), findOptions, globalOptions, args) diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 01fe0e606..58a9d25b7 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -41,6 +41,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 3c0319e55..2a2aae1dc 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -26,6 +26,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runInit(cmd.Context(), initOptions, globalOptions, args) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index c687eca53..80e892f20 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -11,6 +11,7 @@ var cmdKey = &cobra.Command{ The "key" command allows you to set multiple access keys or passwords per repository. `, + GroupID: cmdGroupDefault, } func init() { diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 4aa9f43bb..1a4791e31 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -26,6 +26,7 @@ Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runList(cmd.Context(), globalOptions, args) }, diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 7c712e481..69e278103 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -46,6 +46,7 @@ Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runLs(cmd.Context(), lsOptions, globalOptions, args) }, diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 2cc44bff0..5c3e425ed 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -29,6 +29,7 @@ Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() defer cancel() diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 0b79afe45..2f57a6d1f 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -71,6 +71,7 @@ Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runMount(cmd.Context(), mountOptions, globalOptions, args) }, diff --git a/cmd/restic/cmd_options.go b/cmd/restic/cmd_options.go index 4cd574b68..9c07b2626 100644 --- a/cmd/restic/cmd_options.go +++ b/cmd/restic/cmd_options.go @@ -20,7 +20,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, - Hidden: true, + GroupID: cmdGroupAdvanced, DisableAutoGenTag: true, Run: func(_ *cobra.Command, _ []string) { fmt.Printf("All Extended Options:\n") diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index e19c2e04b..e8473bd6f 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -34,6 +34,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { term, cancel := setupTermstatus() diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 0ff6e2d66..a6ef59cc2 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -28,6 +28,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { return runRecover(cmd.Context(), globalOptions) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index aefe02f3c..65a903a49 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -5,8 +5,9 @@ import ( ) var cmdRepair = &cobra.Command{ - Use: "repair", - Short: "Repair the repository", + Use: "repair", + Short: "Repair the repository", + GroupID: cmdGroupDefault, } func init() { diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index eb437a11d..c58b0b80d 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -38,6 +38,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index d1088d00b..7788016b7 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -44,6 +44,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runRewrite(cmd.Context(), rewriteOptions, globalOptions, args) diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 442c57375..42677918f 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -29,6 +29,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index d26411783..c4438c192 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -55,6 +55,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runStats(cmd.Context(), statsOptions, globalOptions, args) diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 47e3c02ad..c7bf725e9 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -31,6 +31,7 @@ Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runTag(cmd.Context(), tagOptions, globalOptions, args) diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go index 96eef7e02..d87cde065 100644 --- a/cmd/restic/cmd_unlock.go +++ b/cmd/restic/cmd_unlock.go @@ -19,6 +19,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { return runUnlock(cmd.Context(), unlockOptions, globalOptions) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 6661b4f5d..26e45bb38 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -83,6 +83,22 @@ The full documentation can be found at https://restic.readthedocs.io/ . }, } +var cmdGroupDefault = "default" +var cmdGroupAdvanced = "advanced" + +func init() { + cmdRoot.AddGroup( + &cobra.Group{ + ID: cmdGroupDefault, + Title: "Available Commands:", + }, + &cobra.Group{ + ID: cmdGroupAdvanced, + Title: "Advanced Options:", + }, + ) +} + // Distinguish commands that need the password from those that work without, // so we don't run $RESTIC_PASSWORD_COMMAND for no reason (it might prompt the // user for authentication). From 424740f62c61ce0d20379aa8cf34e2cd14624541 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 23 Aug 2024 23:49:20 +0200 Subject: [PATCH 24/52] Add missing DisableAutoGenTag flag for commands --- cmd/restic/cmd_copy.go | 1 + cmd/restic/cmd_debug.go | 1 + cmd/restic/cmd_key.go | 1 + cmd/restic/cmd_repair.go | 1 + 4 files changed, 4 insertions(+) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index cfe574d35..cd92193ac 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -41,6 +41,7 @@ Exit status is 11 if the repository is already locked. Exit status is 12 if the password is incorrect. `, GroupID: cmdGroupDefault, + DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runCopy(cmd.Context(), copyOptions, globalOptions, args) }, diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 2a48762d1..b92192492 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -32,6 +32,7 @@ var cmdDebug = &cobra.Command{ Use: "debug", Short: "Debug commands", GroupID: cmdGroupDefault, + DisableAutoGenTag: true, } var cmdDebugDump = &cobra.Command{ diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 80e892f20..a94caa0d8 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -11,6 +11,7 @@ var cmdKey = &cobra.Command{ The "key" command allows you to set multiple access keys or passwords per repository. `, + DisableAutoGenTag: true, GroupID: cmdGroupDefault, } diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index 65a903a49..6a1a1f9dc 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -8,6 +8,7 @@ var cmdRepair = &cobra.Command{ Use: "repair", Short: "Repair the repository", GroupID: cmdGroupDefault, + DisableAutoGenTag: true, } func init() { From a99b82450811046944038fef0b10b27d0f68aede Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 23 Aug 2024 23:52:21 +0200 Subject: [PATCH 25/52] update docs --- doc/manual_rest.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index a7a0f96e0..031f4fc52 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -28,8 +28,6 @@ Usage help is available: dump Print a backed-up file to stdout find Find a file, a directory or restic IDs forget Remove snapshots from the repository - generate Generate manual pages and auto-completion files (bash, fish, zsh, powershell) - help Help about any command init Initialize a new repository key Manage keys (passwords) list List objects in the repository @@ -41,11 +39,19 @@ Usage help is available: repair Repair the repository restore Extract the data from a snapshot rewrite Rewrite snapshots to exclude unwanted files - self-update Update the restic binary snapshots List all snapshots stats Scan the repository and show basic statistics tag Modify tags on snapshots unlock Remove locks other processes created + + Advanced Options: + features Print list of feature flags + options Print list of extended options + + Additional Commands: + generate Generate manual pages and auto-completion files (bash, fish, zsh, powershell) + help Help about any command + self-update Update the restic binary version Print version information Flags: From 8206cd19c8b7cb97e5b4cce68c05e6890c069921 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 22 Aug 2024 23:16:12 +0200 Subject: [PATCH 26/52] backend/retry: don't trip circuit breaker if context is canceled When the context used for a load operation is canceled, then the result is always an error independent of whether the file could be retrieved from the backend. Do not false positively trip the circuit breaker in this case. The old behavior was problematic when trying to lock a repository. When `Lock.checkForOtherLocks` listed multiple lock files in parallel and one of them fails to load, then all other loads were canceled. This cancelation was remembered by the circuit breaker, such that locking retries would fail. --- changelog/unreleased/pull-5011 | 10 ++++++++ internal/backend/retry/backend_retry.go | 5 ++-- internal/backend/retry/backend_retry_test.go | 24 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/pull-5011 diff --git a/changelog/unreleased/pull-5011 b/changelog/unreleased/pull-5011 new file mode 100644 index 000000000..8bd5ef532 --- /dev/null +++ b/changelog/unreleased/pull-5011 @@ -0,0 +1,10 @@ +Bugfix: Fix rare failures to retry locking a repository + +Restic 0.17.0 could in rare cases fail to retry locking a repository if +one of the lock files failed to load. The lock operation failed with error +`unable to create lock in backend: circuit breaker open for file ` + +The error handling has been fixed to correctly retry locking the repository. + +https://github.com/restic/restic/issues/5005 +https://github.com/restic/restic/pull/5011 diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index 8d0f42bfd..92c285c4b 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -209,9 +209,10 @@ func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offse return be.Backend.Load(ctx, h, length, offset, consumer) }) - if feature.Flag.Enabled(feature.BackendErrorRedesign) && err != nil && !be.IsPermanentError(err) { + if feature.Flag.Enabled(feature.BackendErrorRedesign) && err != nil && ctx.Err() == nil && !be.IsPermanentError(err) { // We've exhausted the retries, the file is likely inaccessible. By excluding permanent - // errors, not found or truncated files are not recorded. + // errors, not found or truncated files are not recorded. Also ignore errors if the context + // was canceled. be.failedLoads.LoadOrStore(key, time.Now()) } diff --git a/internal/backend/retry/backend_retry_test.go b/internal/backend/retry/backend_retry_test.go index fd76200d4..ffb8ae186 100644 --- a/internal/backend/retry/backend_retry_test.go +++ b/internal/backend/retry/backend_retry_test.go @@ -357,6 +357,30 @@ func TestBackendLoadCircuitBreaker(t *testing.T) { test.Equals(t, notFound, err, "expected circuit breaker to reset, got %v") } +func TestBackendLoadCircuitBreakerCancel(t *testing.T) { + cctx, cancel := context.WithCancel(context.Background()) + be := mock.NewBackend() + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + cancel() + return nil, errors.New("something") + } + nilRd := func(rd io.Reader) (err error) { + return nil + } + + TestFastRetries(t) + retryBackend := New(be, 2, nil, nil) + // canceling the context should not trip the circuit breaker + err := retryBackend.Load(cctx, backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Equals(t, context.Canceled, err, "unexpected error") + + // reset context and check that the cirucit breaker does not return an error + cctx, cancel = context.WithCancel(context.Background()) + defer cancel() + err = retryBackend.Load(cctx, backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Equals(t, context.Canceled, err, "unexpected error") +} + func TestBackendStatNotExists(t *testing.T) { // stat should not retry if the error matches IsNotExist notFound := errors.New("not found") From 6eece31dc3ba736f061b162de1498503b65265ff Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 23 Aug 2024 23:24:43 +0200 Subject: [PATCH 27/52] lock: introduce short delay between failed locking retries Failed locking attempts were immediately retried up to three times without any delay between the retries. If a lock file is not found while checking for other locks, with the reworked backend retries there is no delay between those retries. This is a problem if a backend requires a few seconds to reflect file deletions in the file listings. To work around this problem, introduce a short exponentially increasing delay between the retries. The number of retries is now increased to 4. This results in delays of 5, 10 and 20 seconds between the retries. --- .../unreleased/{pull-5011 => issue-5005} | 3 +++ internal/restic/lock.go | 27 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) rename changelog/unreleased/{pull-5011 => issue-5005} (76%) diff --git a/changelog/unreleased/pull-5011 b/changelog/unreleased/issue-5005 similarity index 76% rename from changelog/unreleased/pull-5011 rename to changelog/unreleased/issue-5005 index 8bd5ef532..90c164b07 100644 --- a/changelog/unreleased/pull-5011 +++ b/changelog/unreleased/issue-5005 @@ -5,6 +5,9 @@ one of the lock files failed to load. The lock operation failed with error `unable to create lock in backend: circuit breaker open for file ` The error handling has been fixed to correctly retry locking the repository. +In addition, restic now waits a few seconds between locking retries to +increase chances of success. https://github.com/restic/restic/issues/5005 https://github.com/restic/restic/pull/5011 +https://github.com/restic/restic/pull/5012 diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 49c7cedf2..969d0593d 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -103,10 +103,14 @@ func NewExclusiveLock(ctx context.Context, repo Unpacked) (*Lock, error) { var waitBeforeLockCheck = 200 * time.Millisecond +// delay increases by factor 2 on each retry +var initialWaitBetweenLockRetries = 5 * time.Second + // TestSetLockTimeout can be used to reduce the lock wait timeout for tests. func TestSetLockTimeout(t testing.TB, d time.Duration) { t.Logf("setting lock timeout to %v", d) waitBeforeLockCheck = d + initialWaitBetweenLockRetries = d } func newLock(ctx context.Context, repo Unpacked, excl bool) (*Lock, error) { @@ -170,8 +174,17 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { if l.lockID != nil { checkedIDs.Insert(*l.lockID) } + delay := initialWaitBetweenLockRetries // retry locking a few times - for i := 0; i < 3; i++ { + for i := 0; i < 4; i++ { + if i != 0 { + // sleep between retries to give backend some time to settle + if err := cancelableDelay(ctx, delay); err != nil { + return err + } + delay *= 2 + } + // Store updates in new IDSet to prevent data races var m sync.Mutex newCheckedIDs := NewIDSet(checkedIDs.List()...) @@ -213,6 +226,18 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { return err } +func cancelableDelay(ctx context.Context, delay time.Duration) error { + // delay next try a bit + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + return nil +} + // createLock acquires the lock by creating a file in the repository. func (l *Lock) createLock(ctx context.Context) (ID, error) { id, err := SaveJSONUnpacked(ctx, l.repo, LockFile, l) From 64d628bd75062da7ccdef8d58e53152cda1763c7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 25 Aug 2024 21:52:34 +0200 Subject: [PATCH 28/52] make timeout for slow requests configurable --- changelog/unreleased/issue-4970 | 13 +++++++++++++ cmd/restic/global.go | 1 + doc/faq.rst | 14 ++++++++++++++ internal/backend/http_transport.go | 9 ++++++++- 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-4970 diff --git a/changelog/unreleased/issue-4970 b/changelog/unreleased/issue-4970 new file mode 100644 index 000000000..524e91b75 --- /dev/null +++ b/changelog/unreleased/issue-4970 @@ -0,0 +1,13 @@ +Enhancement: Make timeout for stuck requests customizable + +Restic monitors connections to the backend to detect stuck requests. If a request +does not return any data within five minutes, restic assumes the request is stuck and +retries it. However, for large repositories it sometimes takes longer than that to +collect a list of all files, causing the following error: + +`List(data) returned error, retrying after 1s: [...]: request timeout` + +It is now possible to increase the timeout using the `--stuck-request-timeout` option. + +https://github.com/restic/restic/issues/4970 +https://github.com/restic/restic/pull/5014 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 22aa8a290..375b57f98 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -140,6 +140,7 @@ func init() { f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") f.StringVar(&globalOptions.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests") + f.DurationVar(&globalOptions.StuckRequestTimeout, "stuck-request-timeout", 5*time.Minute, "`duration` after which to retry stuck requests") // Use our "generate" command instead of the cobra provided "completion" command cmdRoot.CompletionOptions.DisableDefaultCmd = true diff --git a/doc/faq.rst b/doc/faq.rst index b26398f8c..74dd77d71 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -228,3 +228,17 @@ Restic backup command fails to find a valid file in Windows If the name of a file in Windows contains an invalid character, Restic will not be able to read the file. To solve this issue, consider renaming the particular file. + +What can I do in case of "request timeout" errors? +-------------------------------------------------- + +Restic monitors connections to the backend to detect stuck requests. If a request +does not return any data within five minutes, restic assumes the request is stuck and +retries it. However, for large repositories it sometimes takes longer than that to +collect a list of all files, causing the following error: + +:: + + List(data) returned error, retrying after 1s: [...]: request timeout + +In this case you can increase the timeout using the ``--stuck-request-timeout`` option. diff --git a/internal/backend/http_transport.go b/internal/backend/http_transport.go index 5162d3571..5a3856e41 100644 --- a/internal/backend/http_transport.go +++ b/internal/backend/http_transport.go @@ -31,6 +31,9 @@ type TransportOptions struct { // Specify Custom User-Agent for the http Client HTTPUserAgent string + + // Timeout after which to retry stuck requests + StuckRequestTimeout time.Duration } // readPEMCertKey reads a file and returns the PEM encoded certificate and key @@ -143,7 +146,11 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) { } if feature.Flag.Enabled(feature.BackendErrorRedesign) { - rt = newWatchdogRoundtripper(rt, 5*time.Minute, 128*1024) + if opts.StuckRequestTimeout == 0 { + opts.StuckRequestTimeout = 5 * time.Minute + } + + rt = newWatchdogRoundtripper(rt, opts.StuckRequestTimeout, 128*1024) } // wrap in the debug round tripper (if active) From 6b4f16f77bf70684c17912426f8a3fa2ce5f8ecb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 25 Aug 2024 23:52:33 +0200 Subject: [PATCH 29/52] doc/backup: link to exit code for scripting section --- doc/040_backup.rst | 7 ++++--- doc/075_scripting.rst | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index b53ae8d09..4c9a44c74 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -778,11 +778,12 @@ environment variables and configuration files; see their respective manuals. Exit status codes ***************** -Restic returns one of the following exit status codes after the backup command is run: +Restic returns an exit status code after the backup command is run: * 0 when the backup was successful (snapshot with all source files created) * 1 when there was a fatal error (no snapshot created) * 3 when some source files could not be read (incomplete snapshot with remaining files created) +* further exit codes are documented in :ref:`exit-codes`. Fatal errors occur for example when restic is unable to write to the backup destination, when there are network connectivity issues preventing successful communication, or when an invalid @@ -795,5 +796,5 @@ file read errors that occurred while running the backup. If there are errors of restic will still try to complete the backup run with all the other files, and create a snapshot that then contains all but the unreadable files. -One can use these exit status codes in scripts and other automation tools, to make them aware of -the outcome of the backup run. To manually inspect the exit code in e.g. Linux, run ``echo $?``. +For use of these exit status codes in scripts and other automation tools, see :ref:`exit-codes`. +To manually inspect the exit code in e.g. Linux, run ``echo $?``. diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 438eaa84f..9fa0da6d0 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -39,6 +39,8 @@ Note that restic will also return exit code ``1`` if a different error is encoun If there are no errors, restic will return a zero exit code and print the repository metadata. +.. _exit-codes: + Exit codes ********** From 5980daea64e3c1111957a819ed833390add58148 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 25 Aug 2024 23:53:12 +0200 Subject: [PATCH 30/52] doc/backup: move exit status codes section up --- doc/040_backup.rst | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 4c9a44c74..696b235cc 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -686,6 +686,30 @@ created as it would only be written at the very (successful) end of the backup operation. Previous snapshots will still be there and will still work. +Exit status codes +***************** + +Restic returns an exit status code after the backup command is run: + +* 0 when the backup was successful (snapshot with all source files created) +* 1 when there was a fatal error (no snapshot created) +* 3 when some source files could not be read (incomplete snapshot with remaining files created) +* further exit codes are documented in :ref:`exit-codes`. + +Fatal errors occur for example when restic is unable to write to the backup destination, when +there are network connectivity issues preventing successful communication, or when an invalid +password or command line argument is provided. When restic returns this exit status code, one +should not expect a snapshot to have been created. + +Source file read errors occur when restic fails to read one or more files or directories that +it was asked to back up, e.g. due to permission problems. Restic displays the number of source +file read errors that occurred while running the backup. If there are errors of this type, +restic will still try to complete the backup run with all the other files, and create a +snapshot that then contains all but the unreadable files. + +For use of these exit status codes in scripts and other automation tools, see :ref:`exit-codes`. +To manually inspect the exit code in e.g. Linux, run ``echo $?``. + Environment Variables ********************* @@ -774,27 +798,3 @@ See :ref:`caching` for the rules concerning cache locations when The external programs that restic may execute include ``rclone`` (for rclone backends) and ``ssh`` (for the SFTP backend). These may respond to further environment variables and configuration files; see their respective manuals. - -Exit status codes -***************** - -Restic returns an exit status code after the backup command is run: - -* 0 when the backup was successful (snapshot with all source files created) -* 1 when there was a fatal error (no snapshot created) -* 3 when some source files could not be read (incomplete snapshot with remaining files created) -* further exit codes are documented in :ref:`exit-codes`. - -Fatal errors occur for example when restic is unable to write to the backup destination, when -there are network connectivity issues preventing successful communication, or when an invalid -password or command line argument is provided. When restic returns this exit status code, one -should not expect a snapshot to have been created. - -Source file read errors occur when restic fails to read one or more files or directories that -it was asked to back up, e.g. due to permission problems. Restic displays the number of source -file read errors that occurred while running the backup. If there are errors of this type, -restic will still try to complete the backup run with all the other files, and create a -snapshot that then contains all but the unreadable files. - -For use of these exit status codes in scripts and other automation tools, see :ref:`exit-codes`. -To manually inspect the exit code in e.g. Linux, run ``echo $?``. From f451001f75bb3f7d946cc14962026df1d25f9d16 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 12:17:43 +0200 Subject: [PATCH 31/52] doc: use regional urls for Amazon S3 and add generic s3 provider section Split description for non-Amazon S3 providers into separate section. The section now also includes the `s3.bucket-lookup` extended option. Switch to using regional URLs for Amazon S3 to replace the need for setting the region. --- doc/030_preparing_a_new_repo.rst | 64 ++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 87975f9fa..462a66d75 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -249,28 +249,22 @@ while creating the bucket. $ export AWS_ACCESS_KEY_ID= $ export AWS_SECRET_ACCESS_KEY= +When using temporary credentials make sure to include the session token via +the environment variable ``AWS_SESSION_TOKEN``. + You can then easily initialize a repository that uses your Amazon S3 as -a backend. If the bucket does not exist it will be created in the -default location: +a backend. Make sure to use the endpoint for the correct region. The example +uses ``us-east-1``. If the bucket does not exist it will be created in that region: .. code-block:: console - $ restic -r s3:s3.amazonaws.com/bucket_name init + $ restic -r s3:s3.us-east-1.amazonaws.com/bucket_name init enter password for new repository: enter password again: - created restic repository eefee03bbd at s3:s3.amazonaws.com/bucket_name + created restic repository eefee03bbd at s3:s3.us-east-1.amazonaws.com/bucket_name Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. -If needed, you can manually specify the region to use by either setting the -environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option -parameter like ``-o s3.region="us-east-1"``. If the region is not specified, -the default region is used. Afterwards, the S3 server (at least for AWS, -``s3.amazonaws.com``) will redirect restic to the correct endpoint. - -When using temporary credentials make sure to include the session token via -then environment variable ``AWS_SESSION_TOKEN``. - Until version 0.8.0, restic used a default prefix of ``restic``, so the files in the bucket were placed in a directory named ``restic``. If you want to access a repository created with an older version of restic, specify the path @@ -278,25 +272,14 @@ after the bucket name like this: .. code-block:: console - $ restic -r s3:s3.amazonaws.com/bucket_name/restic [...] + $ restic -r s3:s3.us-east-1.amazonaws.com/bucket_name/restic [...] -For an S3-compatible server that is not Amazon (like Minio, see below), -or is only available via HTTP, you can specify the URL to the server -like this: ``s3:http://server:port/bucket_name``. - .. note:: restic expects `path-style URLs `__ - like for example ``s3.us-west-2.amazonaws.com/bucket_name``. + like for example ``s3.us-west-2.amazonaws.com/bucket_name`` for Amazon S3. Virtual-hosted–style URLs like ``bucket_name.s3.us-west-2.amazonaws.com``, where the bucket name is part of the hostname are not supported. These must be converted to path-style URLs instead, for example ``s3.us-west-2.amazonaws.com/bucket_name``. - -.. note:: Certain S3-compatible servers do not properly implement the - ``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these - backends, as a temporary workaround, you can provide the - ``-o s3.list-objects-v1=true`` option to use the older - ``ListObjects`` API instead. This option may be removed in future - versions of restic. - + See below for configuration options for S3-compatible storage from other providers. Minio Server ************ @@ -321,13 +304,38 @@ this command. .. code-block:: console - $ ./restic -r s3:http://localhost:9000/restic init + $ restic -r s3:http://localhost:9000/restic init enter password for new repository: enter password again: created restic repository 6ad29560f5 at s3:http://localhost:9000/restic1 Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. +S3-compatible Storage +********************* + +For an S3-compatible server that is not Amazon, you can specify the URL to the server +like this: ``s3:https://server:port/bucket_name``. + +If needed, you can manually specify the region to use by either setting the +environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option +parameter like ``-o s3.region="us-east-1"``. If the region is not specified, +the default region ``us-east-1`` is used. + +To select between path-style and virtual-hosted access, the extended option +``-o s3.bucket-lookup=auto`` can be used. It supports the following values: + +- ``auto``: Default behavior. Uses ``dns`` for Amazon and Google endpoints. Uses + ``path`` for all other endpoints +- ``dns``: Use virtual-hosted-style bucket access +- ``path``: Use path-style bucket access + +Certain S3-compatible servers do not properly implement the ``ListObjectsV2`` API, +most notably Ceph versions before v14.2.5. On these backends, as a temporary +workaround, you can provide the ``-o s3.list-objects-v1=true`` option to use the +older ``ListObjects`` API instead. This option may be removed in future versions +of restic. + Wasabi ************ From 262e85c37fa68d3af021d973a0b1569c304cf9ec Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 12:21:13 +0200 Subject: [PATCH 32/52] doc: shrink wasabi / alibaba cloud example Remove descriptions for both providers and shorten the example to the minimum. --- doc/030_preparing_a_new_repo.rst | 60 ++++++-------------------------- 1 file changed, 10 insertions(+), 50 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 462a66d75..a169f34cc 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -337,73 +337,33 @@ older ``ListObjects`` API instead. This option may be removed in future versions of restic. Wasabi -************ +****** -`Wasabi `__ is a low cost Amazon S3 conformant object storage provider. -Due to its S3 conformance, Wasabi can be used as a storage provider for a restic repository. +S3 storage from `Wasabi `__ can be used as follows. -- Create a Wasabi bucket using the `Wasabi Console `__. -- Determine the correct Wasabi service URL for your bucket `here `__. - -You must first setup the following environment variables with the -credentials of your Wasabi account. +- Determine the correct Wasabi service URL for your bucket `here `__. +- Set environment variables with the necessary account credentials .. code-block:: console $ export AWS_ACCESS_KEY_ID= $ export AWS_SECRET_ACCESS_KEY= - -Now you can easily initialize restic to use Wasabi as a backend with -this command. - -.. code-block:: console - - $ ./restic -r s3:https:/// init - enter password for new repository: - enter password again: - created restic repository xxxxxxxxxx at s3:https:/// - Please note that knowledge of your password is required to access - the repository. Losing your password means that your data is irrecoverably lost. + $ restic -r s3:https:/// init Alibaba Cloud (Aliyun) Object Storage System (OSS) ************************************************** -`Alibaba OSS `__ is an -encrypted, secure, cost-effective, and easy-to-use object storage -service that enables you to store, back up, and archive large amounts -of data in the cloud. +S3 storage from `Alibaba OSS `__ can be used as follows. -Alibaba OSS is S3 compatible so it can be used as a storage provider -for a restic repository with a couple of extra parameters. - -- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` -- You'll need the region name too - this will be something like ``oss-eu-west-1`` - -You must first setup the following environment variables with the -credentials of your Alibaba OSS account. +- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` +- You will need the region name too - this will be something like ``oss-eu-west-1`` +- Set environment variables with the necessary account credentials .. code-block:: console $ export AWS_ACCESS_KEY_ID= $ export AWS_SECRET_ACCESS_KEY= - -Now you can easily initialize restic to use Alibaba OSS as a backend with -this command. - -.. code-block:: console - - $ ./restic -o s3.bucket-lookup=dns -o s3.region= -r s3:https:/// init - enter password for new backend: - enter password again: - created restic backend xxxxxxxxxx at s3:https:/// - Please note that knowledge of your password is required to access - the repository. Losing your password means that your data is irrecoverably lost. - -For example with an actual endpoint: - -.. code-block:: console - - $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init + $ restic -o s3.bucket-lookup=dns -o s3.region= -r s3:https:/// init OpenStack Swift *************** From 97eb81564a68aeaca746c36929a9ec82dcb95cf6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 12:24:02 +0200 Subject: [PATCH 33/52] doc: fix typos --- doc/030_preparing_a_new_repo.rst | 2 +- doc/manual_rest.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index a169f34cc..fd5b31127 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -307,7 +307,7 @@ this command. $ restic -r s3:http://localhost:9000/restic init enter password for new repository: enter password again: - created restic repository 6ad29560f5 at s3:http://localhost:9000/restic1 + created restic repository 6ad29560f5 at s3:http://localhost:9000/restic Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 031f4fc52..d1e5817f3 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -8,7 +8,7 @@ Usage help is available: .. code-block:: console - $ ./restic --help + $ restic --help restic is a backup program which allows saving multiple revisions of files and directories in an encrypted repository stored on different backends. @@ -91,7 +91,7 @@ command: .. code-block:: console - $ ./restic backup --help + $ restic backup --help The "backup" command creates a new snapshot and saves the files and directories given as the arguments. From a45d21e2b9c72a35af96b046e46183f548945ef8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 14:38:32 +0200 Subject: [PATCH 34/52] doc: describe how to handle rewrite encoding error --- doc/045_working_with_repos.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 8dba8439f..f31e75c84 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -305,6 +305,13 @@ In order to preview the changes which ``rewrite`` would make, you can use the modifying the repository. Instead restic will only print the actions it would perform. +.. note:: The ``rewrite`` command verifies that it does not modify snapshots in + unexpected ways and fails with an ``cannot encode tree at "[...]" without loosing information`` + error otherwise. This can occur when rewriting a snapshot created by a newer + version of restic or some third-party implementation. + + To convert a snapshot into the format expected by the ``rewrite`` command + use ``restic repair snapshots ``. Modifying metadata of snapshots =============================== From 71e8068d863b299a96c5af02eb0403c011260d32 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 14:38:45 +0200 Subject: [PATCH 35/52] doc: mark S3 layout as deprecated --- doc/design.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/design.rst b/doc/design.rst index 7fb8b71b2..26f1f333f 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -126,8 +126,8 @@ the option ``-o local.layout=default``, valid values are ``default`` and ``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the s3 backend ``s3.layout``. -S3 Legacy Layout ----------------- +S3 Legacy Layout (deprecated) +----------------------------- Unfortunately during development the Amazon S3 backend uses slightly different paths (directory names use singular instead of plural for ``key``, @@ -152,8 +152,7 @@ the ``data`` directory. The S3 Legacy repository layout looks like this: /snapshot └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec -The S3 backend understands and accepts both forms, new backends are -always created with the default layout for compatibility reasons. +Restic 0.17 is the last version that supports the legacy layout. Pack Format =========== From 7ea558db999d401fd27b12f1285c64e4e788c940 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 14:40:04 +0200 Subject: [PATCH 36/52] doc: JSON encoder must be deterministic --- doc/design.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/design.rst b/doc/design.rst index 26f1f333f..d83ac8b91 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -233,7 +233,9 @@ Individual files for the index, locks or snapshots are encrypted and authenticated like Data and Tree Blobs, so the outer structure is ``IV || Ciphertext || MAC`` again. In repository format version 1 the plaintext always consists of a JSON document which must either be an -object or an array. +object or an array. The JSON encoder must deterministically encode the +document and should match the behavior of the Go standard library implementation +in ``encoding/json``. Repository format version 2 adds support for compression. The plaintext now starts with a header to indicate the encoding version to distinguish @@ -472,6 +474,10 @@ A snapshot references a tree by the SHA-256 hash of the JSON string representation of its contents. Trees and data are saved in pack files in a subdirectory of the directory ``data``. +The JSON encoder must deterministically encode the document and should +match the behavior of the Go standard library implementation in ``encoding/json``. +This ensures that trees can be properly deduplicated. + The command ``restic cat blob`` can be used to inspect the tree referenced above (piping the output of the command to ``jq .`` so that the JSON is indented): From 55ff4e046e029d7150295668d8d3ee139f13bf57 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 14:40:38 +0200 Subject: [PATCH 37/52] doc: full tree blob data structure is in the code --- doc/design.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/design.rst b/doc/design.rst index d83ac8b91..c974e997a 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -512,12 +512,11 @@ this metadata is generated: - The name is quoted using `strconv.Quote `__ before being saved. This handles non-unicode names, but also changes the representation of names containing ``"`` or ``\``. - - The filemode saved is the mode defined by `fs.FileMode `__ masked by ``os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky`` - -When the entry references a directory, the field ``subtree`` contains the plain text -ID of another tree object. +- When the entry references a directory, the field ``subtree`` contains the plain text + ID of another tree object. +- Check the implementation for a full struct definition. When the command ``restic cat blob`` is used, the plaintext ID is needed to print a tree. The tree referenced above can be dumped as follows: From 8828c76f92e29af09043015a41e547a74bef3b8d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 15:32:43 +0200 Subject: [PATCH 38/52] rest: improve handling of HTTP2 goaway The HTTP client can only retry HTTP2 requests after receiving a GOAWAY response if it can rewind the body. As we use a custom data type, explicitly provide an implementation of `GetBody`. --- changelog/unreleased/pull-5018 | 13 +++++++++++++ internal/backend/rest/rest.go | 6 ++++++ 2 files changed, 19 insertions(+) create mode 100644 changelog/unreleased/pull-5018 diff --git a/changelog/unreleased/pull-5018 b/changelog/unreleased/pull-5018 new file mode 100644 index 000000000..1b7b9f428 --- /dev/null +++ b/changelog/unreleased/pull-5018 @@ -0,0 +1,13 @@ +Bugfix: Improve HTTP2 support for rest backend + +If rest-server tried to gracefully shut down an HTTP2 connection still used by the client, +this could result in the following error. + +``` +http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error +``` + +This has been fixed. + +https://github.com/restic/restic/pull/5018 +https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367 diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 1af88ec3f..d0a08175b 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -143,6 +143,12 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR if err != nil { return errors.WithStack(err) } + req.GetBody = func() (io.ReadCloser, error) { + if err := rd.Rewind(); err != nil { + return nil, err + } + return io.NopCloser(rd), nil + } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", ContentTypeV2) From 3e4c1ea1966824bf5e0994beb412c56eb1bb3d5e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 19:31:21 +0200 Subject: [PATCH 39/52] fs: fix race condition in get/set security descriptor Calling `Load()` twice for an atomic variable can return different values each time. This resulted in trying to read the security descriptor with high privileges, but then not entering the code path to switch to low privileges when another thread has already done so concurrently. --- internal/fs/sd_windows.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/fs/sd_windows.go b/internal/fs/sd_windows.go index 0a73cbe53..bccf74992 100644 --- a/internal/fs/sd_windows.go +++ b/internal/fs/sd_windows.go @@ -48,13 +48,15 @@ func GetSecurityDescriptor(filePath string) (securityDescriptor *[]byte, err err var sd *windows.SECURITY_DESCRIPTOR - if lowerPrivileges.Load() { + // store original value to avoid unrelated changes in the error check + useLowerPrivileges := lowerPrivileges.Load() + if useLowerPrivileges { sd, err = getNamedSecurityInfoLow(filePath) } else { sd, err = getNamedSecurityInfoHigh(filePath) } if err != nil { - if !lowerPrivileges.Load() && isHandlePrivilegeNotHeldError(err) { + if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) { // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. lowerPrivileges.Store(true) sd, err = getNamedSecurityInfoLow(filePath) @@ -109,14 +111,16 @@ func SetSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { sacl = nil } - if lowerPrivileges.Load() { + // store original value to avoid unrelated changes in the error check + useLowerPrivileges := lowerPrivileges.Load() + if useLowerPrivileges { err = setNamedSecurityInfoLow(filePath, dacl) } else { err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl) } if err != nil { - if !lowerPrivileges.Load() && isHandlePrivilegeNotHeldError(err) { + if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) { // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. lowerPrivileges.Store(true) err = setNamedSecurityInfoLow(filePath, dacl) From ac5bc7c2f97eae7ebfe3526f1b1b0e4ebfcb82f4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 19:36:43 +0200 Subject: [PATCH 40/52] fs: fix error handling for retried get/set of security descriptor The retry code path did not filter `ERROR_NOT_SUPPORTED`. Just call the original function a second time to correctly follow the low privilege code path. --- internal/fs/sd_windows.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/internal/fs/sd_windows.go b/internal/fs/sd_windows.go index bccf74992..0004f1809 100644 --- a/internal/fs/sd_windows.go +++ b/internal/fs/sd_windows.go @@ -59,10 +59,7 @@ func GetSecurityDescriptor(filePath string) (securityDescriptor *[]byte, err err if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) { // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. lowerPrivileges.Store(true) - sd, err = getNamedSecurityInfoLow(filePath) - if err != nil { - return nil, fmt.Errorf("get low-level named security info failed with: %w", err) - } + return GetSecurityDescriptor(filePath) } else if errors.Is(err, windows.ERROR_NOT_SUPPORTED) { return nil, nil } else { @@ -123,10 +120,7 @@ func SetSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) { // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. lowerPrivileges.Store(true) - err = setNamedSecurityInfoLow(filePath, dacl) - if err != nil { - return fmt.Errorf("set low-level named security info failed with: %w", err) - } + return SetSecurityDescriptor(filePath, securityDescriptor) } else { return fmt.Errorf("set named security info failed with: %w", err) } From a12a6edfd1b1826f8787be00a094c6399ec2130f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 19:43:18 +0200 Subject: [PATCH 41/52] add changelog for security descriptor race condition --- changelog/unreleased/issue-5004 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 changelog/unreleased/issue-5004 diff --git a/changelog/unreleased/issue-5004 b/changelog/unreleased/issue-5004 new file mode 100644 index 000000000..529b65464 --- /dev/null +++ b/changelog/unreleased/issue-5004 @@ -0,0 +1,12 @@ +Bugfix: Fix spurious "A Required Privilege Is Not Held by the Client" error + +On Windows, creating a backup could sometimes print the following error + +``` +error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client. +``` + +This has been fixed. + +https://github.com/restic/restic/issues/5004 +https://github.com/restic/restic/pull/5019 From 1f4c9d280629f2afde9c6a294590f73962b37e83 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 29 Aug 2024 16:32:15 +0200 Subject: [PATCH 42/52] cache: correctly ignore files whose filename is no ID this can for example be the case for temporary files created by the backend implementation. --- internal/backend/cache/backend.go | 5 ++--- internal/backend/cache/backend_test.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/internal/backend/cache/backend.go b/internal/backend/cache/backend.go index 58b03dd38..3754266ba 100644 --- a/internal/backend/cache/backend.go +++ b/internal/backend/cache/backend.go @@ -231,9 +231,8 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen wrapFn := func(f backend.FileInfo) error { id, err := restic.ParseID(f.Name) if err != nil { - // returning error here since, if we cannot parse the ID, the file - // is invalid and the list must exit. - return err + // ignore files with invalid name + return nil } ids.Insert(id) diff --git a/internal/backend/cache/backend_test.go b/internal/backend/cache/backend_test.go index dca51c2bf..7f83e40cb 100644 --- a/internal/backend/cache/backend_test.go +++ b/internal/backend/cache/backend_test.go @@ -296,3 +296,20 @@ func TestAutomaticCacheClear(t *testing.T) { t.Errorf("cache doesn't have file2 after list") } } + +func TestAutomaticCacheClearInvalidFilename(t *testing.T) { + be := mem.New() + c := TestNewCache(t) + + data := test.Random(rand.Int(), 42) + h := backend.Handle{ + Type: backend.IndexFile, + Name: "tmp12345", + } + save(t, be, h, data) + + wbe := c.Wrap(be) + + // list all files in the backend + list(t, wbe, func(_ backend.FileInfo) error { return nil }) +} From 361fbbf58fc15f9604cf2922e425a129e50ede3b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 29 Aug 2024 16:33:18 +0200 Subject: [PATCH 43/52] Add temporary files repositories in integration tests This is intended to catch problems with temporary files stored in the backend, even if the responsible component forgets to test for those. --- cmd/restic/cmd_init_integration_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/restic/cmd_init_integration_test.go b/cmd/restic/cmd_init_integration_test.go index 9b5eed6e0..4795d5510 100644 --- a/cmd/restic/cmd_init_integration_test.go +++ b/cmd/restic/cmd_init_integration_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "os" + "path/filepath" "testing" "github.com/restic/restic/internal/repository" @@ -16,6 +18,11 @@ func testRunInit(t testing.TB, opts GlobalOptions) { rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil)) t.Logf("repository initialized at %v", opts.Repo) + + // create temporary junk files to verify that restic does not trip over them + for _, path := range []string{"index", "snapshots", "keys", "locks", filepath.Join("data", "00")} { + rtest.OK(t, os.WriteFile(filepath.Join(opts.Repo, path, "tmp12345"), []byte("junk file"), 0o600)) + } } func TestInitCopyChunkerParams(t *testing.T) { From 174f20dc4a80c5431901f276ddf2c4aed154f49d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 29 Aug 2024 16:35:48 +0200 Subject: [PATCH 44/52] use OrderedListOnceBackend where possible --- cmd/restic/cmd_prune_integration_test.go | 5 ++--- cmd/restic/integration_test.go | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 746eb5cc9..536ec40d8 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -146,10 +146,9 @@ func TestPruneWithDamagedRepository(t *testing.T) { env.gopts.backendTestHook = oldHook }() // prune should fail - rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + rtest.Equals(t, repository.ErrPacksMissing, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term) - }) == repository.ErrPacksMissing, - "prune should have reported index not complete error") + }), "prune should have reported index not complete error") } // Test repos for edge cases diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 4cecec6bc..df95031dc 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -80,7 +80,7 @@ func TestListOnce(t *testing.T) { defer cleanup() env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { - return newListOnceBackend(r), nil + return newOrderedListOnceBackend(r), nil } pruneOpts := PruneOptions{MaxUnused: "0"} checkOpts := CheckOptions{ReadData: true, CheckUnused: true} @@ -148,7 +148,7 @@ func TestFindListOnce(t *testing.T) { defer cleanup() env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { - return newListOnceBackend(r), nil + return newOrderedListOnceBackend(r), nil } testSetupBackupData(t, env) From ba71141f0a20318ef4c1a171673f146aadf0eb18 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 30 Aug 2024 11:25:51 +0200 Subject: [PATCH 45/52] backup: support specifying volume instead of path on Windows "C:" (volume name) versus "C:\" (path) --- changelog/unreleased/issue-2004 | 19 ++++++++++ internal/archiver/archiver.go | 7 +++- internal/archiver/archiver_test.go | 60 ++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-2004 diff --git a/changelog/unreleased/issue-2004 b/changelog/unreleased/issue-2004 new file mode 100644 index 000000000..45bc07ca8 --- /dev/null +++ b/changelog/unreleased/issue-2004 @@ -0,0 +1,19 @@ +Bugfix: Correctly handle passing volume name to `backup` command + +On Windows, when the specified backup target only included the volume +name without a trailing slash, for example, `C:`, then restoring the +resulting snapshot would result in an error. Note that using `C:\` +as backup target worked correctly. + +Specifying volume names now works correctly. + +To restore snapshots created before this bugfix, use the `:` +syntax. For a snapshot with ID `12345678` and a backup of `C:`, the following +command can be used: + +``` +restic restore 12345678:/C/C:./ --target output/folder +``` + +https://github.com/restic/restic/issues/2004 +https://github.com/restic/restic/pull/5028 diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e44151298..e7c346d3a 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -715,7 +715,12 @@ func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) { debug.Log("targets before resolving: %v", targets) result := make([]string, 0, len(targets)) for _, target := range targets { - target = filesys.Clean(target) + if target != "" && filesys.VolumeName(target) == target { + // special case to allow users to also specify a volume name "C:" instead of a path "C:\" + target = target + filesys.Separator() + } else { + target = filesys.Clean(target) + } pc, _ := pathComponents(filesys, target, false) if len(pc) > 0 { result = append(result, target) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index b519387db..c54f9ea33 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1448,6 +1448,66 @@ func TestArchiverSnapshot(t *testing.T) { } } +func TestResolveRelativeTargetsSpecial(t *testing.T) { + var tests = []struct { + name string + targets []string + expected []string + win bool + }{ + { + name: "basic relative path", + targets: []string{filepath.FromSlash("some/path")}, + expected: []string{filepath.FromSlash("some/path")}, + }, + { + name: "partial relative path", + targets: []string{filepath.FromSlash("../some/path")}, + expected: []string{filepath.FromSlash("../some/path")}, + }, + { + name: "basic absolute path", + targets: []string{filepath.FromSlash("/some/path")}, + expected: []string{filepath.FromSlash("/some/path")}, + }, + { + name: "volume name", + targets: []string{"C:"}, + expected: []string{"C:\\"}, + win: true, + }, + { + name: "volume root path", + targets: []string{"C:\\"}, + expected: []string{"C:\\"}, + win: true, + }, + { + name: "UNC path", + targets: []string{"\\\\server\\volume"}, + expected: []string{"\\\\server\\volume\\"}, + win: true, + }, + { + name: "UNC path with trailing slash", + targets: []string{"\\\\server\\volume\\"}, + expected: []string{"\\\\server\\volume\\"}, + win: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.win && runtime.GOOS != "windows" { + t.Skip("skip test on unix") + } + + targets, err := resolveRelativeTargets(&fs.Local{}, test.targets) + rtest.OK(t, err) + rtest.Equals(t, test.expected, targets) + }) + } +} + func TestArchiverSnapshotSelect(t *testing.T) { var tests = []struct { name string From 259caf942dd2b74ad629ba4d12d00cdeba58a973 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 30 Aug 2024 14:58:32 +0200 Subject: [PATCH 46/52] cleanup changelogs --- changelog/unreleased/issue-2004 | 10 +++++----- changelog/unreleased/issue-4795 | 7 ++++--- changelog/unreleased/issue-4934 | 12 +++++++----- changelog/unreleased/issue-4944 | 7 ++++--- changelog/unreleased/issue-4945 | 6 ++++-- changelog/unreleased/issue-4953 | 4 ++-- changelog/unreleased/issue-4957 | 8 ++++---- changelog/unreleased/issue-4969 | 4 ++-- changelog/unreleased/issue-4970 | 9 +++++---- changelog/unreleased/issue-4975 | 3 ++- changelog/unreleased/issue-5004 | 2 +- changelog/unreleased/issue-5005 | 15 +++++++++------ changelog/unreleased/pull-4958 | 6 +++--- changelog/unreleased/pull-4959 | 6 +++--- changelog/unreleased/pull-4977 | 17 +++++++++-------- changelog/unreleased/pull-4980 | 13 ++++++------- changelog/unreleased/pull-5018 | 8 ++++---- 17 files changed, 74 insertions(+), 63 deletions(-) diff --git a/changelog/unreleased/issue-2004 b/changelog/unreleased/issue-2004 index 45bc07ca8..a15651d75 100644 --- a/changelog/unreleased/issue-2004 +++ b/changelog/unreleased/issue-2004 @@ -1,15 +1,15 @@ -Bugfix: Correctly handle passing volume name to `backup` command +Bugfix: Correctly handle volume names in `backup` command on Windows On Windows, when the specified backup target only included the volume name without a trailing slash, for example, `C:`, then restoring the resulting snapshot would result in an error. Note that using `C:\` as backup target worked correctly. -Specifying volume names now works correctly. +Specifying volume names is now handled correctly. -To restore snapshots created before this bugfix, use the `:` -syntax. For a snapshot with ID `12345678` and a backup of `C:`, the following -command can be used: +To restore snapshots created before this bugfix, use the : +syntax. For example, to restore a snapshot with ID `12345678` that backed up +`C:`, use the following command: ``` restic restore 12345678:/C/C:./ --target output/folder diff --git a/changelog/unreleased/issue-4795 b/changelog/unreleased/issue-4795 index 084335f51..ff86f0931 100644 --- a/changelog/unreleased/issue-4795 +++ b/changelog/unreleased/issue-4795 @@ -1,7 +1,8 @@ -Enhancement: `restore --verify` shows progress with a progress bar +Enhancement: Display progress bar for `restore --verify` -If restore command was run with `--verify` restic didn't show any progress indication, now it shows a progress bar while 'verification' is running. -The progress bar is text only for now and doesn't respect `--json` flag. +When the `restore` command is run with `--verify`, it now displays a progress +bar while the verification step is running. The progress bar is not shown when +the `--json` flag is specified. https://github.com/restic/restic/issues/4795 https://github.com/restic/restic/pull/4989 diff --git a/changelog/unreleased/issue-4934 b/changelog/unreleased/issue-4934 index 6891ca204..56e22ad28 100644 --- a/changelog/unreleased/issue-4934 +++ b/changelog/unreleased/issue-4934 @@ -1,8 +1,10 @@ -Enhancement: Clear removed snapshots from local cache of the current host +Enhancement: Automatically clear removed snapshots from cache -Restic only removed snapshots from the cache on the host that runs the `forget` command. -On other hosts that use the same repository, the old snapshots remained in the cache. -Restic now, automatically clears old snapshots from the local cache of the current host. +Previously, restic only removed snapshots from the cache on the host where the +`forget` command was executed. On other hosts that use the same repository, the +old snapshots remained in the cache. + +Restic now automatically clears old snapshots from the local cache of the current host. https://github.com/restic/restic/issues/4934 -https://github.com/restic/restic/pull/4981 \ No newline at end of file +https://github.com/restic/restic/pull/4981 diff --git a/changelog/unreleased/issue-4944 b/changelog/unreleased/issue-4944 index 02f5ae341..738da8e57 100644 --- a/changelog/unreleased/issue-4944 +++ b/changelog/unreleased/issue-4944 @@ -1,8 +1,9 @@ Enhancement: Print JSON-formatted errors during `restore --json` -Restic printed any restore errors directly to the console as freeform -text messages, even with `--json`. Restic now prints them as JSON formatted -messages when `--json` is passed. +Restic printed any `restore` errors directly to the console as freeform text +messages, even when using the `--json` option. + +Now, when `--json` is specified, restic prints them as JSON formatted messages. https://github.com/restic/restic/issues/4944 https://github.com/restic/restic/pull/4946 diff --git a/changelog/unreleased/issue-4945 b/changelog/unreleased/issue-4945 index 7bbf69fac..024b30b21 100644 --- a/changelog/unreleased/issue-4945 +++ b/changelog/unreleased/issue-4945 @@ -1,7 +1,9 @@ Bugfix: Include missing backup error text with `--json` -Restic was not actually providing the text of an error message during -backup if `--json` was passed, instead only printing `"error": {}`. +Previously, when running a backup with the `--json` option, restic failed to +include the actual error message in the output, resulting in `"error": {}` +being displayed. + Restic now includes the error text in JSON output. https://github.com/restic/restic/issues/4945 diff --git a/changelog/unreleased/issue-4953 b/changelog/unreleased/issue-4953 index 78a266aff..c542377fc 100644 --- a/changelog/unreleased/issue-4953 +++ b/changelog/unreleased/issue-4953 @@ -1,7 +1,7 @@ Bugfix: Correctly handle long paths on older Windows versions -When using older Windows versions, like Windows Server 2012, restic 0.17.0 -failed to back up files with long paths. This has been fixed. +On older Windows versions, like Windows Server 2012, restic 0.17.0 failed to +back up files with long paths. This problem has now been resolved. https://github.com/restic/restic/issues/4953 https://github.com/restic/restic/pull/4954 diff --git a/changelog/unreleased/issue-4957 b/changelog/unreleased/issue-4957 index d18e28ec9..5da7463ae 100644 --- a/changelog/unreleased/issue-4957 +++ b/changelog/unreleased/issue-4957 @@ -1,8 +1,8 @@ -Bugfix: Fix delayed cancelation of some commands +Bugfix: Fix delayed cancellation of certain commands -Since restic 0.17.0, some commands no longer promptly reacted to being canceled -via Ctrl-C (SIGINT) and continued to run for a limited amount of time. The most -affected commands were `diff`,`find`, `ls`, `stats` and `rewrite`. +Since restic 0.17.0, some commands did not immediately respond to cancellation +via Ctrl-C (SIGINT) and continued running for a short period. The most affected +commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This has been fixed. diff --git a/changelog/unreleased/issue-4969 b/changelog/unreleased/issue-4969 index ce76a7389..9015c2eab 100644 --- a/changelog/unreleased/issue-4969 +++ b/changelog/unreleased/issue-4969 @@ -1,7 +1,7 @@ Bugfix: Correctly restore timestamp for files with resource forks on macOS -On macOS, timestamps were incorrectly restored for files with resource forks. -This has been fixed. +On macOS, timestamps were not restored for files with resource forks. This has +been fixed. https://github.com/restic/restic/issues/4969 https://github.com/restic/restic/pull/5006 diff --git a/changelog/unreleased/issue-4970 b/changelog/unreleased/issue-4970 index 524e91b75..2fc9300c9 100644 --- a/changelog/unreleased/issue-4970 +++ b/changelog/unreleased/issue-4970 @@ -1,9 +1,10 @@ Enhancement: Make timeout for stuck requests customizable -Restic monitors connections to the backend to detect stuck requests. If a request -does not return any data within five minutes, restic assumes the request is stuck and -retries it. However, for large repositories it sometimes takes longer than that to -collect a list of all files, causing the following error: +Restic monitors connections to the backend to detect stuck requests. If a +request does not return any data within five minutes, restic assumes the +request is stuck and retries it. However, for large repositories this timeout +might be insufficient to collect a list of all files, causing the following +error: `List(data) returned error, retrying after 1s: [...]: request timeout` diff --git a/changelog/unreleased/issue-4975 b/changelog/unreleased/issue-4975 index 0e29935f5..2503f46da 100644 --- a/changelog/unreleased/issue-4975 +++ b/changelog/unreleased/issue-4975 @@ -1,6 +1,7 @@ Bugfix: Prevent `backup --stdin-from-command` from panicking -If --stdin-from-command is used, restic now checks whether there is a command behind it. +Restic would previously crash if `--stdin-from-command` was specified without +providing a command. This issue has been fixed. https://github.com/restic/restic/issues/4975 https://github.com/restic/restic/pull/4976 diff --git a/changelog/unreleased/issue-5004 b/changelog/unreleased/issue-5004 index 529b65464..a0df9478e 100644 --- a/changelog/unreleased/issue-5004 +++ b/changelog/unreleased/issue-5004 @@ -1,6 +1,6 @@ Bugfix: Fix spurious "A Required Privilege Is Not Held by the Client" error -On Windows, creating a backup could sometimes print the following error +On Windows, creating a backup could sometimes trigger the following error: ``` error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client. diff --git a/changelog/unreleased/issue-5005 b/changelog/unreleased/issue-5005 index 90c164b07..eb712b967 100644 --- a/changelog/unreleased/issue-5005 +++ b/changelog/unreleased/issue-5005 @@ -1,12 +1,15 @@ Bugfix: Fix rare failures to retry locking a repository -Restic 0.17.0 could in rare cases fail to retry locking a repository if -one of the lock files failed to load. The lock operation failed with error -`unable to create lock in backend: circuit breaker open for file ` +Restic 0.17.0 could in rare cases fail to retry locking a repository if one of +the lock files failed to load, resulting in the error: -The error handling has been fixed to correctly retry locking the repository. -In addition, restic now waits a few seconds between locking retries to -increase chances of success. +``` +unable to create lock in backend: circuit breaker open for file +``` + +This issue has been addressed. The error handling now properly retries the +locking operation. In addition, restic waits a few seconds between locking +retries to increase chances of successful locking. https://github.com/restic/restic/issues/5005 https://github.com/restic/restic/pull/5011 diff --git a/changelog/unreleased/pull-4958 b/changelog/unreleased/pull-4958 index bbb28a97b..02574ad33 100644 --- a/changelog/unreleased/pull-4958 +++ b/changelog/unreleased/pull-4958 @@ -1,7 +1,7 @@ Bugfix: Don't ignore metadata-setting errors during restore -Restic was accidentally ignoring errors when setting timestamps, -attributes, or file modes during restore. It will now report those -errors (unless it's just a permission error when not running as root). +Restic ignored errors when setting timestamps, attributes, or file modes during +a restore. It now reports those, except for permission errors when running +without root privileges. https://github.com/restic/restic/pull/4958 diff --git a/changelog/unreleased/pull-4959 b/changelog/unreleased/pull-4959 index 120527e22..80b2780b2 100644 --- a/changelog/unreleased/pull-4959 +++ b/changelog/unreleased/pull-4959 @@ -1,6 +1,6 @@ -Enhancement: Return exit code 12 for "bad password" +Enhancement: Return exit code 12 for "bad password" errors -Restic now returns exit code 12 when it can't open the repository -because of a bad password. +Restic now returns exit code 12 when it cannot open the repository due to an +incorrect password. https://github.com/restic/restic/pull/4959 diff --git a/changelog/unreleased/pull-4977 b/changelog/unreleased/pull-4977 index 702df29a7..85e4091c5 100644 --- a/changelog/unreleased/pull-4977 +++ b/changelog/unreleased/pull-4977 @@ -1,14 +1,15 @@ -Change: let `backup` store files with incomplete metadata +Change: `backup` includes files with incomplete metadata -If restic failed to read the extended metadata for a file or folder while -creating a backup, then the file or folder was not included in the resulting -snapshot. Instead, only a warning message was printed along with exiting -with exit code 3. +If restic failed to read extended metadata for a file or folder during a +backup, then the file or folder was not included in the resulting snapshot. +Instead, a warning message was printed along with returning exit code 3 once +the backup was finished. + +Now, restic also includes items for which the extended metadata could not be +read in a snapshot. The warning message has been updated to: -Now, restic also includes items for which the extended metadata could not -be read in a snapshot. The warning message has been changed to read ``` -incomplete metadata for /path/to/file: details on error +incomplete metadata for /path/to/file:
``` https://github.com/restic/restic/issues/4953 diff --git a/changelog/unreleased/pull-4980 b/changelog/unreleased/pull-4980 index 5713db7a2..4b1de54bf 100644 --- a/changelog/unreleased/pull-4980 +++ b/changelog/unreleased/pull-4980 @@ -1,11 +1,10 @@ -Bugfix: Skip EA processing in Windows for volumes that do not support EA +Bugfix: Skip extended attribute processing on unsupported Windows volumes -Restic was failing to backup files on some windows paths like network -drives because of errors while fetching extended attributes. -Either they return error codes like windows.E_NOT_SET or -windows.ERROR_INVALID_FUNCTION or it results in slower backups. -Restic now completely skips the attempt to fetch extended attributes -for such volumes where it is not supported. +For restic 0.17.0, backups of certain Windows paths, such as network drives, +failed due to errors while fetching extended attributes. + +Restic now skips extended attribute processing for volumes where they are not +supported. https://github.com/restic/restic/pull/4980 https://github.com/restic/restic/pull/4998 diff --git a/changelog/unreleased/pull-5018 b/changelog/unreleased/pull-5018 index 1b7b9f428..84f36355c 100644 --- a/changelog/unreleased/pull-5018 +++ b/changelog/unreleased/pull-5018 @@ -1,13 +1,13 @@ -Bugfix: Improve HTTP2 support for rest backend +Bugfix: Improve HTTP/2 support for REST backend -If rest-server tried to gracefully shut down an HTTP2 connection still used by the client, -this could result in the following error. +If `rest-server` tried to gracefully shut down an HTTP/2 connection still in +use by the client, it could result in the following error: ``` http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error ``` -This has been fixed. +This issue has been resolved. https://github.com/restic/restic/pull/5018 https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367 From 5d658f216c1c8f9e53c8be65c703df046a1bc834 Mon Sep 17 00:00:00 2001 From: "Leo R. Lundgren" Date: Sat, 31 Aug 2024 17:28:24 +0200 Subject: [PATCH 47/52] doc: Polish unreleased changelogs --- changelog/unreleased/issue-2004 | 9 ++++----- changelog/unreleased/issue-4934 | 3 ++- changelog/unreleased/issue-4944 | 4 ++-- changelog/unreleased/issue-4945 | 2 +- changelog/unreleased/issue-4957 | 4 +--- changelog/unreleased/issue-4969 | 2 +- changelog/unreleased/issue-4970 | 3 ++- changelog/unreleased/issue-4975 | 2 +- changelog/unreleased/issue-5004 | 2 +- changelog/unreleased/issue-5005 | 2 +- changelog/unreleased/pull-4958 | 6 +++--- changelog/unreleased/pull-4977 | 2 +- changelog/unreleased/pull-4980 | 2 +- changelog/unreleased/pull-5018 | 2 +- 14 files changed, 22 insertions(+), 23 deletions(-) diff --git a/changelog/unreleased/issue-2004 b/changelog/unreleased/issue-2004 index a15651d75..5372eeb8c 100644 --- a/changelog/unreleased/issue-2004 +++ b/changelog/unreleased/issue-2004 @@ -5,11 +5,10 @@ name without a trailing slash, for example, `C:`, then restoring the resulting snapshot would result in an error. Note that using `C:\` as backup target worked correctly. -Specifying volume names is now handled correctly. - -To restore snapshots created before this bugfix, use the : -syntax. For example, to restore a snapshot with ID `12345678` that backed up -`C:`, use the following command: +Specifying volume names is now handled correctly. To restore snapshots +created before this bugfix, use the : syntax. For +example, to restore a snapshot with ID `12345678` that backed up `C:`, +use the following command: ``` restic restore 12345678:/C/C:./ --target output/folder diff --git a/changelog/unreleased/issue-4934 b/changelog/unreleased/issue-4934 index 56e22ad28..df77109a7 100644 --- a/changelog/unreleased/issue-4934 +++ b/changelog/unreleased/issue-4934 @@ -4,7 +4,8 @@ Previously, restic only removed snapshots from the cache on the host where the `forget` command was executed. On other hosts that use the same repository, the old snapshots remained in the cache. -Restic now automatically clears old snapshots from the local cache of the current host. +Restic now automatically clears old snapshots from the local cache of the +current host. https://github.com/restic/restic/issues/4934 https://github.com/restic/restic/pull/4981 diff --git a/changelog/unreleased/issue-4944 b/changelog/unreleased/issue-4944 index 738da8e57..95ae24c03 100644 --- a/changelog/unreleased/issue-4944 +++ b/changelog/unreleased/issue-4944 @@ -1,7 +1,7 @@ Enhancement: Print JSON-formatted errors during `restore --json` -Restic printed any `restore` errors directly to the console as freeform text -messages, even when using the `--json` option. +Restic used to print any `restore` errors directly to the console as freeform +text messages, even when using the `--json` option. Now, when `--json` is specified, restic prints them as JSON formatted messages. diff --git a/changelog/unreleased/issue-4945 b/changelog/unreleased/issue-4945 index 024b30b21..a7a483fed 100644 --- a/changelog/unreleased/issue-4945 +++ b/changelog/unreleased/issue-4945 @@ -4,7 +4,7 @@ Previously, when running a backup with the `--json` option, restic failed to include the actual error message in the output, resulting in `"error": {}` being displayed. -Restic now includes the error text in JSON output. +This has now been fixed, and restic now includes the error text in JSON output. https://github.com/restic/restic/issues/4945 https://github.com/restic/restic/pull/4946 diff --git a/changelog/unreleased/issue-4957 b/changelog/unreleased/issue-4957 index 5da7463ae..59c73b5c7 100644 --- a/changelog/unreleased/issue-4957 +++ b/changelog/unreleased/issue-4957 @@ -2,9 +2,7 @@ Bugfix: Fix delayed cancellation of certain commands Since restic 0.17.0, some commands did not immediately respond to cancellation via Ctrl-C (SIGINT) and continued running for a short period. The most affected -commands were `diff`,`find`, `ls`, `stats` and `rewrite`. - -This has been fixed. +commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This is now resolved. https://github.com/restic/restic/issues/4957 https://github.com/restic/restic/pull/4960 diff --git a/changelog/unreleased/issue-4969 b/changelog/unreleased/issue-4969 index 9015c2eab..d92392a20 100644 --- a/changelog/unreleased/issue-4969 +++ b/changelog/unreleased/issue-4969 @@ -1,7 +1,7 @@ Bugfix: Correctly restore timestamp for files with resource forks on macOS On macOS, timestamps were not restored for files with resource forks. This has -been fixed. +now been fixed. https://github.com/restic/restic/issues/4969 https://github.com/restic/restic/pull/5006 diff --git a/changelog/unreleased/issue-4970 b/changelog/unreleased/issue-4970 index 2fc9300c9..422ae3c25 100644 --- a/changelog/unreleased/issue-4970 +++ b/changelog/unreleased/issue-4970 @@ -8,7 +8,8 @@ error: `List(data) returned error, retrying after 1s: [...]: request timeout` -It is now possible to increase the timeout using the `--stuck-request-timeout` option. +It is now possible to increase the timeout using the `--stuck-request-timeout` +option. https://github.com/restic/restic/issues/4970 https://github.com/restic/restic/pull/5014 diff --git a/changelog/unreleased/issue-4975 b/changelog/unreleased/issue-4975 index 2503f46da..614642c06 100644 --- a/changelog/unreleased/issue-4975 +++ b/changelog/unreleased/issue-4975 @@ -1,7 +1,7 @@ Bugfix: Prevent `backup --stdin-from-command` from panicking Restic would previously crash if `--stdin-from-command` was specified without -providing a command. This issue has been fixed. +providing a command. This issue has now been fixed. https://github.com/restic/restic/issues/4975 https://github.com/restic/restic/pull/4976 diff --git a/changelog/unreleased/issue-5004 b/changelog/unreleased/issue-5004 index a0df9478e..72e98a9a4 100644 --- a/changelog/unreleased/issue-5004 +++ b/changelog/unreleased/issue-5004 @@ -6,7 +6,7 @@ On Windows, creating a backup could sometimes trigger the following error: error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client. ``` -This has been fixed. +This has now been fixed. https://github.com/restic/restic/issues/5004 https://github.com/restic/restic/pull/5019 diff --git a/changelog/unreleased/issue-5005 b/changelog/unreleased/issue-5005 index eb712b967..16ac83b4a 100644 --- a/changelog/unreleased/issue-5005 +++ b/changelog/unreleased/issue-5005 @@ -7,7 +7,7 @@ the lock files failed to load, resulting in the error: unable to create lock in backend: circuit breaker open for file ``` -This issue has been addressed. The error handling now properly retries the +This issue has now been addressed. The error handling now properly retries the locking operation. In addition, restic waits a few seconds between locking retries to increase chances of successful locking. diff --git a/changelog/unreleased/pull-4958 b/changelog/unreleased/pull-4958 index 02574ad33..dae9b2c8e 100644 --- a/changelog/unreleased/pull-4958 +++ b/changelog/unreleased/pull-4958 @@ -1,7 +1,7 @@ Bugfix: Don't ignore metadata-setting errors during restore -Restic ignored errors when setting timestamps, attributes, or file modes during -a restore. It now reports those, except for permission errors when running -without root privileges. +Previously, restic used to ignore errors when setting timestamps, attributes, +or file modes during a restore. It now reports those errors, except for +permission related errors when running without root privileges. https://github.com/restic/restic/pull/4958 diff --git a/changelog/unreleased/pull-4977 b/changelog/unreleased/pull-4977 index 85e4091c5..781576a56 100644 --- a/changelog/unreleased/pull-4977 +++ b/changelog/unreleased/pull-4977 @@ -1,4 +1,4 @@ -Change: `backup` includes files with incomplete metadata +Change: Also back up files with incomplete metadata If restic failed to read extended metadata for a file or folder during a backup, then the file or folder was not included in the resulting snapshot. diff --git a/changelog/unreleased/pull-4980 b/changelog/unreleased/pull-4980 index 4b1de54bf..b51ee8d59 100644 --- a/changelog/unreleased/pull-4980 +++ b/changelog/unreleased/pull-4980 @@ -1,6 +1,6 @@ Bugfix: Skip extended attribute processing on unsupported Windows volumes -For restic 0.17.0, backups of certain Windows paths, such as network drives, +With restic 0.17.0, backups of certain Windows paths, such as network drives, failed due to errors while fetching extended attributes. Restic now skips extended attribute processing for volumes where they are not diff --git a/changelog/unreleased/pull-5018 b/changelog/unreleased/pull-5018 index 84f36355c..ca600c3e1 100644 --- a/changelog/unreleased/pull-5018 +++ b/changelog/unreleased/pull-5018 @@ -7,7 +7,7 @@ use by the client, it could result in the following error: http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error ``` -This issue has been resolved. +This issue has now been resolved. https://github.com/restic/restic/pull/5018 https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367 From 17e54b04abad9eeb3143aec93e7f04ec0555671f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 5 Sep 2024 21:25:19 +0200 Subject: [PATCH 48/52] Prepare changelog for 0.17.1 --- changelog/{unreleased => 0.17.1_2024-09-05}/issue-2004 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4795 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4934 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4944 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4945 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4953 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4957 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4969 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4970 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-4975 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-5004 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/issue-5005 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/pull-4958 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/pull-4959 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/pull-4977 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/pull-4980 | 0 changelog/{unreleased => 0.17.1_2024-09-05}/pull-5018 | 0 17 files changed, 0 insertions(+), 0 deletions(-) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-2004 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4795 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4934 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4944 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4945 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4953 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4957 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4969 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4970 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-4975 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-5004 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/issue-5005 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/pull-4958 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/pull-4959 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/pull-4977 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/pull-4980 (100%) rename changelog/{unreleased => 0.17.1_2024-09-05}/pull-5018 (100%) diff --git a/changelog/unreleased/issue-2004 b/changelog/0.17.1_2024-09-05/issue-2004 similarity index 100% rename from changelog/unreleased/issue-2004 rename to changelog/0.17.1_2024-09-05/issue-2004 diff --git a/changelog/unreleased/issue-4795 b/changelog/0.17.1_2024-09-05/issue-4795 similarity index 100% rename from changelog/unreleased/issue-4795 rename to changelog/0.17.1_2024-09-05/issue-4795 diff --git a/changelog/unreleased/issue-4934 b/changelog/0.17.1_2024-09-05/issue-4934 similarity index 100% rename from changelog/unreleased/issue-4934 rename to changelog/0.17.1_2024-09-05/issue-4934 diff --git a/changelog/unreleased/issue-4944 b/changelog/0.17.1_2024-09-05/issue-4944 similarity index 100% rename from changelog/unreleased/issue-4944 rename to changelog/0.17.1_2024-09-05/issue-4944 diff --git a/changelog/unreleased/issue-4945 b/changelog/0.17.1_2024-09-05/issue-4945 similarity index 100% rename from changelog/unreleased/issue-4945 rename to changelog/0.17.1_2024-09-05/issue-4945 diff --git a/changelog/unreleased/issue-4953 b/changelog/0.17.1_2024-09-05/issue-4953 similarity index 100% rename from changelog/unreleased/issue-4953 rename to changelog/0.17.1_2024-09-05/issue-4953 diff --git a/changelog/unreleased/issue-4957 b/changelog/0.17.1_2024-09-05/issue-4957 similarity index 100% rename from changelog/unreleased/issue-4957 rename to changelog/0.17.1_2024-09-05/issue-4957 diff --git a/changelog/unreleased/issue-4969 b/changelog/0.17.1_2024-09-05/issue-4969 similarity index 100% rename from changelog/unreleased/issue-4969 rename to changelog/0.17.1_2024-09-05/issue-4969 diff --git a/changelog/unreleased/issue-4970 b/changelog/0.17.1_2024-09-05/issue-4970 similarity index 100% rename from changelog/unreleased/issue-4970 rename to changelog/0.17.1_2024-09-05/issue-4970 diff --git a/changelog/unreleased/issue-4975 b/changelog/0.17.1_2024-09-05/issue-4975 similarity index 100% rename from changelog/unreleased/issue-4975 rename to changelog/0.17.1_2024-09-05/issue-4975 diff --git a/changelog/unreleased/issue-5004 b/changelog/0.17.1_2024-09-05/issue-5004 similarity index 100% rename from changelog/unreleased/issue-5004 rename to changelog/0.17.1_2024-09-05/issue-5004 diff --git a/changelog/unreleased/issue-5005 b/changelog/0.17.1_2024-09-05/issue-5005 similarity index 100% rename from changelog/unreleased/issue-5005 rename to changelog/0.17.1_2024-09-05/issue-5005 diff --git a/changelog/unreleased/pull-4958 b/changelog/0.17.1_2024-09-05/pull-4958 similarity index 100% rename from changelog/unreleased/pull-4958 rename to changelog/0.17.1_2024-09-05/pull-4958 diff --git a/changelog/unreleased/pull-4959 b/changelog/0.17.1_2024-09-05/pull-4959 similarity index 100% rename from changelog/unreleased/pull-4959 rename to changelog/0.17.1_2024-09-05/pull-4959 diff --git a/changelog/unreleased/pull-4977 b/changelog/0.17.1_2024-09-05/pull-4977 similarity index 100% rename from changelog/unreleased/pull-4977 rename to changelog/0.17.1_2024-09-05/pull-4977 diff --git a/changelog/unreleased/pull-4980 b/changelog/0.17.1_2024-09-05/pull-4980 similarity index 100% rename from changelog/unreleased/pull-4980 rename to changelog/0.17.1_2024-09-05/pull-4980 diff --git a/changelog/unreleased/pull-5018 b/changelog/0.17.1_2024-09-05/pull-5018 similarity index 100% rename from changelog/unreleased/pull-5018 rename to changelog/0.17.1_2024-09-05/pull-5018 From d8870a2f731faa2709e569f0166e18218c0d2490 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 5 Sep 2024 21:25:20 +0200 Subject: [PATCH 49/52] Generate CHANGELOG.md for 0.17.1 --- CHANGELOG.md | 225 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a6926755..9a5393915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.17.1](#changelog-for-restic-0171-2024-09-05) * [Changelog for 0.17.0](#changelog-for-restic-0170-2024-07-26) * [Changelog for 0.16.5](#changelog-for-restic-0165-2024-07-01) * [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) @@ -35,6 +36,230 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.17.1 (2024-09-05) +The following sections list the changes in restic 0.17.1 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #2004: Correctly handle volume names in `backup` command on Windows + * Fix #4945: Include missing backup error text with `--json` + * Fix #4953: Correctly handle long paths on older Windows versions + * Fix #4957: Fix delayed cancellation of certain commands + * Fix #4958: Don't ignore metadata-setting errors during restore + * Fix #4969: Correctly restore timestamp for files with resource forks on macOS + * Fix #4975: Prevent `backup --stdin-from-command` from panicking + * Fix #4980: Skip extended attribute processing on unsupported Windows volumes + * Fix #5004: Fix spurious "A Required Privilege Is Not Held by the Client" error + * Fix #5005: Fix rare failures to retry locking a repository + * Fix #5018: Improve HTTP/2 support for REST backend + * Chg #4953: Also back up files with incomplete metadata + * Enh #4795: Display progress bar for `restore --verify` + * Enh #4934: Automatically clear removed snapshots from cache + * Enh #4944: Print JSON-formatted errors during `restore --json` + * Enh #4959: Return exit code 12 for "bad password" errors + * Enh #4970: Make timeout for stuck requests customizable + +## Details + + * Bugfix #2004: Correctly handle volume names in `backup` command on Windows + + On Windows, when the specified backup target only included the volume name + without a trailing slash, for example, `C:`, then restoring the resulting + snapshot would result in an error. Note that using `C:\` as backup target worked + correctly. + + Specifying volume names is now handled correctly. To restore snapshots created + before this bugfix, use the : syntax. For example, to restore + a snapshot with ID `12345678` that backed up `C:`, use the following command: + + ``` + restic restore 12345678:/C/C:./ --target output/folder + ``` + + https://github.com/restic/restic/issues/2004 + https://github.com/restic/restic/pull/5028 + + * Bugfix #4945: Include missing backup error text with `--json` + + Previously, when running a backup with the `--json` option, restic failed to + include the actual error message in the output, resulting in `"error": {}` being + displayed. + + This has now been fixed, and restic now includes the error text in JSON output. + + https://github.com/restic/restic/issues/4945 + https://github.com/restic/restic/pull/4946 + + * Bugfix #4953: Correctly handle long paths on older Windows versions + + On older Windows versions, like Windows Server 2012, restic 0.17.0 failed to + back up files with long paths. This problem has now been resolved. + + https://github.com/restic/restic/issues/4953 + https://github.com/restic/restic/pull/4954 + + * Bugfix #4957: Fix delayed cancellation of certain commands + + Since restic 0.17.0, some commands did not immediately respond to cancellation + via Ctrl-C (SIGINT) and continued running for a short period. The most affected + commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This is now resolved. + + https://github.com/restic/restic/issues/4957 + https://github.com/restic/restic/pull/4960 + + * Bugfix #4958: Don't ignore metadata-setting errors during restore + + Previously, restic used to ignore errors when setting timestamps, attributes, or + file modes during a restore. It now reports those errors, except for permission + related errors when running without root privileges. + + https://github.com/restic/restic/pull/4958 + + * Bugfix #4969: Correctly restore timestamp for files with resource forks on macOS + + On macOS, timestamps were not restored for files with resource forks. This has + now been fixed. + + https://github.com/restic/restic/issues/4969 + https://github.com/restic/restic/pull/5006 + + * Bugfix #4975: Prevent `backup --stdin-from-command` from panicking + + Restic would previously crash if `--stdin-from-command` was specified without + providing a command. This issue has now been fixed. + + https://github.com/restic/restic/issues/4975 + https://github.com/restic/restic/pull/4976 + + * Bugfix #4980: Skip extended attribute processing on unsupported Windows volumes + + With restic 0.17.0, backups of certain Windows paths, such as network drives, + failed due to errors while fetching extended attributes. + + Restic now skips extended attribute processing for volumes where they are not + supported. + + https://github.com/restic/restic/issues/4955 + https://github.com/restic/restic/issues/4950 + https://github.com/restic/restic/pull/4980 + https://github.com/restic/restic/pull/4998 + + * Bugfix #5004: Fix spurious "A Required Privilege Is Not Held by the Client" error + + On Windows, creating a backup could sometimes trigger the following error: + + ``` + error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client. + ``` + + This has now been fixed. + + https://github.com/restic/restic/issues/5004 + https://github.com/restic/restic/pull/5019 + + * Bugfix #5005: Fix rare failures to retry locking a repository + + Restic 0.17.0 could in rare cases fail to retry locking a repository if one of + the lock files failed to load, resulting in the error: + + ``` + unable to create lock in backend: circuit breaker open for file + ``` + + This issue has now been addressed. The error handling now properly retries the + locking operation. In addition, restic waits a few seconds between locking + retries to increase chances of successful locking. + + https://github.com/restic/restic/issues/5005 + https://github.com/restic/restic/pull/5011 + https://github.com/restic/restic/pull/5012 + + * Bugfix #5018: Improve HTTP/2 support for REST backend + + If `rest-server` tried to gracefully shut down an HTTP/2 connection still in use + by the client, it could result in the following error: + + ``` + http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error + ``` + + This issue has now been resolved. + + https://github.com/restic/restic/pull/5018 + https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367 + + * Change #4953: Also back up files with incomplete metadata + + If restic failed to read extended metadata for a file or folder during a backup, + then the file or folder was not included in the resulting snapshot. Instead, a + warning message was printed along with returning exit code 3 once the backup was + finished. + + Now, restic also includes items for which the extended metadata could not be + read in a snapshot. The warning message has been updated to: + + ``` + incomplete metadata for /path/to/file:
+ ``` + + https://github.com/restic/restic/issues/4953 + https://github.com/restic/restic/pull/4977 + + * Enhancement #4795: Display progress bar for `restore --verify` + + When the `restore` command is run with `--verify`, it now displays a progress + bar while the verification step is running. The progress bar is not shown when + the `--json` flag is specified. + + https://github.com/restic/restic/issues/4795 + https://github.com/restic/restic/pull/4989 + + * Enhancement #4934: Automatically clear removed snapshots from cache + + Previously, restic only removed snapshots from the cache on the host where the + `forget` command was executed. On other hosts that use the same repository, the + old snapshots remained in the cache. + + Restic now automatically clears old snapshots from the local cache of the + current host. + + https://github.com/restic/restic/issues/4934 + https://github.com/restic/restic/pull/4981 + + * Enhancement #4944: Print JSON-formatted errors during `restore --json` + + Restic used to print any `restore` errors directly to the console as freeform + text messages, even when using the `--json` option. + + Now, when `--json` is specified, restic prints them as JSON formatted messages. + + https://github.com/restic/restic/issues/4944 + https://github.com/restic/restic/pull/4946 + + * Enhancement #4959: Return exit code 12 for "bad password" errors + + Restic now returns exit code 12 when it cannot open the repository due to an + incorrect password. + + https://github.com/restic/restic/pull/4959 + + * Enhancement #4970: Make timeout for stuck requests customizable + + Restic monitors connections to the backend to detect stuck requests. If a + request does not return any data within five minutes, restic assumes the request + is stuck and retries it. However, for large repositories this timeout might be + insufficient to collect a list of all files, causing the following error: + + `List(data) returned error, retrying after 1s: [...]: request timeout` + + It is now possible to increase the timeout using the `--stuck-request-timeout` + option. + + https://github.com/restic/restic/issues/4970 + https://github.com/restic/restic/pull/5014 + + # Changelog for restic 0.17.0 (2024-07-26) The following sections list the changes in restic 0.17.0 relevant to restic users. The changes are ordered by importance. From a98370cc9e9bbfb5bca3b73ec423ad2ad131213f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 5 Sep 2024 21:25:21 +0200 Subject: [PATCH 50/52] Update manpages and auto-completion --- doc/bash-completion.sh | 229 ++++++++++++++++++++++++++++++ doc/man/restic-backup.1 | 5 + doc/man/restic-cache.1 | 4 + doc/man/restic-cat.1 | 5 + doc/man/restic-check.1 | 5 + doc/man/restic-copy.1 | 7 +- doc/man/restic-diff.1 | 5 + doc/man/restic-dump.1 | 5 + doc/man/restic-features.1 | 146 +++++++++++++++++++ doc/man/restic-find.1 | 5 + doc/man/restic-forget.1 | 5 + doc/man/restic-generate.1 | 4 + doc/man/restic-init.1 | 6 +- doc/man/restic-key-add.1 | 5 + doc/man/restic-key-list.1 | 5 + doc/man/restic-key-passwd.1 | 5 + doc/man/restic-key-remove.1 | 5 + doc/man/restic-key.1 | 4 + doc/man/restic-list.1 | 5 + doc/man/restic-ls.1 | 5 + doc/man/restic-migrate.1 | 5 + doc/man/restic-mount.1 | 5 + doc/man/restic-options.1 | 135 ++++++++++++++++++ doc/man/restic-prune.1 | 5 + doc/man/restic-recover.1 | 5 + doc/man/restic-repair-index.1 | 5 + doc/man/restic-repair-packs.1 | 5 + doc/man/restic-repair-snapshots.1 | 5 + doc/man/restic-repair.1 | 4 + doc/man/restic-restore.1 | 5 + doc/man/restic-rewrite.1 | 5 + doc/man/restic-self-update.1 | 5 + doc/man/restic-snapshots.1 | 5 + doc/man/restic-stats.1 | 5 + doc/man/restic-tag.1 | 5 + doc/man/restic-unlock.1 | 4 + doc/man/restic-version.1 | 4 + doc/man/restic.1 | 6 +- 38 files changed, 680 insertions(+), 3 deletions(-) create mode 100644 doc/man/restic-features.1 create mode 100644 doc/man/restic-options.1 diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh index 9d64871ca..0517fdf7c 100644 --- a/doc/bash-completion.sh +++ b/doc/bash-completion.sh @@ -516,6 +516,8 @@ _restic_backup() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -592,6 +594,8 @@ _restic_cache() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -660,6 +664,8 @@ _restic_cat() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -667,6 +673,15 @@ _restic_cat() must_have_one_flag=() must_have_one_noun=() + must_have_one_noun+=("blob") + must_have_one_noun+=("config") + must_have_one_noun+=("index") + must_have_one_noun+=("key") + must_have_one_noun+=("lock") + must_have_one_noun+=("masterkey") + must_have_one_noun+=("pack") + must_have_one_noun+=("snapshot") + must_have_one_noun+=("tree") noun_aliases=() } @@ -736,6 +751,8 @@ _restic_check() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -840,6 +857,8 @@ _restic_copy() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -910,6 +929,8 @@ _restic_diff() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1004,6 +1025,78 @@ _restic_dump() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_features() +{ + last_command="restic_features" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1122,6 +1215,8 @@ _restic_find() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1298,6 +1393,8 @@ _restic_forget() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1386,6 +1483,8 @@ _restic_generate() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1450,6 +1549,8 @@ _restic_help() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1547,6 +1648,8 @@ _restic_init() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1629,6 +1732,8 @@ _restic_key_add() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1693,6 +1798,8 @@ _restic_key_help() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1762,6 +1869,8 @@ _restic_key_list() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1844,6 +1953,8 @@ _restic_key_passwd() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1912,6 +2023,8 @@ _restic_key_remove() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1985,6 +2098,8 @@ _restic_key() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2053,6 +2168,8 @@ _restic_list() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2145,6 +2262,8 @@ _restic_ls() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2217,6 +2336,8 @@ _restic_migrate() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2313,6 +2434,78 @@ _restic_mount() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_options() +{ + last_command="restic_options" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2403,6 +2596,8 @@ _restic_prune() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2471,6 +2666,8 @@ _restic_recover() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2535,6 +2732,8 @@ _restic_repair_help() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2606,6 +2805,8 @@ _restic_repair_index() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2674,6 +2875,8 @@ _restic_repair_packs() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2762,6 +2965,8 @@ _restic_repair_snapshots() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2834,6 +3039,8 @@ _restic_repair() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2970,6 +3177,8 @@ _restic_restore() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3084,6 +3293,8 @@ _restic_rewrite() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3156,6 +3367,8 @@ _restic_self-update() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3252,6 +3465,8 @@ _restic_snapshots() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3288,6 +3503,8 @@ _restic_stats() local_nonpersistent_flags+=("-H") flags+=("--mode=") two_word_flags+=("--mode") + flags_with_completion+=("--mode") + flags_completion+=("__restic_handle_go_custom_completion") local_nonpersistent_flags+=("--mode") local_nonpersistent_flags+=("--mode=") flags+=("--path=") @@ -3338,6 +3555,8 @@ _restic_stats() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3432,6 +3651,8 @@ _restic_tag() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3502,6 +3723,8 @@ _restic_unlock() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3570,6 +3793,8 @@ _restic_version() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3594,6 +3819,7 @@ _restic_root_command() commands+=("copy") commands+=("diff") commands+=("dump") + commands+=("features") commands+=("find") commands+=("forget") commands+=("generate") @@ -3604,6 +3830,7 @@ _restic_root_command() commands+=("ls") commands+=("migrate") commands+=("mount") + commands+=("options") commands+=("prune") commands+=("recover") commands+=("repair") @@ -3666,6 +3893,8 @@ _restic_root_command() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index cda4aadff..a84b955ba 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -24,6 +24,7 @@ Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -229,6 +230,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index f868b8a6b..fb23fe8a9 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -129,6 +129,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index 2298c58cf..cab1b85a5 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -119,6 +120,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index c0d1b07a8..60d17a313 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -136,6 +137,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index 63b67e5e7..96c394139 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -36,12 +36,13 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS .PP \fB--from-insecure-no-password\fP[=false] - use an empty password for the source repository, must be passed to every restic command (insecure) + use an empty password for the source repository (insecure) .PP \fB--from-key-hint\fP="" @@ -169,6 +170,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index f4ffa2737..f4c8a1d14 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -49,6 +49,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -150,6 +151,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index 00cb3c8b6..657570f6d 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -34,6 +34,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -151,6 +152,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-features.1 b/doc/man/restic-features.1 new file mode 100644 index 000000000..b288f655a --- /dev/null +++ b/doc/man/restic-features.1 @@ -0,0 +1,146 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-features - Print list of feature flags + + +.SH SYNOPSIS +.PP +\fBrestic features [flags]\fP + + +.SH DESCRIPTION +.PP +The "features" command prints a list of supported feature flags. + +.PP +To pass feature flags to restic, set the RESTIC_FEATURES environment variable +to "featureA=true,featureB=false". Specifying an unknown feature flag is an error. + +.PP +A feature can either be in alpha, beta, stable or deprecated state. +An \fIalpha\fP feature is disabled by default and may change in arbitrary ways between restic versions or be removed. +A \fIbeta\fP feature is enabled by default, but still can change in minor ways or be removed. +A \fIstable\fP feature is always enabled and cannot be disabled. The flag will be removed in a future restic version. +A \fIdeprecated\fP feature is always disabled and cannot be enabled. The flag will be removed in a future restic version. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for features + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic(1)\fP diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 2d81decd3..e8d974527 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -165,6 +165,10 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) @@ -190,6 +194,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .EE diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index 55705288f..058dbee25 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -36,6 +36,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -237,6 +238,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index f2db39bac..f17a6fcd0 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -138,6 +138,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index de439add5..50fa00b71 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -29,7 +29,7 @@ Exit status is 1 if there was any error. .PP \fB--from-insecure-no-password\fP[=false] - use an empty password for the source repository, must be passed to every restic command (insecure) + use an empty password for the source repository (insecure) .PP \fB--from-key-hint\fP="" @@ -149,6 +149,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-key-add.1 b/doc/man/restic-key-add.1 index 6a24e1e67..ff33408b4 100644 --- a/doc/man/restic-key-add.1 +++ b/doc/man/restic-key-add.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -135,6 +136,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-key-list.1 b/doc/man/restic-key-list.1 index a00b116b9..7deb05793 100644 --- a/doc/man/restic-key-list.1 +++ b/doc/man/restic-key-list.1 @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -121,6 +122,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-key-passwd.1 b/doc/man/restic-key-passwd.1 index 42315d72a..68e81edd9 100644 --- a/doc/man/restic-key-passwd.1 +++ b/doc/man/restic-key-passwd.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -136,6 +137,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-key-remove.1 b/doc/man/restic-key-remove.1 index 6ee826059..ff1a0ceb9 100644 --- a/doc/man/restic-key-remove.1 +++ b/doc/man/restic-key-remove.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -120,6 +121,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index 43da808cc..4fd1f6caf 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -112,6 +112,10 @@ per repository. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index f8a1db005..29945e859 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -119,6 +120,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index 6cc662583..b990d2ec8 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -37,6 +37,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -162,6 +163,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index 2272294bf..c0fa2dbc1 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -125,6 +126,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index a256d2a5f..5ec59391d 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -64,6 +64,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -193,6 +194,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-options.1 b/doc/man/restic-options.1 new file mode 100644 index 000000000..8ea8bea63 --- /dev/null +++ b/doc/man/restic-options.1 @@ -0,0 +1,135 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-options - Print list of extended options + + +.SH SYNOPSIS +.PP +\fBrestic options [flags]\fP + + +.SH DESCRIPTION +.PP +The "options" command prints a list of extended options. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for options + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic(1)\fP diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index 7e16748ab..1ee262b61 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -148,6 +149,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 0529360ae..382a91ceb 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -121,6 +122,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-repair-index.1 b/doc/man/restic-repair-index.1 index 60327a916..341f90d59 100644 --- a/doc/man/restic-repair-index.1 +++ b/doc/man/restic-repair-index.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -124,6 +125,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-repair-packs.1 b/doc/man/restic-repair-packs.1 index 01a2f6540..d0091725b 100644 --- a/doc/man/restic-repair-packs.1 +++ b/doc/man/restic-repair-packs.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -120,6 +121,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-repair-snapshots.1 b/doc/man/restic-repair-snapshots.1 index c4439f131..d9e12ddf1 100644 --- a/doc/man/restic-repair-snapshots.1 +++ b/doc/man/restic-repair-snapshots.1 @@ -41,6 +41,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -158,6 +159,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-repair.1 b/doc/man/restic-repair.1 index 7fa313aab..b06562486 100644 --- a/doc/man/restic-repair.1 +++ b/doc/man/restic-repair.1 @@ -111,6 +111,10 @@ Repair the repository \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index 876b18bf8..e9ef4ef94 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -31,6 +31,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -196,6 +197,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index d3dd92436..c0d4a7e1a 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -39,6 +39,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -180,6 +181,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index e6dd4faf2..d475f13cb 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -25,6 +25,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -126,6 +127,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index 25d5274e3..f59240b44 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -143,6 +144,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index fe4074ca5..1e6e79dac 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -52,6 +52,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -165,6 +166,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 7ab1911e5..89c677867 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -29,6 +29,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -150,6 +151,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index a24a4f815..74679ef91 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -121,6 +121,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index e9df439ed..8d5fe6c65 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -118,6 +118,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/doc/man/restic.1 b/doc/man/restic.1 index ee423c6ad..bd8009aac 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -113,6 +113,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) @@ -124,4 +128,4 @@ The full documentation can be found at https://restic.readthedocs.io/ . .SH SEE ALSO .PP -\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-repair(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP +\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-features(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-options(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-repair(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP From 975aa41e1e6a1c88deb501451f23cbdbb013f1da Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 5 Sep 2024 21:25:21 +0200 Subject: [PATCH 51/52] Add version for 0.17.1 --- VERSION | 2 +- cmd/restic/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index a0073758b..7cca7711a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.17.0-dev +0.17.1 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 375b57f98..9df009d8c 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -47,7 +47,7 @@ import ( // to a missing backend storage location or config file var ErrNoRepository = errors.New("repository does not exist") -var version = "0.17.0-dev (compiled manually)" +var version = "0.17.1" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From 76a647febf5d0e58e26b3d6561cfcb37ddb97135 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 5 Sep 2024 21:25:24 +0200 Subject: [PATCH 52/52] Set development version for 0.17.1 --- VERSION | 2 +- cmd/restic/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 7cca7711a..21997e69a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.17.1 +0.17.1-dev diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 9df009d8c..99f9df8cf 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -47,7 +47,7 @@ import ( // to a missing backend storage location or config file var ErrNoRepository = errors.New("repository does not exist") -var version = "0.17.1" +var version = "0.17.1-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05"