From 0164f5310d627b2e920f7427c265483634b020da Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Feb 2024 20:10:29 +0100 Subject: [PATCH 01/19] Downgrade klauspost/compress to fix data corruption at max. compression --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f07fc10ee..970770a90 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( google.golang.org/api v0.149.0 ) +replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2 + require ( cloud.google.com/go v0.110.9 // indirect cloud.google.com/go/compute v1.23.1 // indirect diff --git a/go.sum b/go.sum index 01e58cccb..129eafed2 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= From 29e1caf8250c1965467fb6da1f6155e953b4075d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:52:51 +0100 Subject: [PATCH 02/19] add changelog draft for data corruption on max compression --- changelog/unreleased/issue-4677 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 changelog/unreleased/issue-4677 diff --git a/changelog/unreleased/issue-4677 b/changelog/unreleased/issue-4677 new file mode 100644 index 000000000..8fa6cf65b --- /dev/null +++ b/changelog/unreleased/issue-4677 @@ -0,0 +1,19 @@ +Bugfix: Downgrade zstd library to fix rare data corruption at max. compression + +In restic 0.16.3, backups where the compression level was set to `max` (using +`--compression max`) could in rare and very specific circumstances result in +data corruption due to a bug in the library used for compressing data. Restic +0.16.1 and 0.16.2 were not affected. + +Restic now uses the previous version of the library used to compress data, the +same version used by restic 0.16.2. Please note that the `auto` compression +level (which restic uses by default) was never affected, and even if you used +`max` compression, chances of being affected by this issue are small. + +To check a repository for any corruption, run `restic check --read-data`. This +will download and verify the whole repository and can be used at any time to +completely verify the integrity of a repository. If the `check` command detects +anomalies, follow the suggested steps. + +https://github.com/restic/restic/issues/4677 +https://github.com/restic/restic/pull/4679 From 0ea62b5ac6804a425ebbb9dfbfe65c296836e36b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:47:36 +0100 Subject: [PATCH 03/19] repository: make repo.Options configurable for test repos --- internal/archiver/archiver_test.go | 2 +- internal/migrations/upgrade_repo_v2_test.go | 2 +- internal/repository/fuzz_test.go | 3 +-- internal/repository/testing.go | 9 +++++---- internal/restic/lock_test.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 3c87055d8..5a9896a48 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1879,7 +1879,7 @@ func TestArchiverContextCanceled(t *testing.T) { }) // Ensure that the archiver itself reports the canceled context and not just the backend - repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0) + repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) back := restictest.Chdir(t, tempdir) defer back() diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go index 96fc7788e..7f251de93 100644 --- a/internal/migrations/upgrade_repo_v2_test.go +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) { Backend: be, } - repo := repository.TestRepositoryWithBackend(t, be, 1) + repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{}) if repo.Config().Version != 1 { t.Fatal("test repo has wrong version") } diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index b4036288c..80372f8e0 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) { } id := restic.Hash(blob) - repo := TestRepositoryWithBackend(t, mem.New(), 2) + repo := TestRepositoryWithVersion(t, 2) var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 4936cc368..9bdd65901 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -43,7 +43,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, opts Options) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -52,7 +52,7 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) re be = TestBackend(t) } - repo, err := New(be, Options{}) + repo, err := New(be, opts) if err != nil { t.Fatalf("TestRepository(): new repo failed: %v", err) } @@ -78,6 +78,7 @@ func TestRepository(t testing.TB) restic.Repository { func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") + opts := Options{} if dir != "" { _, err := os.Stat(dir) if err != nil { @@ -85,7 +86,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } - return TestRepositoryWithBackend(t, be, version) + return TestRepositoryWithBackend(t, be, version, opts) } if err == nil { @@ -93,7 +94,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { } } - return TestRepositoryWithBackend(t, nil, version) + return TestRepositoryWithBackend(t, nil, version, opts) } // TestOpenLocal opens a local repository. diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index f3c405c9c..2eb22be1b 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -65,7 +65,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} - repo := repository.TestRepositoryWithBackend(t, be, 0) + repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) From 2f30c940b2f6ca645f87a06a841ff137e3153afe Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Feb 2024 21:15:39 +0100 Subject: [PATCH 04/19] backup: verify blobs before upload This only covers the blobs themselves, the pack header is not verified so far. Unpacked files are also not covered by the integrity check. --- internal/repository/repository.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 0b50382b8..7855438c9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -423,6 +423,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data // encrypt blob ciphertext = r.key.Seal(ciphertext, nonce, data, nil) + if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { + // FIXME call to action + return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err) + } + // find suitable packer and add blob var pm *packerManager @@ -438,6 +443,27 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) } +func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if uncompressedLength != 0 { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + if !restic.Hash(plaintext).Equal(id) { + return errors.New("hash mismatch") + } + + return nil +} + func (r *Repository) compressUnpacked(p []byte) ([]byte, error) { // compression is only available starting from version 2 if r.cfg.Version < 2 { From cb85fb46ddf15d8c6d149d23cadcaab5e7164786 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:30:58 +0100 Subject: [PATCH 05/19] backup: verify unpacked files before upload --- internal/repository/repository.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 7855438c9..4fa7e487c 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -500,7 +500,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) { // SaveUnpacked encrypts data and stores it in the backend. Returned is the // storage hash. -func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { +func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) { + p := buf if t != restic.ConfigFile { p, err = r.compressUnpacked(p) if err != nil { @@ -515,6 +516,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by ciphertext = r.key.Seal(ciphertext, nonce, p, nil) + if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { + // FIXME call to action + return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err) + } + if t == restic.ConfigFile { id = restic.ID{} } else { @@ -532,6 +538,25 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by return id, nil } +func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if t != restic.ConfigFile { + plaintext, err = r.decompressUnpacked(plaintext) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + + if !bytes.Equal(plaintext, expected) { + return errors.New("data mismatch") + } + return nil +} + // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { if err := r.flushPacks(ctx); err != nil { From 885431ec2b64bae061246ef8d0fab19322e29fe3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:47:48 +0100 Subject: [PATCH 06/19] repository: Allow skipping verification for tests Some tests have to explicitly create pack files with blobs that don't match their ID. For those blobs the builtin verification of the repository must be disabled. --- internal/repository/repack_test.go | 6 ++++-- internal/repository/repository.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index c8570a9d4..00567aad9 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -346,7 +346,8 @@ func TestRepackWrongBlob(t *testing.T) { } func testRepackWrongBlob(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -371,7 +372,8 @@ func TestRepackBlobFallback(t *testing.T) { } func testRepackBlobFallback(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 4fa7e487c..c7c36a74a 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,8 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint + Compression CompressionMode + PackSize uint + NoVerifyPack bool } // CompressionMode configures if data should be compressed. @@ -444,6 +445,10 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + if r.opts.NoVerifyPack { + return nil + } + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) if err != nil { @@ -539,6 +544,10 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + if r.opts.NoVerifyPack { + return nil + } + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) if err != nil { From 8ef5425351944a1d52fa76eb510cbd619c10ea8c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:13:34 +0100 Subject: [PATCH 07/19] repository: test verification of blobs/unpacked data --- .../repository/repository_internal_test.go | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index e5ab6e5b7..2a9976ace 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -3,8 +3,10 @@ package repository import ( "math/rand" "sort" + "strings" "testing" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -72,3 +74,101 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { sortCachedPacksFirst(cache, cpy[:]) } } + +func TestBlobVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "hash mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + id := restic.Hash(plaintext) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + uncompressedLength := uint(len(plaintext)) + plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil) + + if test.damage == damageCompressed { + plaintext = plaintext[:len(plaintext)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} + +func TestUnpackedVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "data mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + orig := append([]byte{}, plaintext...) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + compressed := []byte{2} + compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed) + + if test.damage == damageCompressed { + compressed = compressed[:len(compressed)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} From dc11d012bb5d09dd0ae246c2e5c911b6808fde00 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:17:18 +0100 Subject: [PATCH 08/19] Make --no-verify-pack globally available Verifying all blobs before upload comes with a notable performance impact. Allow users to skip it if necessary. --- cmd/restic/global.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 9e3a3a7a9..5bdf03bdc 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -67,6 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint + NoVerifyPack bool backend.TransportOptions limiter.Limits @@ -139,6 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") + f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -453,8 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoVerifyPack: opts.NoVerifyPack, }) if err != nil { return nil, errors.Fatal(err.Error()) From d8916bc3d9987a376f9748da7a306bdad8e79ed4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:42:02 +0100 Subject: [PATCH 09/19] repository: ask users to report corrupted data while saving blobs --- internal/repository/repository.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index c7c36a74a..1e253c24a 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -425,8 +425,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data ciphertext = r.key.Seal(ciphertext, nonce, data, nil) if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { - // FIXME call to action - return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err) + return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err) } // find suitable packer and add blob @@ -522,8 +521,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] ciphertext = r.key.Seal(ciphertext, nonce, p, nil) if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { - // FIXME call to action - return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err) + return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err) } if t == restic.ConfigFile { From 75e72d826ce9b117430f1d594a24d4954691f60c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 11:58:29 +0100 Subject: [PATCH 10/19] pack: verify integrity of pack file header --- internal/pack/pack.go | 43 +++++++++++++++------ internal/pack/pack_internal_test.go | 58 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 34ad9d071..34e87f1f9 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "bytes" "context" "encoding/binary" "fmt" @@ -74,7 +75,7 @@ func (p *Packer) Finalize() error { p.m.Lock() defer p.m.Unlock() - header, err := p.makeHeader() + header, err := makeHeader(p.blobs) if err != nil { return err } @@ -83,6 +84,11 @@ func (p *Packer) Finalize() error { nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { + return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err) + } // append the header n, err := p.wr.Write(encryptedHeader) @@ -90,18 +96,33 @@ func (p *Packer) Finalize() error { return errors.Wrap(err, "Write") } - hdrBytes := len(encryptedHeader) - if n != hdrBytes { + if n != len(encryptedHeader) { return errors.New("wrong number of bytes written") } + p.bytes += uint(len(encryptedHeader)) - // write length - err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) + return nil +} + +func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error { + // do not offer a way to skip the pack header verification, as pack headers are usually small enough + // to not result in a significant performance impact + + decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header))) if err != nil { - return errors.Wrap(err, "binary.Write") + return fmt.Errorf("header decoding failed: %w", err) + } + if hdrSize != uint32(len(header)) { + return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header)) + } + if len(decoded) != len(expected) { + return fmt.Errorf("pack header size mismatch") + } + for i := 0; i < len(decoded); i++ { + if decoded[i] != expected[i] { + return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i]) + } } - p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - return nil } @@ -111,10 +132,10 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func (p *Packer) makeHeader() ([]byte, error) { - buf := make([]byte, 0, len(p.blobs)*int(entrySize)) +func makeHeader(blobs []restic.Blob) ([]byte, error) { + buf := make([]byte, 0, len(blobs)*int(entrySize)) - for _, b := range p.blobs { + for _, b := range blobs { switch { case b.Type == restic.DataBlob && b.UncompressedLength == 0: buf = append(buf, 0) diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go index c1a4867ea..2e7400ad0 100644 --- a/internal/pack/pack_internal_test.go +++ b/internal/pack/pack_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "strings" "testing" "github.com/restic/restic/internal/crypto" @@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) { } } } + +func TestUnpackedVerification(t *testing.T) { + // create random keys + k := crypto.NewRandomKey() + blobs := []restic.Blob{ + { + BlobHandle: restic.NewRandomBlobHandle(), + Length: 42, + Offset: 0, + UncompressedLength: 2 * 42, + }, + } + + type DamageType string + const ( + damageData DamageType = "data" + damageCiphertext DamageType = "ciphertext" + damageLength DamageType = "length" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "pack header entry mismatch"}, + {damageCiphertext, "ciphertext verification failed"}, + {damageLength, "header decoding failed"}, + } { + header, err := makeHeader(blobs) + rtest.OK(t, err) + + if test.damage == damageData { + header[8] ^= 0x42 + } + + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) + nonce := crypto.NewRandomNonce() + encryptedHeader = append(encryptedHeader, nonce...) + encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if test.damage == damageCiphertext { + encryptedHeader[8] ^= 0x42 + } + if test.damage == damageLength { + encryptedHeader[len(encryptedHeader)-1] ^= 0x42 + } + + err = verifyHeader(k, encryptedHeader, blobs) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} From 4589da7eb9bc9037c9dc3b85ee0ced0569aeb2bf Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 15:48:11 +0100 Subject: [PATCH 11/19] add data verification changelog entry --- changelog/unreleased/issue-4529 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog/unreleased/issue-4529 diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 new file mode 100644 index 000000000..2e8bbbed7 --- /dev/null +++ b/changelog/unreleased/issue-4529 @@ -0,0 +1,14 @@ +Enhancement: Verify data integrity before upload + +Hardware issues or a bug in restic could cause restic to create corrupted files +that were then uploaded to the repository. Detecting such corruption usually +required explicitly running the `check --read-data` command. + +To prevent the upload of corrupted data to the repository, restic now +additionally verifies that files can be decoded and contain the correct data +beforehand. This increases the CPU usage during backups. If absolutely +necessary, you can disable the verification using the option +`--no-verify-pack`. + +https://github.com/restic/restic/issues/4529 +https://github.com/restic/restic/pull/4681 From 2a0bd2b637b4b791f92f3d84c565aaaf39b9ddb2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 16:50:50 +0100 Subject: [PATCH 12/19] rename `--no-verify-pack` to `--no-extra-verify` --- changelog/unreleased/issue-4529 | 2 +- cmd/restic/global.go | 10 +++++----- internal/repository/repack_test.go | 4 ++-- internal/repository/repository.go | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 index 2e8bbbed7..c3ec69510 100644 --- a/changelog/unreleased/issue-4529 +++ b/changelog/unreleased/issue-4529 @@ -8,7 +8,7 @@ To prevent the upload of corrupted data to the repository, restic now additionally verifies that files can be decoded and contain the correct data beforehand. This increases the CPU usage during backups. If absolutely necessary, you can disable the verification using the option -`--no-verify-pack`. +`--no-extra-verify`. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 5bdf03bdc..8568e41c3 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -67,7 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint - NoVerifyPack bool + NoExtraVerify bool backend.TransportOptions limiter.Limits @@ -140,7 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") - f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -455,9 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, - NoVerifyPack: opts.NoVerifyPack, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoExtraVerify: opts.NoExtraVerify, }) if err != nil { return nil, errors.Fatal(err.Error()) diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 00567aad9..1ecbf3e1c 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -347,7 +347,7 @@ func TestRepackWrongBlob(t *testing.T) { func testRepackWrongBlob(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -373,7 +373,7 @@ func TestRepackBlobFallback(t *testing.T) { func testRepackBlobFallback(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 1e253c24a..cea43b0d3 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,9 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint - NoVerifyPack bool + Compression CompressionMode + PackSize uint + NoExtraVerify bool } // CompressionMode configures if data should be compressed. @@ -444,7 +444,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { - if r.opts.NoVerifyPack { + if r.opts.NoExtraVerify { return nil } @@ -542,7 +542,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { - if r.opts.NoVerifyPack { + if r.opts.NoExtraVerify { return nil } From 261b1455c79335152edb74cbc3626a6248d568f7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 17:11:49 +0100 Subject: [PATCH 13/19] add documentation for --no-extra-verify option --- doc/047_tuning_backup_parameters.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 6ea39dc75..5773ac161 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -60,6 +60,17 @@ only applied for the single run of restic. The option can also be set via the en variable ``RESTIC_COMPRESSION``. +Data Verification +================= + +To prevent the upload of corrupted data to the repository, restic verifies that files can +be decoded and contain the correct data beforehand. This increases the CPU usage during +backups. If necessary, you can disable this verification using the option ``--no-extra-verify``. +However, in this case you should verify the repository integrity more actively using +``restic check --read-data``. Otherwise, data corruption due to hardware issues or software +bugs might go unnoticed. + + File Read Concurrency ===================== From a72c2b74f3a3ffa259e406462b4afda49726187d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 18:09:32 +0100 Subject: [PATCH 14/19] Apply changelog entry / documentation improvements from review --- changelog/unreleased/issue-4529 | 22 +++++++++++++--------- cmd/restic/global.go | 2 +- doc/047_tuning_backup_parameters.rst | 15 +++++++++------ internal/pack/pack.go | 3 ++- internal/repository/repository.go | 6 ++++-- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 index c3ec69510..fed726d2d 100644 --- a/changelog/unreleased/issue-4529 +++ b/changelog/unreleased/issue-4529 @@ -1,14 +1,18 @@ -Enhancement: Verify data integrity before upload +Enhancement: Add extra verification of data integrity before upload -Hardware issues or a bug in restic could cause restic to create corrupted files -that were then uploaded to the repository. Detecting such corruption usually -required explicitly running the `check --read-data` command. +Hardware issues, or a bug in restic or its dependencies, could previously cause +corruption in the files restic created and stored in the repository. Detecting +such corruption previously required explicitly running the `check --read-data` +or `check --read-data-subset` commands. -To prevent the upload of corrupted data to the repository, restic now -additionally verifies that files can be decoded and contain the correct data -beforehand. This increases the CPU usage during backups. If absolutely -necessary, you can disable the verification using the option -`--no-extra-verify`. +To further ensure data integrity, even in the case of hardware issues or +software bugs, restic now performs additional verification of the files about +to be uploaded to the repository. + +These extra checks will increase CPU usage during backups. They can therefore, +if absolutely necessary, be disabled using the `--no-extra-verify` global +option. Please note that this should be combined with more active checking +using the previously mentioned check commands. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 8568e41c3..528c6e129 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -140,7 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") - f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 5773ac161..d8fb2c9b6 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -63,12 +63,15 @@ variable ``RESTIC_COMPRESSION``. Data Verification ================= -To prevent the upload of corrupted data to the repository, restic verifies that files can -be decoded and contain the correct data beforehand. This increases the CPU usage during -backups. If necessary, you can disable this verification using the option ``--no-extra-verify``. -However, in this case you should verify the repository integrity more actively using -``restic check --read-data``. Otherwise, data corruption due to hardware issues or software -bugs might go unnoticed. +To prevent the upload of corrupted data to the repository, which can happen due +to hardware issues or software bugs, restic verifies that generated files can +be decoded and contain the correct data beforehand. This increases the CPU usage +during backups. If necessary, you can disable this verification using the +``--no-extra-verify`` option of the ``backup`` command. However, in this case +you should verify the repository integrity more actively using +``restic check --read-data`` (or the similar ``--read-data-subset`` option). +Otherwise, data corruption due to hardware issues or software bugs might go +unnoticed. File Read Concurrency diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 34e87f1f9..f9e7896e0 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -87,7 +87,8 @@ func (p *Packer) Finalize() error { encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { - return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err) + //nolint:revive // ignore linter warnings about error message spelling + return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err) } // append the header diff --git a/internal/repository/repository.go b/internal/repository/repository.go index cea43b0d3..40794508f 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -425,7 +425,8 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data ciphertext = r.key.Seal(ciphertext, nonce, data, nil) if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { - return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err) + //nolint:revive // ignore linter warnings about error message spelling + return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err) } // find suitable packer and add blob @@ -521,7 +522,8 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] ciphertext = r.key.Seal(ciphertext, nonce, p, nil) if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { - return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err) + //nolint:revive // ignore linter warnings about error message spelling + return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err) } if t == restic.ConfigFile { From 6cd2804bfff06957e835e76d7a67c444d202dcbf Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:34 +0100 Subject: [PATCH 15/19] Prepare changelog for 0.16.4 --- changelog/{unreleased => 0.16.4_2024-02-04}/issue-4529 | 0 changelog/{unreleased => 0.16.4_2024-02-04}/issue-4677 | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename changelog/{unreleased => 0.16.4_2024-02-04}/issue-4529 (100%) rename changelog/{unreleased => 0.16.4_2024-02-04}/issue-4677 (100%) diff --git a/changelog/unreleased/issue-4529 b/changelog/0.16.4_2024-02-04/issue-4529 similarity index 100% rename from changelog/unreleased/issue-4529 rename to changelog/0.16.4_2024-02-04/issue-4529 diff --git a/changelog/unreleased/issue-4677 b/changelog/0.16.4_2024-02-04/issue-4677 similarity index 100% rename from changelog/unreleased/issue-4677 rename to changelog/0.16.4_2024-02-04/issue-4677 From b0ead75de5d0f50b87bfe5146d5544dd3d92c28f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:34 +0100 Subject: [PATCH 16/19] Generate CHANGELOG.md for 0.16.4 --- CHANGELOG.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba6fa7a11..b8969a443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) * [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14) * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) * [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) @@ -32,6 +33,57 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.16.4 (2024-02-04) +The following sections list the changes in restic 0.16.4 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4677: Downgrade zstd library to fix rare data corruption at max. compression + * Enh #4529: Add extra verification of data integrity before upload + +## Details + + * Bugfix #4677: Downgrade zstd library to fix rare data corruption at max. compression + + In restic 0.16.3, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. Restic + 0.16.1 and 0.16.2 were not affected. + + Restic now uses the previous version of the library used to compress data, the + same version used by restic 0.16.2. Please note that the `auto` compression + level (which restic uses by default) was never affected, and even if you used + `max` compression, chances of being affected by this issue are small. + + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. + + https://github.com/restic/restic/issues/4677 + https://github.com/restic/restic/pull/4679 + + * Enhancement #4529: Add extra verification of data integrity before upload + + Hardware issues, or a bug in restic or its dependencies, could previously cause + corruption in the files restic created and stored in the repository. Detecting + such corruption previously required explicitly running the `check --read-data` + or `check --read-data-subset` commands. + + To further ensure data integrity, even in the case of hardware issues or + software bugs, restic now performs additional verification of the files about to + be uploaded to the repository. + + These extra checks will increase CPU usage during backups. They can therefore, + if absolutely necessary, be disabled using the `--no-extra-verify` global + option. Please note that this should be combined with more active checking using + the previously mentioned check commands. + + https://github.com/restic/restic/issues/4529 + https://github.com/restic/restic/pull/4681 + + # Changelog for restic 0.16.3 (2024-01-14) The following sections list the changes in restic 0.16.3 relevant to restic users. The changes are ordered by importance. From 811be5984d5d42acf58714d13aeb93cebf4fe2e0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:51 +0100 Subject: [PATCH 17/19] Update manpages and auto-completion --- doc/bash-completion.sh | 33 +++++++++++++++++++++++++++++++ doc/man/restic-backup.1 | 4 ++++ doc/man/restic-cache.1 | 4 ++++ doc/man/restic-cat.1 | 4 ++++ doc/man/restic-check.1 | 4 ++++ doc/man/restic-copy.1 | 4 ++++ doc/man/restic-diff.1 | 4 ++++ doc/man/restic-dump.1 | 4 ++++ doc/man/restic-find.1 | 4 ++++ doc/man/restic-forget.1 | 4 ++++ doc/man/restic-generate.1 | 4 ++++ doc/man/restic-init.1 | 4 ++++ doc/man/restic-key.1 | 4 ++++ doc/man/restic-list.1 | 4 ++++ doc/man/restic-ls.1 | 4 ++++ doc/man/restic-migrate.1 | 4 ++++ doc/man/restic-mount.1 | 4 ++++ doc/man/restic-prune.1 | 4 ++++ doc/man/restic-recover.1 | 4 ++++ doc/man/restic-repair-index.1 | 4 ++++ doc/man/restic-repair-packs.1 | 4 ++++ doc/man/restic-repair-snapshots.1 | 4 ++++ doc/man/restic-repair.1 | 4 ++++ doc/man/restic-restore.1 | 4 ++++ doc/man/restic-rewrite.1 | 4 ++++ doc/man/restic-self-update.1 | 4 ++++ doc/man/restic-snapshots.1 | 4 ++++ doc/man/restic-stats.1 | 4 ++++ doc/man/restic-tag.1 | 4 ++++ doc/man/restic-unlock.1 | 4 ++++ doc/man/restic-version.1 | 4 ++++ doc/man/restic.1 | 4 ++++ 32 files changed, 157 insertions(+) diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh index e691af363..cae37a6ca 100644 --- a/doc/bash-completion.sh +++ b/doc/bash-completion.sh @@ -488,6 +488,7 @@ _restic_backup() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -560,6 +561,7 @@ _restic_cache() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -624,6 +626,7 @@ _restic_cat() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -696,6 +699,7 @@ _restic_check() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -794,6 +798,7 @@ _restic_copy() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -860,6 +865,7 @@ _restic_diff() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -944,6 +950,7 @@ _restic_dump() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1058,6 +1065,7 @@ _restic_find() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1228,6 +1236,7 @@ _restic_forget() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1312,6 +1321,7 @@ _restic_generate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1372,6 +1382,7 @@ _restic_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1463,6 +1474,7 @@ _restic_init() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1539,6 +1551,7 @@ _restic_key() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1603,6 +1616,7 @@ _restic_list() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1689,6 +1703,7 @@ _restic_ls() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1757,6 +1772,7 @@ _restic_migrate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1849,6 +1865,7 @@ _restic_mount() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1935,6 +1952,7 @@ _restic_prune() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1999,6 +2017,7 @@ _restic_recover() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2059,6 +2078,7 @@ _restic_repair_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2126,6 +2146,7 @@ _restic_repair_index() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2190,6 +2211,7 @@ _restic_repair_packs() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2274,6 +2296,7 @@ _restic_repair_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2342,6 +2365,7 @@ _restic_repair() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2450,6 +2474,7 @@ _restic_restore() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2552,6 +2577,7 @@ _restic_rewrite() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2620,6 +2646,7 @@ _restic_self-update() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2712,6 +2739,7 @@ _restic_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2794,6 +2822,7 @@ _restic_stats() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2884,6 +2913,7 @@ _restic_tag() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2950,6 +2980,7 @@ _restic_unlock() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3014,6 +3045,7 @@ _restic_version() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3106,6 +3138,7 @@ _restic_root_command() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index c3bccdfa5..730685271 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -171,6 +171,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index 3ae27ea57..c170c1624 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index c1df138aa..b42a58e14 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index 17eb972bc..9c1dc77e5 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -85,6 +85,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index be8f21e25..bd9795f44 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -109,6 +109,10 @@ new destination repository using the "init" command. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index a01a2562b..28f3a4838 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -93,6 +93,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index 6fa1f8200..7fa3f777d 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 72bc3a0b6..c3297c43f 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -117,6 +117,10 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index 757022a21..d0c4cfc74 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -179,6 +179,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index aef3a5e55..84f659ef2 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -89,6 +89,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 27d7f5874..5f19c8f8c 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index 855ef5443..8d1813188 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index 95eeac5f7..e399038a2 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index 0cd0f5a88..10b0657a3 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index eca0ef8e1..7e48f726c 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -74,6 +74,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index 33c016ffa..aab607fcf 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -144,6 +144,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index e4a32cac3..c54d5d7ff 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -97,6 +97,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 26d2fc7bd..010fbafd7 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -70,6 +70,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-index.1 b/doc/man/restic-repair-index.1 index 35e2845b8..f06be64c0 100644 --- a/doc/man/restic-repair-index.1 +++ b/doc/man/restic-repair-index.1 @@ -73,6 +73,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-packs.1 b/doc/man/restic-repair-packs.1 index b21211925..f3671fe18 100644 --- a/doc/man/restic-repair-packs.1 +++ b/doc/man/restic-repair-packs.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-snapshots.1 b/doc/man/restic-repair-snapshots.1 index f59067f05..9369f25f2 100644 --- a/doc/man/restic-repair-snapshots.1 +++ b/doc/man/restic-repair-snapshots.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair.1 b/doc/man/restic-repair.1 index dbe783df4..77aecc173 100644 --- a/doc/man/restic-repair.1 +++ b/doc/man/restic-repair.1 @@ -63,6 +63,10 @@ Repair the repository \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index d8c1b72e1..4635b1e43 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -117,6 +117,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index 8a06aef40..d63c653e6 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -121,6 +121,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index 28fd24a92..92ab5add3 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index cb34d6c8e..6203bbf2b 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -92,6 +92,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index cf0374351..9d37163de 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -114,6 +114,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 162d50d29..b1468c74d 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -99,6 +99,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index 0274c56e8..0b3b43f2a 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index 774e19453..ccc23038f 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -69,6 +69,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic.1 b/doc/man/restic.1 index 427ce7c65..333eab76a 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -65,6 +65,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories From 3786536dc18ef27aedcfa8e4c6953b48353eee79 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:52 +0100 Subject: [PATCH 18/19] Add version for 0.16.4 --- VERSION | 2 +- cmd/restic/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 7eb3095a3..5f2491c5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.16.3 +0.16.4 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 528c6e129..e979dcc2b 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.3-dev (compiled manually)" +var version = "0.16.4" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From 0f9fa44de5a27efd22a17c9e49669a29d5ecf33a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:56 +0100 Subject: [PATCH 19/19] Set development version for 0.16.4 --- cmd/restic/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index e979dcc2b..c179e8d33 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.4" +var version = "0.16.4-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05"