From bad6c54a338ce5fc986c72d4162d5d88108673cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Phillipp=20R=C3=B6ll?= Date: Fri, 13 Sep 2024 18:21:07 +0200 Subject: [PATCH 1/4] dump: add --compress-zip flag to compress zip archives --- changelog/unreleased/pull-5054 | 10 ++++++++++ cmd/restic/cmd_dump.go | 8 +++++--- internal/dump/common.go | 20 +++++++++++--------- internal/dump/common_test.go | 4 ++-- internal/dump/tar.go | 13 ++++++++++++- internal/dump/tar_test.go | 3 ++- internal/dump/zip.go | 4 ++++ internal/dump/zip_test.go | 3 ++- 8 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 changelog/unreleased/pull-5054 diff --git a/changelog/unreleased/pull-5054 b/changelog/unreleased/pull-5054 new file mode 100644 index 000000000..7b5291e2d --- /dev/null +++ b/changelog/unreleased/pull-5054 @@ -0,0 +1,10 @@ +Enhancement: Add `--compress` flag to `dump` command to compress archive + +Restic did not compress the archives that was created by using the +`dump` command. It now allows to save some disk space when exporting +archives by adding a `--compress` flag. The DEFLATE algorithm is used +for "zip" archives, and the gzip algorithm for "tar" archives, +resulting in a .tar.gz or .tgz file. Not compressing the archive +is still the default. + +https://github.com/restic/restic/pull/5054 diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 6b7f8d012..23c502635 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -50,8 +50,9 @@ Exit status is 12 if the password is incorrect. // DumpOptions collects all options for the dump command. type DumpOptions struct { restic.SnapshotFilter - Archive string - Target string + Archive string + Target string + Compress bool } var dumpOptions DumpOptions @@ -63,6 +64,7 @@ func init() { initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`") + flags.BoolVarP(&dumpOptions.Compress, "compress", "c", false, "compress archive contents. When enabled, the DEFLATE algorithm is applied for \"zip\" archives, and the gzip algorithm for \"tar\" archives, resulting in a .tar.gz or .tgz file. (default: false)") } func splitPath(p string) []string { @@ -187,7 +189,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] canWriteArchiveFunc = func() error { return nil } } - d := dump.New(opts.Archive, repo, outputFileWriter) + d := dump.New(opts.Archive, opts.Compress, repo, outputFileWriter) err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) diff --git a/internal/dump/common.go b/internal/dump/common.go index 4bc404fe0..619e946e9 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -15,18 +15,20 @@ import ( // A Dumper writes trees and files from a repository to a Writer // in an archive format. type Dumper struct { - cache *bloblru.Cache - format string - repo restic.Loader - w io.Writer + cache *bloblru.Cache + format string + repo restic.Loader + w io.Writer + compress bool } -func New(format string, repo restic.Loader, w io.Writer) *Dumper { +func New(format string, compress bool, repo restic.Loader, w io.Writer) *Dumper { return &Dumper{ - cache: bloblru.New(64 << 20), - format: format, - repo: repo, - w: w, + cache: bloblru.New(64 << 20), + format: format, + repo: repo, + w: w, + compress: compress, } } diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go index afd19df63..f581e31cb 100644 --- a/internal/dump/common_test.go +++ b/internal/dump/common_test.go @@ -23,7 +23,7 @@ func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.R type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error -func WriteTest(t *testing.T, format string, cd CheckDump) { +func WriteTest(t *testing.T, format string, compress bool, cd CheckDump) { tests := []struct { name string args archiver.TestDir @@ -85,7 +85,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { rtest.OK(t, err) dst := &bytes.Buffer{} - d := New(format, repo, dst) + d := New(format, compress, repo, dst) if err := d.DumpTree(ctx, tree, tt.target); err != nil { t.Fatalf("Dumper.Run error = %v", err) } diff --git a/internal/dump/tar.go b/internal/dump/tar.go index c5933d4f8..0358fba5d 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -2,6 +2,7 @@ package dump import ( "archive/tar" + "compress/gzip" "context" "fmt" "os" @@ -13,12 +14,22 @@ import ( ) func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *restic.Node) (err error) { - w := tar.NewWriter(d.w) + outer := d.w + + if d.compress { + outer = gzip.NewWriter(outer) + } + w := tar.NewWriter(outer) defer func() { if err == nil { err = w.Close() err = errors.Wrap(err, "Close") + + if gz, ok := outer.(*gzip.Writer); ok { + err = gz.Close() + err = errors.Wrap(err, "Close") + } } }() diff --git a/internal/dump/tar_test.go b/internal/dump/tar_test.go index cb3cb08c4..ae575efad 100644 --- a/internal/dump/tar_test.go +++ b/internal/dump/tar_test.go @@ -18,7 +18,8 @@ import ( ) func TestWriteTar(t *testing.T) { - WriteTest(t, "tar", checkTar) + WriteTest(t, "tar", false, checkTar) + WriteTest(t, "tar", true, checkTar) } func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error { diff --git a/internal/dump/zip.go b/internal/dump/zip.go index d32475770..0d210eaa1 100644 --- a/internal/dump/zip.go +++ b/internal/dump/zip.go @@ -40,6 +40,10 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri } header.SetMode(node.Mode) + if d.compress { + header.Method = zip.Deflate + } + if node.Type == restic.NodeTypeDir { header.Name += "/" } diff --git a/internal/dump/zip_test.go b/internal/dump/zip_test.go index 6f5f60f54..ab955858c 100644 --- a/internal/dump/zip_test.go +++ b/internal/dump/zip_test.go @@ -12,7 +12,8 @@ import ( ) func TestWriteZip(t *testing.T) { - WriteTest(t, "zip", checkZip) + WriteTest(t, "zip", true, checkZip) + WriteTest(t, "zip", false, checkZip) } func readZipFile(f *zip.File) ([]byte, error) { From 1a7fafc7ebca93f214ed0b1ac45781ee6cd9dccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Phillipp=20R=C3=B6ll?= Date: Sun, 15 Sep 2024 21:01:23 +0200 Subject: [PATCH 2/4] dump: compress zip archives --- changelog/unreleased/pull-5054 | 11 ++++------- cmd/restic/cmd_dump.go | 8 +++----- internal/dump/common.go | 20 +++++++++----------- internal/dump/common_test.go | 4 ++-- internal/dump/tar.go | 13 +------------ internal/dump/tar_test.go | 3 +-- internal/dump/zip.go | 5 +---- internal/dump/zip_test.go | 3 +-- 8 files changed, 22 insertions(+), 45 deletions(-) diff --git a/changelog/unreleased/pull-5054 b/changelog/unreleased/pull-5054 index 7b5291e2d..6efd5882c 100644 --- a/changelog/unreleased/pull-5054 +++ b/changelog/unreleased/pull-5054 @@ -1,10 +1,7 @@ -Enhancement: Add `--compress` flag to `dump` command to compress archive +Enhancement: Compress ZIP archives created by `dump` command -Restic did not compress the archives that was created by using the -`dump` command. It now allows to save some disk space when exporting -archives by adding a `--compress` flag. The DEFLATE algorithm is used -for "zip" archives, and the gzip algorithm for "tar" archives, -resulting in a .tar.gz or .tgz file. Not compressing the archive -is still the default. +Restic did not compress the archives that were created by using +the `dump` command. It now saves some disk space when exporting +archives using the DEFLATE algorithm for "zip" archives. https://github.com/restic/restic/pull/5054 diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 23c502635..6b7f8d012 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -50,9 +50,8 @@ Exit status is 12 if the password is incorrect. // DumpOptions collects all options for the dump command. type DumpOptions struct { restic.SnapshotFilter - Archive string - Target string - Compress bool + Archive string + Target string } var dumpOptions DumpOptions @@ -64,7 +63,6 @@ func init() { initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`") - flags.BoolVarP(&dumpOptions.Compress, "compress", "c", false, "compress archive contents. When enabled, the DEFLATE algorithm is applied for \"zip\" archives, and the gzip algorithm for \"tar\" archives, resulting in a .tar.gz or .tgz file. (default: false)") } func splitPath(p string) []string { @@ -189,7 +187,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] canWriteArchiveFunc = func() error { return nil } } - d := dump.New(opts.Archive, opts.Compress, repo, outputFileWriter) + d := dump.New(opts.Archive, repo, outputFileWriter) err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) diff --git a/internal/dump/common.go b/internal/dump/common.go index 619e946e9..4bc404fe0 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -15,20 +15,18 @@ import ( // A Dumper writes trees and files from a repository to a Writer // in an archive format. type Dumper struct { - cache *bloblru.Cache - format string - repo restic.Loader - w io.Writer - compress bool + cache *bloblru.Cache + format string + repo restic.Loader + w io.Writer } -func New(format string, compress bool, repo restic.Loader, w io.Writer) *Dumper { +func New(format string, repo restic.Loader, w io.Writer) *Dumper { return &Dumper{ - cache: bloblru.New(64 << 20), - format: format, - repo: repo, - w: w, - compress: compress, + cache: bloblru.New(64 << 20), + format: format, + repo: repo, + w: w, } } diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go index f581e31cb..afd19df63 100644 --- a/internal/dump/common_test.go +++ b/internal/dump/common_test.go @@ -23,7 +23,7 @@ func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.R type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error -func WriteTest(t *testing.T, format string, compress bool, cd CheckDump) { +func WriteTest(t *testing.T, format string, cd CheckDump) { tests := []struct { name string args archiver.TestDir @@ -85,7 +85,7 @@ func WriteTest(t *testing.T, format string, compress bool, cd CheckDump) { rtest.OK(t, err) dst := &bytes.Buffer{} - d := New(format, compress, repo, dst) + d := New(format, repo, dst) if err := d.DumpTree(ctx, tree, tt.target); err != nil { t.Fatalf("Dumper.Run error = %v", err) } diff --git a/internal/dump/tar.go b/internal/dump/tar.go index 0358fba5d..c5933d4f8 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -2,7 +2,6 @@ package dump import ( "archive/tar" - "compress/gzip" "context" "fmt" "os" @@ -14,22 +13,12 @@ import ( ) func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *restic.Node) (err error) { - outer := d.w - - if d.compress { - outer = gzip.NewWriter(outer) - } - w := tar.NewWriter(outer) + w := tar.NewWriter(d.w) defer func() { if err == nil { err = w.Close() err = errors.Wrap(err, "Close") - - if gz, ok := outer.(*gzip.Writer); ok { - err = gz.Close() - err = errors.Wrap(err, "Close") - } } }() diff --git a/internal/dump/tar_test.go b/internal/dump/tar_test.go index ae575efad..cb3cb08c4 100644 --- a/internal/dump/tar_test.go +++ b/internal/dump/tar_test.go @@ -18,8 +18,7 @@ import ( ) func TestWriteTar(t *testing.T) { - WriteTest(t, "tar", false, checkTar) - WriteTest(t, "tar", true, checkTar) + WriteTest(t, "tar", checkTar) } func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error { diff --git a/internal/dump/zip.go b/internal/dump/zip.go index 0d210eaa1..6041c5187 100644 --- a/internal/dump/zip.go +++ b/internal/dump/zip.go @@ -39,10 +39,7 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri Modified: node.ModTime, } header.SetMode(node.Mode) - - if d.compress { - header.Method = zip.Deflate - } + header.Method = zip.Deflate if node.Type == restic.NodeTypeDir { header.Name += "/" diff --git a/internal/dump/zip_test.go b/internal/dump/zip_test.go index ab955858c..6f5f60f54 100644 --- a/internal/dump/zip_test.go +++ b/internal/dump/zip_test.go @@ -12,8 +12,7 @@ import ( ) func TestWriteZip(t *testing.T) { - WriteTest(t, "zip", true, checkZip) - WriteTest(t, "zip", false, checkZip) + WriteTest(t, "zip", checkZip) } func readZipFile(f *zip.File) ([]byte, error) { From da3c02405b51d9374fc42574e213988b7e510910 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 16 Oct 2024 21:09:05 +0200 Subject: [PATCH 3/4] dump/zip: only compress regular files --- internal/dump/zip.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/dump/zip.go b/internal/dump/zip.go index 6041c5187..17aeb4829 100644 --- a/internal/dump/zip.go +++ b/internal/dump/zip.go @@ -39,7 +39,9 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri Modified: node.ModTime, } header.SetMode(node.Mode) - header.Method = zip.Deflate + if node.Type == restic.NodeTypeFile { + header.Method = zip.Deflate + } if node.Type == restic.NodeTypeDir { header.Name += "/" From e29d38f8bfd9e7a1a99c26c1a12e4f475cb1b383 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 16 Oct 2024 21:11:24 +0200 Subject: [PATCH 4/4] dump/zip: test that files are compressed --- internal/dump/zip_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/dump/zip_test.go b/internal/dump/zip_test.go index 6f5f60f54..c6eb04206 100644 --- a/internal/dump/zip_test.go +++ b/internal/dump/zip_test.go @@ -101,6 +101,9 @@ func checkZip(t *testing.T, testDir string, srcZip *bytes.Buffer) error { return fmt.Errorf("symlink target does not match, got %s want %s", string(linkName), target) } default: + if f.Method != zip.Deflate { + return fmt.Errorf("expected compression method got %v want %v", f.Method, zip.Deflate) + } if uint64(match.Size()) != f.UncompressedSize64 { return fmt.Errorf("size does not match got %v want %v", f.UncompressedSize64, match.Size()) }