dump: compress zip archives

This commit is contained in:
Phillipp Röll 2024-09-15 21:01:23 +02:00
parent bad6c54a33
commit 1a7fafc7eb
8 changed files with 22 additions and 45 deletions

View file

@ -1,10 +1,7 @@
Enhancement: Add `--compress` flag to `dump` command to compress archive Enhancement: Compress ZIP archives created by `dump` command
Restic did not compress the archives that was created by using the Restic did not compress the archives that were created by using
`dump` command. It now allows to save some disk space when exporting the `dump` command. It now saves some disk space when exporting
archives by adding a `--compress` flag. The DEFLATE algorithm is used archives using the DEFLATE algorithm for "zip" archives.
for "zip" archives, and the gzip algorithm for "tar" archives,
resulting in a .tar.gz or .tgz file. Not compressing the archive
is still the default.
https://github.com/restic/restic/pull/5054 https://github.com/restic/restic/pull/5054

View file

@ -52,7 +52,6 @@ type DumpOptions struct {
restic.SnapshotFilter restic.SnapshotFilter
Archive string Archive string
Target string Target string
Compress bool
} }
var dumpOptions DumpOptions var dumpOptions DumpOptions
@ -64,7 +63,6 @@ func init() {
initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`") flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`")
flags.BoolVarP(&dumpOptions.Compress, "compress", "c", false, "compress archive contents. When enabled, the DEFLATE algorithm is applied for \"zip\" archives, and the gzip algorithm for \"tar\" archives, resulting in a .tar.gz or .tgz file. (default: false)")
} }
func splitPath(p string) []string { func splitPath(p string) []string {
@ -189,7 +187,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
canWriteArchiveFunc = func() error { return nil } canWriteArchiveFunc = func() error { return nil }
} }
d := dump.New(opts.Archive, opts.Compress, repo, outputFileWriter) d := dump.New(opts.Archive, repo, outputFileWriter)
err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc)
if err != nil { if err != nil {
return errors.Fatalf("cannot dump file: %v", err) return errors.Fatalf("cannot dump file: %v", err)

View file

@ -19,16 +19,14 @@ type Dumper struct {
format string format string
repo restic.Loader repo restic.Loader
w io.Writer w io.Writer
compress bool
} }
func New(format string, compress bool, repo restic.Loader, w io.Writer) *Dumper { func New(format string, repo restic.Loader, w io.Writer) *Dumper {
return &Dumper{ return &Dumper{
cache: bloblru.New(64 << 20), cache: bloblru.New(64 << 20),
format: format, format: format,
repo: repo, repo: repo,
w: w, w: w,
compress: compress,
} }
} }

View file

@ -23,7 +23,7 @@ func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.R
type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error
func WriteTest(t *testing.T, format string, compress bool, cd CheckDump) { func WriteTest(t *testing.T, format string, cd CheckDump) {
tests := []struct { tests := []struct {
name string name string
args archiver.TestDir args archiver.TestDir
@ -85,7 +85,7 @@ func WriteTest(t *testing.T, format string, compress bool, cd CheckDump) {
rtest.OK(t, err) rtest.OK(t, err)
dst := &bytes.Buffer{} dst := &bytes.Buffer{}
d := New(format, compress, repo, dst) d := New(format, repo, dst)
if err := d.DumpTree(ctx, tree, tt.target); err != nil { if err := d.DumpTree(ctx, tree, tt.target); err != nil {
t.Fatalf("Dumper.Run error = %v", err) t.Fatalf("Dumper.Run error = %v", err)
} }

View file

@ -2,7 +2,6 @@ package dump
import ( import (
"archive/tar" "archive/tar"
"compress/gzip"
"context" "context"
"fmt" "fmt"
"os" "os"
@ -14,22 +13,12 @@ import (
) )
func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *restic.Node) (err error) { func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *restic.Node) (err error) {
outer := d.w w := tar.NewWriter(d.w)
if d.compress {
outer = gzip.NewWriter(outer)
}
w := tar.NewWriter(outer)
defer func() { defer func() {
if err == nil { if err == nil {
err = w.Close() err = w.Close()
err = errors.Wrap(err, "Close") err = errors.Wrap(err, "Close")
if gz, ok := outer.(*gzip.Writer); ok {
err = gz.Close()
err = errors.Wrap(err, "Close")
}
} }
}() }()

View file

@ -18,8 +18,7 @@ import (
) )
func TestWriteTar(t *testing.T) { func TestWriteTar(t *testing.T) {
WriteTest(t, "tar", false, checkTar) WriteTest(t, "tar", checkTar)
WriteTest(t, "tar", true, checkTar)
} }
func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error { func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error {

View file

@ -39,10 +39,7 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri
Modified: node.ModTime, Modified: node.ModTime,
} }
header.SetMode(node.Mode) header.SetMode(node.Mode)
if d.compress {
header.Method = zip.Deflate header.Method = zip.Deflate
}
if node.Type == restic.NodeTypeDir { if node.Type == restic.NodeTypeDir {
header.Name += "/" header.Name += "/"

View file

@ -12,8 +12,7 @@ import (
) )
func TestWriteZip(t *testing.T) { func TestWriteZip(t *testing.T) {
WriteTest(t, "zip", true, checkZip) WriteTest(t, "zip", checkZip)
WriteTest(t, "zip", false, checkZip)
} }
func readZipFile(f *zip.File) ([]byte, error) { func readZipFile(f *zip.File) ([]byte, error) {