diff --git a/changelog/unreleased/issue-2291 b/changelog/unreleased/issue-2291 index 0ee9a7196..407e01765 100644 --- a/changelog/unreleased/issue-2291 +++ b/changelog/unreleased/issue-2291 @@ -1,7 +1,7 @@ Enhancement: Allow pack size customization Restic now uses a target pack size of 16 MiB by default. It can be customized -using the `--packsize size` option. Supported pack sizes range between 4 and +using the `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB. It is possible to migrate an existing repository to _larger_ pack files using diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index ee3ec4b10..8742990f4 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -88,7 +88,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { s, err := repository.New(be, repository.Options{ Compression: gopts.Compression, - PackSize: gopts.MinPackSize * 1024 * 1024, + PackSize: gopts.PackSize * 1024 * 1024, }) if err != nil { return err diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index dc7fa5a74..7a55ccc88 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -425,7 +425,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption var repackCandidates []packInfoWithID repoVersion := repo.Config().Version - minPackSize := repo.MinPackSize() + targetPackSize := repo.PackSize() // loop over all packs and decide what to do bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed") @@ -467,7 +467,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption // use a flag that pack must be compressed p.uncompressed = mustCompress - packIsLargeEnough := !opts.RepackSmall || packSize >= int64(minPackSize) + packIsLargeEnough := !opts.RepackSmall || packSize >= int64(targetPackSize) // decide what to do switch { @@ -539,9 +539,9 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption return true case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob: return false - case opts.RepackSmall && pi.unusedSize+pi.usedSize < uint64(minPackSize) && pj.unusedSize+pj.usedSize >= uint64(minPackSize): + case opts.RepackSmall && pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize): return true - case opts.RepackSmall && pj.unusedSize+pj.usedSize < uint64(minPackSize) && pi.unusedSize+pi.usedSize >= uint64(minPackSize): + case opts.RepackSmall && pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize): return false } return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize @@ -561,7 +561,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption for _, p := range repackCandidates { reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter) reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes - packIsLargeEnough := !opts.RepackSmall || p.unusedSize+p.usedSize >= uint64(minPackSize) + packIsLargeEnough := !opts.RepackSmall || p.unusedSize+p.usedSize >= uint64(targetPackSize) switch { case reachedRepackSize: diff --git a/cmd/restic/global.go b/cmd/restic/global.go index d52dee34f..2173a22b7 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -63,7 +63,7 @@ type GlobalOptions struct { NoCache bool CleanupCache bool Compression repository.CompressionMode - MinPackSize uint + PackSize uint backend.TransportOptions limiter.Limits @@ -104,8 +104,8 @@ func init() { return nil }) - // parse min pack size from env, on error the default value will be used - minPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_MIN_PACKSIZE"), 10, 32) + // parse target pack size from env, on error the default value will be used + targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32) f := cmdRoot.PersistentFlags() f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)") @@ -126,7 +126,7 @@ func init() { f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") - f.UintVar(&globalOptions.MinPackSize, "min-packsize", uint(minPackSize), "set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE)") + f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB. (default: $RESTIC_PACK_SIZE)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") // Use our "generate" command instead of the cobra provided "completion" command cmdRoot.CompletionOptions.DisableDefaultCmd = true @@ -448,7 +448,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { s, err := repository.New(be, repository.Options{ Compression: opts.Compression, - PackSize: opts.MinPackSize * 1024 * 1024, + PackSize: opts.PackSize * 1024 * 1024, }) if err != nil { return nil, err diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 1a126f841..7ae04cde5 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -554,6 +554,7 @@ environment variables. The following lists these environment variables: RESTIC_CACHE_DIR Location of the cache directory RESTIC_COMPRESSION Compression mode (only available for repository format version 2) RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated + RESTIC_PACK_SIZE Target size for pack files TMPDIR Location for temporary files diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index b3d19512c..642847b22 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -59,8 +59,8 @@ upload connections, it is desirable to use larger pack sizes to reduce the numbe files in the repository and improve upload performance. Notable examples are OpenStack Swift and some Google Drive Team accounts, where there are hard limits on the total number of files. Larger pack sizes can also improve the backup speed for a repository -stored on a local HDD. This can be achieved by either using the ``--min-packsize`` option -or defining the ``$RESTIC_MIN_PACKSIZE`` environment variable. Restic currently defaults +stored on a local HDD. This can be achieved by either using the ``--pack-size`` option +or defining the ``$RESTIC_PACK_SIZE`` environment variable. Restic currently defaults to a 16 MiB pack size. The side effect of increasing the pack size is requiring more disk space for temporary pack diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 317fb51d4..e17e5cd8e 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -56,7 +56,7 @@ Usage help is available: --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) - --min-packsize uint set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE) + --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE) --no-cache do not use a local cache --no-lock do not lock the repository, this allows some operations on read-only repositories -o, --option key=value set extended option (key=value, can be specified multiple times) @@ -129,7 +129,7 @@ command: --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) - --min-packsize uint set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE) + --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE) --no-cache do not use a local cache --no-lock do not lock the repository, this allows some operations on read-only repositories -o, --option key=value set extended option (key=value, can be specified multiple times) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 2b7750648..872b2d71e 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -143,8 +143,8 @@ func (r *Repository) Config() restic.Config { return r.cfg } -// MinPackSize return the minimum size of a pack file before uploading -func (r *Repository) MinPackSize() uint { +// PackSize return the target size of a pack file when uploading +func (r *Repository) PackSize() uint { return r.opts.PackSize } @@ -516,8 +516,8 @@ func (r *Repository) StartPackUploader(ctx context.Context, wg *errgroup.Group) innerWg, ctx := errgroup.WithContext(ctx) r.packerWg = innerWg r.uploader = newPackerUploader(ctx, innerWg, r, r.be.Connections()) - r.treePM = newPackerManager(r.key, restic.TreeBlob, r.MinPackSize(), r.uploader.QueuePacker) - r.dataPM = newPackerManager(r.key, restic.DataBlob, r.MinPackSize(), r.uploader.QueuePacker) + r.treePM = newPackerManager(r.key, restic.TreeBlob, r.PackSize(), r.uploader.QueuePacker) + r.dataPM = newPackerManager(r.key, restic.DataBlob, r.PackSize(), r.uploader.QueuePacker) wg.Go(func() error { return innerWg.Wait() diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 1e64289e5..36f5a73bf 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -25,7 +25,7 @@ type Repository interface { LookupBlobSize(ID, BlobType) (uint, bool) Config() Config - MinPackSize() uint + PackSize() uint // List calls the function fn for each file of type t in the repository. // When an error is returned by fn, processing stops and List() returns the