KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

rename option to --pack-size

Browse source
This commit is contained in:
Michael Eischer 2022-07-02 23:52:02 +02:00
parent d7e2892048
commit 1b076cda97
9 changed files with 22 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
changelog/unreleased/issue-2291
View file

@ -1,7 +1,7 @@
Enhancement: Allow pack size customization Enhancement: Allow pack size customization
Restic now uses a target pack size of 16 MiB by default. It can be customized Restic now uses a target pack size of 16 MiB by default. It can be customized
using the `--packsize size` option. Supported pack sizes range between 4 and using the `--pack-size size` option. Supported pack sizes range between 4 and
128 MiB. 128 MiB.
It is possible to migrate an existing repository to _larger_ pack files using It is possible to migrate an existing repository to _larger_ pack files using

2
cmd/restic/cmd_init.go
View file

@ -88,7 +88,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
s, err := repository.New(be, repository.Options{ s, err := repository.New(be, repository.Options{
Compression: gopts.Compression, Compression: gopts.Compression,
PackSize: gopts.MinPackSize * 1024 * 1024, PackSize: gopts.PackSize * 1024 * 1024,
}) })
if err != nil { if err != nil {
return err return err

10
cmd/restic/cmd_prune.go
View file

@ -425,7 +425,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
var repackCandidates []packInfoWithID var repackCandidates []packInfoWithID
repoVersion := repo.Config().Version repoVersion := repo.Config().Version
minPackSize := repo.MinPackSize() targetPackSize := repo.PackSize()
// loop over all packs and decide what to do // loop over all packs and decide what to do
bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed") bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
@ -467,7 +467,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
// use a flag that pack must be compressed // use a flag that pack must be compressed
p.uncompressed = mustCompress p.uncompressed = mustCompress
packIsLargeEnough := !opts.RepackSmall || packSize >= int64(minPackSize) packIsLargeEnough := !opts.RepackSmall || packSize >= int64(targetPackSize)
// decide what to do // decide what to do
switch { switch {
@ -539,9 +539,9 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
return true return true
case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob: case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
return false return false
case opts.RepackSmall && pi.unusedSize+pi.usedSize < uint64(minPackSize) && pj.unusedSize+pj.usedSize >= uint64(minPackSize): case opts.RepackSmall && pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
return true return true
case opts.RepackSmall && pj.unusedSize+pj.usedSize < uint64(minPackSize) && pi.unusedSize+pi.usedSize >= uint64(minPackSize): case opts.RepackSmall && pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
return false return false
} }
return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
@ -561,7 +561,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
for _, p := range repackCandidates { for _, p := range repackCandidates {
reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter) reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
packIsLargeEnough := !opts.RepackSmall || p.unusedSize+p.usedSize >= uint64(minPackSize) packIsLargeEnough := !opts.RepackSmall || p.unusedSize+p.usedSize >= uint64(targetPackSize)
switch { switch {
case reachedRepackSize: case reachedRepackSize:

10
cmd/restic/global.go
View file

@ -63,7 +63,7 @@ type GlobalOptions struct {
NoCache bool NoCache bool
CleanupCache bool CleanupCache bool
Compression repository.CompressionMode Compression repository.CompressionMode
MinPackSize uint PackSize uint
backend.TransportOptions backend.TransportOptions
limiter.Limits limiter.Limits
@ -104,8 +104,8 @@ func init() {
return nil return nil
}) })
// parse min pack size from env, on error the default value will be used // parse target pack size from env, on error the default value will be used
minPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_MIN_PACKSIZE"), 10, 32) targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
f := cmdRoot.PersistentFlags() f := cmdRoot.PersistentFlags()
f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)") f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
@ -126,7 +126,7 @@ func init() {
f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)")
f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
f.UintVar(&globalOptions.MinPackSize, "min-packsize", uint(minPackSize), "set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE)") f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB. (default: $RESTIC_PACK_SIZE)")
f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
// Use our "generate" command instead of the cobra provided "completion" command // Use our "generate" command instead of the cobra provided "completion" command
cmdRoot.CompletionOptions.DisableDefaultCmd = true cmdRoot.CompletionOptions.DisableDefaultCmd = true
@ -448,7 +448,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
s, err := repository.New(be, repository.Options{ s, err := repository.New(be, repository.Options{
Compression: opts.Compression, Compression: opts.Compression,
PackSize: opts.MinPackSize * 1024 * 1024, PackSize: opts.PackSize * 1024 * 1024,
}) })
if err != nil { if err != nil {
return nil, err return nil, err

1
doc/040_backup.rst
View file

@ -554,6 +554,7 @@ environment variables. The following lists these environment variables:
RESTIC_CACHE_DIR Location of the cache directory RESTIC_CACHE_DIR Location of the cache directory
RESTIC_COMPRESSION Compression mode (only available for repository format version 2) RESTIC_COMPRESSION Compression mode (only available for repository format version 2)
RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated
RESTIC_PACK_SIZE Target size for pack files
TMPDIR Location for temporary files TMPDIR Location for temporary files

4
doc/047_tuning_backup_parameters.rst
View file

@ -59,8 +59,8 @@ upload connections, it is desirable to use larger pack sizes to reduce the numbe
files in the repository and improve upload performance. Notable examples are OpenStack files in the repository and improve upload performance. Notable examples are OpenStack
Swift and some Google Drive Team accounts, where there are hard limits on the total Swift and some Google Drive Team accounts, where there are hard limits on the total
number of files. Larger pack sizes can also improve the backup speed for a repository number of files. Larger pack sizes can also improve the backup speed for a repository
stored on a local HDD. This can be achieved by either using the ``--min-packsize`` option stored on a local HDD. This can be achieved by either using the ``--pack-size`` option
or defining the ``$RESTIC_MIN_PACKSIZE`` environment variable. Restic currently defaults or defining the ``$RESTIC_PACK_SIZE`` environment variable. Restic currently defaults
to a 16 MiB pack size. to a 16 MiB pack size.
The side effect of increasing the pack size is requiring more disk space for temporary pack The side effect of increasing the pack size is requiring more disk space for temporary pack

4
doc/manual_rest.rst
View file

@ -56,7 +56,7 @@ Usage help is available:
--key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
--limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited)
--limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited)
--min-packsize uint set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE) --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE)
--no-cache do not use a local cache --no-cache do not use a local cache
--no-lock do not lock the repository, this allows some operations on read-only repositories --no-lock do not lock the repository, this allows some operations on read-only repositories
-o, --option key=value set extended option (key=value, can be specified multiple times) -o, --option key=value set extended option (key=value, can be specified multiple times)
@ -129,7 +129,7 @@ command:
--key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
--limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited)
--limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited)
--min-packsize uint set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE) --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE)
--no-cache do not use a local cache --no-cache do not use a local cache
--no-lock do not lock the repository, this allows some operations on read-only repositories --no-lock do not lock the repository, this allows some operations on read-only repositories
-o, --option key=value set extended option (key=value, can be specified multiple times) -o, --option key=value set extended option (key=value, can be specified multiple times)

8
internal/repository/repository.go
View file

@ -143,8 +143,8 @@ func (r *Repository) Config() restic.Config {
return r.cfg return r.cfg
} }
// MinPackSize return the minimum size of a pack file before uploading // PackSize return the target size of a pack file when uploading
func (r *Repository) MinPackSize() uint { func (r *Repository) PackSize() uint {
return r.opts.PackSize return r.opts.PackSize
} }
@ -516,8 +516,8 @@ func (r *Repository) StartPackUploader(ctx context.Context, wg *errgroup.Group)
innerWg, ctx := errgroup.WithContext(ctx) innerWg, ctx := errgroup.WithContext(ctx)
r.packerWg = innerWg r.packerWg = innerWg
r.uploader = newPackerUploader(ctx, innerWg, r, r.be.Connections()) r.uploader = newPackerUploader(ctx, innerWg, r, r.be.Connections())
r.treePM = newPackerManager(r.key, restic.TreeBlob, r.MinPackSize(), r.uploader.QueuePacker) r.treePM = newPackerManager(r.key, restic.TreeBlob, r.PackSize(), r.uploader.QueuePacker)
r.dataPM = newPackerManager(r.key, restic.DataBlob, r.MinPackSize(), r.uploader.QueuePacker) r.dataPM = newPackerManager(r.key, restic.DataBlob, r.PackSize(), r.uploader.QueuePacker)
wg.Go(func() error { wg.Go(func() error {
return innerWg.Wait() return innerWg.Wait()

2
internal/restic/repository.go
View file

@ -25,7 +25,7 @@ type Repository interface {
LookupBlobSize(ID, BlobType) (uint, bool) LookupBlobSize(ID, BlobType) (uint, bool)
Config() Config Config() Config
MinPackSize() uint PackSize() uint
// List calls the function fn for each file of type t in the repository. // List calls the function fn for each file of type t in the repository.
// When an error is returned by fn, processing stops and List() returns the // When an error is returned by fn, processing stops and List() returns the