Compare commits

..

No commits in common. "master" and "tcl/master" have entirely different histories.

233 changed files with 5772 additions and 4666 deletions

View file

@ -28,15 +28,13 @@ Checklist
You do not need to check all the boxes below all at once. Feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box. Enable a checkbox by replacing [ ] with [x].
Please always follow these steps:
- Read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
- Enable [maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
- Run `gofmt` on the code in all commits.
- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
-->
- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
- [ ] I have added tests for all code changes.
- [ ] I have added documentation for relevant changes (in the manual).
- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
- [ ] I have run `gofmt` on the code in all commits.
- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
- [ ] I'm done! This pull request is ready for review.

View file

@ -13,7 +13,7 @@ permissions:
contents: read
env:
latest_go: "1.23.x"
latest_go: "1.22.x"
GO111MODULE: on
jobs:
@ -23,34 +23,39 @@ jobs:
# list of jobs to run:
include:
- job_name: Windows
go: 1.23.x
go: 1.22.x
os: windows-latest
- job_name: macOS
go: 1.23.x
go: 1.22.x
os: macOS-latest
test_fuse: false
- job_name: Linux
go: 1.23.x
go: 1.22.x
os: ubuntu-latest
test_cloud_backends: true
test_fuse: true
check_changelog: true
- job_name: Linux (race)
go: 1.23.x
go: 1.22.x
os: ubuntu-latest
test_fuse: true
test_opts: "-race"
- job_name: Linux
go: 1.22.x
go: 1.21.x
os: ubuntu-latest
test_fuse: true
- job_name: Linux
go: 1.21.x
go: 1.20.x
os: ubuntu-latest
test_fuse: true
- job_name: Linux
go: 1.19.x
os: ubuntu-latest
test_fuse: true
@ -259,7 +264,7 @@ jobs:
uses: golangci/golangci-lint-action@v6
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.61.0
version: v1.57.1
args: --verbose --timeout 5m
# only run golangci-lint for pull requests, otherwise ALL hints get

View file

@ -1 +1 @@
0.17.3-dev
0.17.3

View file

@ -58,7 +58,7 @@ var config = Config{
Main: "./cmd/restic", // package name for the main package
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 21, Patch: 0}, // minimum Go version supported
MinVersion: GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
}
// Config configures the build.

View file

@ -5,8 +5,6 @@ Enhancement: Allow custom bar in the foo command
# Describe the problem in the past tense, the new behavior in the present
# tense. Mention the affected commands, backends, operating systems, etc.
# If the problem description just says that a feature was missing, then
# only explain the new behavior.
# Focus on user-facing behavior, not the implementation.
# Use "Restic now ..." instead of "We have changed ...".

View file

@ -1,9 +0,0 @@
Bugfix: Correctly restore timestamp on long filepaths on old Windows versions
The `restore` command did not restore timestamps on file paths longer than 256
characters on Windows versions before Windows 10 1607.
This issue is now resolved.
https://github.com/restic/restic/issues/1843
https://github.com/restic/restic/pull/5061

View file

@ -1,16 +0,0 @@
Bugfix: Ignore disappeared backup source files
If during a backup files were removed between restic listing the directory
content and backing up the file in question, the following error could occur:
```
error: lstat /some/file/name: no such file or directory
```
The backup command now ignores this particular error and silently skips the
removed file.
https://github.com/restic/restic/issues/2165
https://github.com/restic/restic/issues/3098
https://github.com/restic/restic/pull/5143
https://github.com/restic/restic/pull/5145

View file

@ -1,6 +0,0 @@
Enhancement: Allow generating shell completions to stdout
Restic `generate` now supports passing `-` passed as file name to `--[shell]-completion` option.
https://github.com/restic/restic/issues/2511
https://github.com/restic/restic/pull/5053

View file

@ -1,21 +0,0 @@
Enhancement: Add config option to set Microsoft Blob Storage Access Tier
The `azure.access-tier` option can be passed to Restic (using `-o`) to
specify the access tier for Microsoft Blob Storage objects created by Restic.
The access tier is passed as-is to Microsoft Blob Storage, so it needs to be
understood by the API. The allowed values are `Hot`, `Cool`, or `Cold`.
If unspecified, the default is inferred from the default configured on the
storage account.
You can mix access tiers in the same container, and the setting isn't
stored in the restic repository, so be sure to specify it with each
command that writes to Microsoft Blob Storage.
There is no official `Archive` storage support in restic, use this option at
your own risk. To restore any data, it is still necessary to manually warm up
the required data in the `Archive` tier.
https://github.com/restic/restic/issues/4521
https://github.com/restic/restic/pull/5046

View file

@ -1,6 +0,0 @@
Enhancement: Format exit errors as JSON with --json
Restic now prints any exit error messages as JSON when requested.
https://github.com/restic/restic/issues/4948
https://github.com/restic/restic/pull/4952

View file

@ -1,7 +0,0 @@
Enhancement: Retry loading repository config
Restic now retries loading the repository config file when opening a repository.
In addition, the `init` command now also retries backend operations.
https://github.com/restic/restic/issues/5081
https://github.com/restic/restic/pull/5095

View file

@ -1,8 +0,0 @@
Enhancement: Indicate the of deleted files/directories during restore
Restic now indicates the number of deleted files/directories during restore.
The `--json` output now includes a `files_deleted` field that shows the number
of files and directories that were deleted during restore.
https://github.com/restic/restic/issues/5092
https://github.com/restic/restic/pull/5100

View file

@ -1,6 +0,0 @@
Enhancement: Add DragonflyBSD support
Restic can now be compiled on DragonflyBSD.
https://github.com/restic/restic/issues/5131
https://github.com/restic/restic/pull/5138

View file

@ -1,7 +0,0 @@
Change: Update dependencies and require Go 1.21 or newer
We have updated all dependencies. Since some libraries require newer Go standard
library features, support for Go 1.19 and 1.20 has been dropped, which means that
restic now requires at least Go 1.21 to build.
https://github.com/restic/restic/pull/4938

View file

@ -1,7 +0,0 @@
Enhancement: Compress ZIP archives created by `dump` command
Restic did not compress the archives that were created by using
the `dump` command. It now saves some disk space when exporting
archives using the DEFLATE algorithm for "zip" archives.
https://github.com/restic/restic/pull/5054

View file

@ -1,6 +0,0 @@
Enhancement: Include backup start and end in JSON output
The JSON output of the backup command now also includes the timestamps
of the `backup_start` and `backup_end` times.
https://github.com/restic/restic/pull/5119

View file

@ -1,7 +0,0 @@
Enhancement: Provide clear error message if AZURE_ACCOUNT_NAME is not set
If AZURE_ACCOUNT_NAME is not set, any command related to an Azure repository
would result in a misleading networking error. Restic will now detect this and
provide a clear warning that the variable is not defined.
https://github.com/restic/restic/pull/5141

View file

@ -20,12 +20,10 @@ import (
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/textfile"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/backup"
"github.com/restic/restic/internal/ui/termstatus"
)
@ -68,7 +66,7 @@ Exit status is 12 if the password is incorrect.
// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
filter.ExcludePatternOptions
excludePatternOptions
Parent string
GroupBy restic.SnapshotGroupByOptions
@ -111,7 +109,7 @@ func init() {
f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
backupOptions.ExcludePatternOptions.Add(f)
initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
@ -300,7 +298,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
// collectRejectByNameFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path only
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
// exclude restic cache
if repo.Cache != nil {
f, err := rejectResticCache(repo)
@ -311,12 +309,23 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
fs = append(fs, f)
}
fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
if err != nil {
return nil, err
}
for _, pat := range fsPatterns {
fs = append(fs, archiver.RejectByNameFunc(pat))
fs = append(fs, fsPatterns...)
if opts.ExcludeCaches {
opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
}
for _, spec := range opts.ExcludeIfPresent {
f, err := rejectIfPresent(spec)
if err != nil {
return nil, err
}
fs = append(fs, f)
}
return fs, nil
@ -324,43 +333,25 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
// collectRejectFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path and file info
func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS) (funcs []archiver.RejectFunc, err error) {
func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) {
// allowed devices
if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand {
f, err := archiver.RejectByDevice(targets, fs)
if opts.ExcludeOtherFS && !opts.Stdin {
f, err := rejectByDevice(targets)
if err != nil {
return nil, err
}
funcs = append(funcs, f)
fs = append(fs, f)
}
if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin && !opts.StdinCommand {
maxSize, err := ui.ParseBytes(opts.ExcludeLargerThan)
if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin {
f, err := rejectBySize(opts.ExcludeLargerThan)
if err != nil {
return nil, err
}
f, err := archiver.RejectBySize(maxSize)
if err != nil {
return nil, err
}
funcs = append(funcs, f)
fs = append(fs, f)
}
if opts.ExcludeCaches {
opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
}
for _, spec := range opts.ExcludeIfPresent {
f, err := archiver.RejectIfPresent(spec, Warnf)
if err != nil {
return nil, err
}
funcs = append(funcs, f)
}
return funcs, nil
return fs, nil
}
// collectTargets returns a list of target files/dirs from several sources.
@ -515,6 +506,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
return err
}
// rejectFuncs collect functions that can reject items from the backup based on path and file info
rejectFuncs, err := collectRejectFuncs(opts, targets)
if err != nil {
return err
}
var parentSnapshot *restic.Snapshot
if !opts.Stdin {
parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
@ -536,11 +533,30 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
}
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
selectByNameFilter := func(item string) bool {
for _, reject := range rejectByNameFuncs {
if reject(item) {
return false
}
}
return true
}
selectFilter := func(item string, fi os.FileInfo) bool {
for _, reject := range rejectFuncs {
if reject(item, fi) {
return false
}
}
return true
}
var targetFS fs.FS = fs.Local{}
if runtime.GOOS == "windows" && opts.UseFsSnapshot {
if err = fs.HasSufficientPrivilegesForVSS(); err != nil {
@ -587,15 +603,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
targetFS = backupFSTestHook(targetFS)
}
// rejectFuncs collect functions that can reject items from the backup based on path and file info
rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS)
if err != nil {
return err
}
selectByNameFilter := archiver.CombineRejectByNames(rejectByNameFuncs)
selectFilter := archiver.CombineRejects(rejectFuncs)
wg, wgCtx := errgroup.WithContext(ctx)
cancelCtx, cancel := context.WithCancel(wgCtx)
defer cancel()

View file

@ -31,7 +31,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts
func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
rtest.Assert(t, err == nil, "Error while backing up: %v", err)
rtest.Assert(t, err == nil, "Error while backing up")
}
func TestBackup(t *testing.T) {
@ -132,7 +132,7 @@ type vssDeleteOriginalFS struct {
hasRemoved bool
}
func (f *vssDeleteOriginalFS) Lstat(name string) (*fs.ExtendedFileInfo, error) {
func (f *vssDeleteOriginalFS) Lstat(name string) (os.FileInfo, error) {
if !f.hasRemoved {
// call Lstat to trigger snapshot creation
_, _ = f.FS.Lstat(name)
@ -365,7 +365,12 @@ func TestBackupExclude(t *testing.T) {
for _, filename := range backupExcludeFilenames {
fp := filepath.Join(datadir, filename)
rtest.OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
rtest.OK(t, os.WriteFile(fp, []byte(filename), 0o666))
f, err := os.Create(fp)
rtest.OK(t, err)
fmt.Fprint(f, filename)
rtest.OK(t, f.Close())
}
snapshots := make(map[string]struct{})

View file

@ -39,24 +39,21 @@ func TestCollectTargets(t *testing.T) {
f1, err := os.Create(filepath.Join(dir, "fromfile"))
rtest.OK(t, err)
// Empty lines should be ignored. A line starting with '#' is a comment.
_, err = fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
rtest.OK(t, err)
fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
rtest.OK(t, f1.Close())
f2, err := os.Create(filepath.Join(dir, "fromfile-verbatim"))
rtest.OK(t, err)
for _, filename := range []string{fooSpace, barStar} {
// Empty lines should be ignored. CR+LF is allowed.
_, err = fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
rtest.OK(t, err)
fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
}
rtest.OK(t, f2.Close())
f3, err := os.Create(filepath.Join(dir, "fromfile-raw"))
rtest.OK(t, err)
for _, filename := range []string{"baz", "quux"} {
_, err = fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
rtest.OK(t, err)
fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
}
rtest.OK(t, err)
rtest.OK(t, f3.Close())

View file

@ -10,6 +10,7 @@ import (
"github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
@ -88,7 +89,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
for _, item := range oldDirs {
dir := filepath.Join(cachedir, item.Name())
err = os.RemoveAll(dir)
err = fs.RemoveAll(dir)
if err != nil {
Warnf("unable to remove %v: %v\n", dir, err)
}

View file

@ -14,6 +14,7 @@ import (
"github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
@ -201,7 +202,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
printer.P("using temporary cache in %v\n", tempdir)
cleanup = func() {
err := os.RemoveAll(tempdir)
err := fs.RemoveAll(tempdir)
if err != nil {
printer.E("error removing temporary cache directory: %v\n", err)
}
@ -244,12 +245,17 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
errorsFound := false
suggestIndexRebuild := false
suggestLegacyIndexRebuild := false
mixedFound := false
for _, hint := range hints {
switch hint.(type) {
case *checker.ErrDuplicatePacks:
term.Print(hint.Error())
suggestIndexRebuild = true
case *checker.ErrOldIndexFormat:
printer.E("error: %v\n", hint)
suggestLegacyIndexRebuild = true
errorsFound = true
case *checker.ErrMixedPack:
term.Print(hint.Error())
mixedFound = true
@ -262,6 +268,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
if suggestIndexRebuild {
term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
}
if suggestLegacyIndexRebuild {
printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
}
if mixedFound {
term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
}
@ -295,6 +304,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
errorsFound = true
printer.E("%v\n", err)
}
} else if err == checker.ErrLegacyLayout {
errorsFound = true
printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n")
} else {
errorsFound = true
printer.E("%v\n", err)

View file

@ -143,7 +143,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id)
if err != nil {
return err

View file

@ -108,9 +108,9 @@ func (s *DiffStat) Add(node *restic.Node) {
}
switch node.Type {
case restic.NodeTypeFile:
case "file":
s.Files++
case restic.NodeTypeDir:
case "dir":
s.Dirs++
default:
s.Others++
@ -124,7 +124,7 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
}
switch node.Type {
case restic.NodeTypeFile:
case "file":
for _, blob := range node.Content {
h := restic.BlobHandle{
ID: blob,
@ -132,7 +132,7 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
}
bs.Insert(h)
}
case restic.NodeTypeDir:
case "dir":
h := restic.BlobHandle{
ID: *node.Subtree,
Type: restic.TreeBlob,
@ -184,14 +184,14 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
}
name := path.Join(prefix, node.Name)
if node.Type == restic.NodeTypeDir {
if node.Type == "dir" {
name += "/"
}
c.printChange(NewChange(name, mode))
stats.Add(node)
addBlobs(blobs, node)
if node.Type == restic.NodeTypeDir {
if node.Type == "dir" {
err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
if err != nil && err != context.Canceled {
Warnf("error: %v\n", err)
@ -216,7 +216,7 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest
addBlobs(blobs, node)
if node.Type == restic.NodeTypeDir {
if node.Type == "dir" {
err := c.collectDir(ctx, blobs, *node.Subtree)
if err != nil && err != context.Canceled {
Warnf("error: %v\n", err)
@ -284,12 +284,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
mod += "T"
}
if node2.Type == restic.NodeTypeDir {
if node2.Type == "dir" {
name += "/"
}
if node1.Type == restic.NodeTypeFile &&
node2.Type == restic.NodeTypeFile &&
if node1.Type == "file" &&
node2.Type == "file" &&
!reflect.DeepEqual(node1.Content, node2.Content) {
mod += "M"
stats.ChangedFiles++
@ -311,7 +311,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
c.printChange(NewChange(name, mod))
}
if node1.Type == restic.NodeTypeDir && node2.Type == restic.NodeTypeDir {
if node1.Type == "dir" && node2.Type == "dir" {
var err error
if (*node1.Subtree).Equal(*node2.Subtree) {
err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree)
@ -324,13 +324,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
}
case t1 && !t2:
prefix := path.Join(prefix, name)
if node1.Type == restic.NodeTypeDir {
if node1.Type == "dir" {
prefix += "/"
}
c.printChange(NewChange(prefix, "-"))
stats.Removed.Add(node1)
if node1.Type == restic.NodeTypeDir {
if node1.Type == "dir" {
err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
if err != nil && err != context.Canceled {
Warnf("error: %v\n", err)
@ -338,13 +338,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
}
case !t1 && t2:
prefix := path.Join(prefix, name)
if node2.Type == restic.NodeTypeDir {
if node2.Type == "dir" {
prefix += "/"
}
c.printChange(NewChange(prefix, "+"))
stats.Added.Add(node2)
if node2.Type == restic.NodeTypeDir {
if node2.Type == "dir" {
err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
if err != nil && err != context.Canceled {
Warnf("error: %v\n", err)

View file

@ -95,15 +95,15 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
// first item it finds and dump that according to the switch case below.
if node.Name == pathComponents[0] {
switch {
case l == 1 && node.Type == restic.NodeTypeFile:
case l == 1 && dump.IsFile(node):
return d.WriteNode(ctx, node)
case l > 1 && node.Type == restic.NodeTypeDir:
case l > 1 && dump.IsDir(node):
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil {
return errors.Wrapf(err, "cannot load subtree for %q", item)
}
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
case node.Type == restic.NodeTypeDir:
case dump.IsDir(node):
if err := canWriteArchiveFunc(); err != nil {
return err
}
@ -114,7 +114,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
return d.DumpTree(ctx, subtree, item)
case l > 1:
return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
case node.Type != restic.NodeTypeFile:
case !dump.IsFile(node):
return fmt.Errorf("%q should be a file, but is a %q", item, node.Type)
}
}

View file

@ -298,7 +298,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
}
var errIfNoMatch error
if node.Type == restic.NodeTypeDir {
if node.Type == "dir" {
var childMayMatch bool
for _, pat := range f.pat.pattern {
mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
@ -357,7 +357,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
return nil
}
if node.Type == restic.NodeTypeDir && f.treeIDs != nil {
if node.Type == "dir" && f.treeIDs != nil {
treeID := node.Subtree
found := false
if _, ok := f.treeIDs[treeID.Str()]; ok {
@ -377,7 +377,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
}
}
if node.Type == restic.NodeTypeFile && f.blobIDs != nil {
if node.Type == "file" && f.blobIDs != nil {
for _, id := range node.Content {
if ctx.Err() != nil {
return ctx.Err()

View file

@ -1,8 +1,6 @@
package main
import (
"io"
"os"
"time"
"github.com/restic/restic/internal/errors"
@ -43,10 +41,10 @@ func init() {
cmdRoot.AddCommand(cmdGenerate)
fs := cmdGenerate.Flags()
fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file` (`-` for stdout)")
fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file` (`-` for stdout)")
fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file` (`-` for stdout)")
fs.StringVar(&genOpts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file` (`-` for stdout)")
fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
fs.StringVar(&genOpts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file`")
}
func writeManpages(dir string) error {
@ -67,44 +65,32 @@ func writeManpages(dir string) error {
return doc.GenManTree(cmdRoot, header, dir)
}
func writeCompletion(filename string, shell string, generate func(w io.Writer) error) (err error) {
func writeBashCompletion(file string) error {
if stdoutIsTerminal() {
Verbosef("writing %s completion file to %v\n", shell, filename)
Verbosef("writing bash completion file to %v\n", file)
}
var outWriter io.Writer
if filename != "-" {
var outFile *os.File
outFile, err = os.Create(filename)
if err != nil {
return
}
defer func() { err = outFile.Close() }()
outWriter = outFile
} else {
outWriter = globalOptions.stdout
}
err = generate(outWriter)
return
return cmdRoot.GenBashCompletionFile(file)
}
func checkStdoutForSingleShell(opts generateOptions) error {
completionFileOpts := []string{
opts.BashCompletionFile,
opts.FishCompletionFile,
opts.ZSHCompletionFile,
opts.PowerShellCompletionFile,
func writeFishCompletion(file string) error {
if stdoutIsTerminal() {
Verbosef("writing fish completion file to %v\n", file)
}
seenIsStdout := false
for _, completionFileOpt := range completionFileOpts {
if completionFileOpt == "-" {
if seenIsStdout {
return errors.Fatal("the generate command can generate shell completions to stdout for single shell only")
}
seenIsStdout = true
}
return cmdRoot.GenFishCompletionFile(file, true)
}
func writeZSHCompletion(file string) error {
if stdoutIsTerminal() {
Verbosef("writing zsh completion file to %v\n", file)
}
return nil
return cmdRoot.GenZshCompletionFile(file)
}
func writePowerShellCompletion(file string) error {
if stdoutIsTerminal() {
Verbosef("writing powershell completion file to %v\n", file)
}
return cmdRoot.GenPowerShellCompletionFile(file)
}
func runGenerate(opts generateOptions, args []string) error {
@ -119,34 +105,29 @@ func runGenerate(opts generateOptions, args []string) error {
}
}
err := checkStdoutForSingleShell(opts)
if err != nil {
return err
}
if opts.BashCompletionFile != "" {
err := writeCompletion(opts.BashCompletionFile, "bash", cmdRoot.GenBashCompletion)
err := writeBashCompletion(opts.BashCompletionFile)
if err != nil {
return err
}
}
if opts.FishCompletionFile != "" {
err := writeCompletion(opts.FishCompletionFile, "fish", func(w io.Writer) error { return cmdRoot.GenFishCompletion(w, true) })
err := writeFishCompletion(opts.FishCompletionFile)
if err != nil {
return err
}
}
if opts.ZSHCompletionFile != "" {
err := writeCompletion(opts.ZSHCompletionFile, "zsh", cmdRoot.GenZshCompletion)
err := writeZSHCompletion(opts.ZSHCompletionFile)
if err != nil {
return err
}
}
if opts.PowerShellCompletionFile != "" {
err := writeCompletion(opts.PowerShellCompletionFile, "powershell", cmdRoot.GenPowerShellCompletion)
err := writePowerShellCompletion(opts.PowerShellCompletionFile)
if err != nil {
return err
}

View file

@ -1,40 +0,0 @@
package main
import (
"bytes"
"strings"
"testing"
rtest "github.com/restic/restic/internal/test"
)
func TestGenerateStdout(t *testing.T) {
testCases := []struct {
name string
opts generateOptions
}{
{"bash", generateOptions{BashCompletionFile: "-"}},
{"fish", generateOptions{FishCompletionFile: "-"}},
{"zsh", generateOptions{ZSHCompletionFile: "-"}},
{"powershell", generateOptions{PowerShellCompletionFile: "-"}},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
err := runGenerate(tc.opts, []string{})
rtest.OK(t, err)
completionString := buf.String()
rtest.Assert(t, strings.Contains(completionString, "# "+tc.name+" completion for restic"), "has no expected completion header")
})
}
t.Run("Generate shell completions to stdout for two shells", func(t *testing.T) {
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
opts := generateOptions{BashCompletionFile: "-", FishCompletionFile: "-"}
err := runGenerate(opts, []string{})
rtest.Assert(t, err != nil, "generate shell completions to stdout for two shells fails")
})
}

View file

@ -66,7 +66,7 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
case "locks":
t = restic.LockFile
case "blobs":
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
if err != nil {
return err
}

View file

@ -75,17 +75,17 @@ func init() {
}
type lsPrinter interface {
Snapshot(sn *restic.Snapshot) error
Node(path string, node *restic.Node, isPrefixDirectory bool) error
LeaveDir(path string) error
Close() error
Snapshot(sn *restic.Snapshot)
Node(path string, node *restic.Node, isPrefixDirectory bool)
LeaveDir(path string)
Close()
}
type jsonLsPrinter struct {
enc *json.Encoder
}
func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) error {
func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
type lsSnapshot struct {
*restic.Snapshot
ID *restic.ID `json:"id"`
@ -94,21 +94,27 @@ func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) error {
StructType string `json:"struct_type"` // "snapshot", deprecated
}
return p.enc.Encode(lsSnapshot{
err := p.enc.Encode(lsSnapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
MessageType: "snapshot",
StructType: "snapshot",
})
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
}
// Print node in our custom JSON format, followed by a newline.
func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
if isPrefixDirectory {
return nil
return
}
err := lsNodeJSON(p.enc, path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
return lsNodeJSON(p.enc, path, node)
}
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
@ -131,7 +137,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
size uint64 // Target for Size pointer.
}{
Name: node.Name,
Type: string(node.Type),
Type: node.Type,
Path: path,
UID: node.UID,
GID: node.GID,
@ -147,15 +153,15 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
}
// Always print size for regular files, even when empty,
// but never for other types.
if node.Type == restic.NodeTypeFile {
if node.Type == "file" {
n.Size = &n.size
}
return enc.Encode(n)
}
func (p *jsonLsPrinter) LeaveDir(_ string) error { return nil }
func (p *jsonLsPrinter) Close() error { return nil }
func (p *jsonLsPrinter) LeaveDir(_ string) {}
func (p *jsonLsPrinter) Close() {}
type ncduLsPrinter struct {
out io.Writer
@ -165,17 +171,16 @@ type ncduLsPrinter struct {
// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) error {
func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
const NcduMajorVer = 1
const NcduMinorVer = 2
snapshotBytes, err := json.Marshal(sn)
if err != nil {
return err
Warnf("JSON encode failed: %v\n", err)
}
p.depth++
_, err = fmt.Fprintf(p.out, "[%d, %d, %s, [{\"name\":\"/\"}", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
return err
fmt.Fprintf(p.out, "[%d, %d, %s, [{\"name\":\"/\"}", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
}
func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
@ -203,7 +208,7 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
Dev: node.DeviceID,
Ino: node.Inode,
NLink: node.Links,
NotReg: node.Type != restic.NodeTypeDir && node.Type != restic.NodeTypeFile,
NotReg: node.Type != "dir" && node.Type != "file",
UID: node.UID,
GID: node.GID,
Mode: uint16(node.Mode & os.ModePerm),
@ -227,30 +232,27 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
return json.Marshal(outNode)
}
func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) error {
func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) {
out, err := lsNcduNode(path, node)
if err != nil {
return err
Warnf("JSON encode failed: %v\n", err)
}
if node.Type == restic.NodeTypeDir {
_, err = fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out))
if node.Type == "dir" {
fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out))
p.depth++
} else {
_, err = fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out))
fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out))
}
return err
}
func (p *ncduLsPrinter) LeaveDir(_ string) error {
func (p *ncduLsPrinter) LeaveDir(_ string) {
p.depth--
_, err := fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth))
return err
fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth))
}
func (p *ncduLsPrinter) Close() error {
_, err := fmt.Fprint(p.out, "\n]\n]\n")
return err
func (p *ncduLsPrinter) Close() {
fmt.Fprint(p.out, "\n]\n]\n")
}
type textLsPrinter struct {
@ -259,23 +261,17 @@ type textLsPrinter struct {
HumanReadable bool
}
func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) error {
func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
Verbosef("%v filtered by %v:\n", sn, p.dirs)
return nil
}
func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
if !isPrefixDirectory {
Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
}
return nil
}
func (p *textLsPrinter) LeaveDir(_ string) error {
return nil
}
func (p *textLsPrinter) Close() error {
return nil
}
func (p *textLsPrinter) LeaveDir(_ string) {}
func (p *textLsPrinter) Close() {}
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
@ -378,9 +374,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
if err := printer.Snapshot(sn); err != nil {
return err
}
printer.Snapshot(sn)
processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
@ -393,9 +387,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
printedDir := false
if withinDir(nodepath) {
// if we're within a target path, print the node
if err := printer.Node(nodepath, node, false); err != nil {
return err
}
printer.Node(nodepath, node, false)
printedDir = true
// if recursive listing is requested, signal the walker that it
@ -410,19 +402,17 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
if approachingMatchingTree(nodepath) {
// print node leading up to the target paths
if !printedDir {
return printer.Node(nodepath, node, true)
printer.Node(nodepath, node, true)
}
return nil
}
// otherwise, signal the walker to not walk recursively into any
// subdirs
if node.Type == restic.NodeTypeDir {
if node.Type == "dir" {
// immediately generate leaveDir if the directory is skipped
if printedDir {
if err := printer.LeaveDir(nodepath); err != nil {
return err
}
printer.LeaveDir(nodepath)
}
return walker.ErrSkipNode
}
@ -431,12 +421,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
ProcessNode: processNode,
LeaveDir: func(path string) error {
LeaveDir: func(path string) {
// the root path `/` has no corresponding node and is thus also skipped by processNode
if path != "/" {
return printer.LeaveDir(path)
printer.LeaveDir(path)
}
return nil
},
})
@ -444,5 +433,6 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
return printer.Close()
printer.Close()
return nil
}

View file

@ -23,7 +23,7 @@ var lsTestNodes = []lsTestNode{
path: "/bar/baz",
Node: restic.Node{
Name: "baz",
Type: restic.NodeTypeFile,
Type: "file",
Size: 12345,
UID: 10000000,
GID: 20000000,
@ -39,7 +39,7 @@ var lsTestNodes = []lsTestNode{
path: "/foo/empty",
Node: restic.Node{
Name: "empty",
Type: restic.NodeTypeFile,
Type: "file",
Size: 0,
UID: 1001,
GID: 1001,
@ -56,7 +56,7 @@ var lsTestNodes = []lsTestNode{
path: "/foo/link",
Node: restic.Node{
Name: "link",
Type: restic.NodeTypeSymlink,
Type: "symlink",
Mode: os.ModeSymlink | 0777,
LinkTarget: "not printed",
},
@ -66,7 +66,7 @@ var lsTestNodes = []lsTestNode{
path: "/some/directory",
Node: restic.Node{
Name: "directory",
Type: restic.NodeTypeDir,
Type: "dir",
Mode: os.ModeDir | 0755,
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
@ -79,7 +79,7 @@ var lsTestNodes = []lsTestNode{
path: "/some/sticky",
Node: restic.Node{
Name: "sticky",
Type: restic.NodeTypeDir,
Type: "dir",
Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
},
},
@ -134,29 +134,29 @@ func TestLsNcdu(t *testing.T) {
}
modTime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
rtest.OK(t, printer.Snapshot(&restic.Snapshot{
printer.Snapshot(&restic.Snapshot{
Hostname: "host",
Paths: []string{"/example"},
}))
rtest.OK(t, printer.Node("/directory", &restic.Node{
Type: restic.NodeTypeDir,
})
printer.Node("/directory", &restic.Node{
Type: "dir",
Name: "directory",
ModTime: modTime,
}, false))
rtest.OK(t, printer.Node("/directory/data", &restic.Node{
Type: restic.NodeTypeFile,
}, false)
printer.Node("/directory/data", &restic.Node{
Type: "file",
Name: "data",
Size: 42,
ModTime: modTime,
}, false))
rtest.OK(t, printer.LeaveDir("/directory"))
rtest.OK(t, printer.Node("/file", &restic.Node{
Type: restic.NodeTypeFile,
}, false)
printer.LeaveDir("/directory")
printer.Node("/file", &restic.Node{
Type: "file",
Name: "file",
Size: 12345,
ModTime: modTime,
}, false))
rtest.OK(t, printer.Close())
}, false)
printer.Close()
rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, [{"name":"/"},
[

View file

@ -15,6 +15,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
resticfs "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/fuse"
systemFuse "github.com/anacrolix/fuse"
@ -121,7 +122,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
// Check the existence of the mount point at the earliest stage to
// prevent unnecessary computations while opening the repository.
if _, err := os.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
return err
}

View file

@ -74,7 +74,7 @@ func init() {
func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) {
f := c.Flags()
f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "stop after repacking this much data in total (allowed suffixes for `size`: k/K, m/M, g/G, t/T)")
f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
f.BoolVar(&pruneOptions.RepackCacheableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size")
f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")

View file

@ -88,7 +88,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
}
for _, node := range tree.Nodes {
if node.Type == restic.NodeTypeDir && node.Subtree != nil {
if node.Type == "dir" && node.Subtree != nil {
trees[*node.Subtree] = true
}
}
@ -128,7 +128,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
for id := range roots {
var subtreeID = id
node := restic.Node{
Type: restic.NodeTypeDir,
Type: "dir",
Name: id.Str(),
Mode: 0755,
Subtree: &subtreeID,

View file

@ -92,11 +92,11 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
// - files whose contents are not fully available (-> file will be modified)
rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if node.Type == restic.NodeTypeIrregular || node.Type == restic.NodeTypeInvalid {
if node.Type == "irregular" || node.Type == "" {
Verbosef(" file %q: removed node with invalid type %q\n", path, node.Type)
return nil
}
if node.Type != restic.NodeTypeFile {
if node.Type != "file" {
return node
}

View file

@ -7,7 +7,6 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer"
"github.com/restic/restic/internal/ui"
@ -50,8 +49,8 @@ Exit status is 12 if the password is incorrect.
// RestoreOptions collects all options for the restore command.
type RestoreOptions struct {
filter.ExcludePatternOptions
filter.IncludePatternOptions
excludePatternOptions
includePatternOptions
Target string
restic.SnapshotFilter
DryRun bool
@ -69,8 +68,8 @@ func init() {
flags := cmdRestore.Flags()
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
restoreOptions.ExcludePatternOptions.Add(flags)
restoreOptions.IncludePatternOptions.Add(flags)
initExcludePatternOptions(flags, &restoreOptions.excludePatternOptions)
initIncludePatternOptions(flags, &restoreOptions.includePatternOptions)
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done")
@ -83,12 +82,12 @@ func init() {
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
term *termstatus.Terminal, args []string) error {
excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
if err != nil {
return err
}
includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf)
includePatternFns, err := opts.includePatternOptions.CollectPatterns()
if err != nil {
return err
}

View file

@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
@ -402,21 +403,36 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
"meta data of intermediate directory hasn't been restore")
}
func TestRestoreDefaultLayout(t *testing.T) {
func TestRestoreLocalLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
env, cleanup := withTestEnvironment(t)
defer cleanup()
datafile := filepath.Join("..", "..", "internal", "backend", "testdata", "repo-layout-default.tar.gz")
var tests = []struct {
filename string
layout string
}{
{"repo-layout-default.tar.gz", ""},
{"repo-layout-s3legacy.tar.gz", ""},
{"repo-layout-default.tar.gz", "default"},
{"repo-layout-s3legacy.tar.gz", "s3legacy"},
}
rtest.SetupTarTestFixture(t, env.base, datafile)
for _, test := range tests {
datafile := filepath.Join("..", "..", "internal", "backend", "testdata", test.filename)
// check the repo
testRunCheck(t, env.gopts)
rtest.SetupTarTestFixture(t, env.base, datafile)
// restore latest snapshot
target := filepath.Join(env.base, "restore")
testRunRestoreLatest(t, env.gopts, target, nil, nil)
env.gopts.extended["local.layout"] = test.layout
rtest.RemoveAll(t, filepath.Join(env.base, "repo"))
rtest.RemoveAll(t, target)
// check the repo
testRunCheck(t, env.gopts)
// restore latest snapshot
target := filepath.Join(env.base, "restore")
testRunRestoreLatest(t, env.gopts, target, nil, nil)
rtest.RemoveAll(t, filepath.Join(env.base, "repo"))
rtest.RemoveAll(t, target)
}
}

View file

@ -9,7 +9,6 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
@ -88,7 +87,7 @@ type RewriteOptions struct {
Metadata snapshotMetadataArgs
restic.SnapshotFilter
filter.ExcludePatternOptions
excludePatternOptions
}
var rewriteOptions RewriteOptions
@ -103,7 +102,7 @@ func init() {
f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
rewriteOptions.ExcludePatternOptions.Add(f)
initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
}
type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
@ -113,7 +112,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
}
rejectByNameFuncs, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
if err != nil {
return false, err
}
@ -263,7 +262,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
}
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
}

View file

@ -5,7 +5,6 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
@ -13,7 +12,7 @@ import (
func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
opts := RewriteOptions{
ExcludePatternOptions: filter.ExcludePatternOptions{
excludePatternOptions: excludePatternOptions{
Excludes: excludes,
},
Forget: forget,

View file

@ -296,9 +296,7 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
}
// Info
if _, err := fmt.Fprintf(stdout, "snapshots"); err != nil {
return err
}
fmt.Fprintf(stdout, "snapshots")
var infoStrings []string
if key.Hostname != "" {
infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
@ -310,13 +308,11 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
}
if infoStrings != nil {
if _, err := fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", ")); err != nil {
return err
}
fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
}
_, err = fmt.Fprintf(stdout, ":\n")
fmt.Fprintf(stdout, ":\n")
return err
return nil
}
// Snapshot helps to print Snapshots as JSON with their ID included.
@ -333,7 +329,7 @@ type SnapshotGroup struct {
Snapshots []Snapshot `json:"snapshots"`
}
// printSnapshotGroupJSON writes the JSON representation of list to stdout.
// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {
if grouped {
snapshotGroups := []SnapshotGroup{}

View file

@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"path/filepath"
@ -17,6 +16,7 @@ import (
"github.com/restic/restic/internal/ui/table"
"github.com/restic/restic/internal/walker"
"github.com/minio/sha256-simd"
"github.com/spf13/cobra"
)
@ -276,7 +276,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
// will still be restored
stats.TotalFileCount++
if node.Links == 1 || node.Type == restic.NodeTypeDir {
if node.Links == 1 || node.Type == "dir" {
stats.TotalSize += node.Size
} else {
// if hardlinks are present only count each deviceID+inode once

View file

@ -25,19 +25,17 @@ Exit status is 1 if there was any error.
Run: func(_ *cobra.Command, _ []string) {
if globalOptions.JSON {
type jsonVersion struct {
MessageType string `json:"message_type"` // version
Version string `json:"version"`
GoVersion string `json:"go_version"`
GoOS string `json:"go_os"`
GoArch string `json:"go_arch"`
Version string `json:"version"`
GoVersion string `json:"go_version"`
GoOS string `json:"go_os"`
GoArch string `json:"go_arch"`
}
jsonS := jsonVersion{
MessageType: "version",
Version: version,
GoVersion: runtime.Version(),
GoOS: runtime.GOOS,
GoArch: runtime.GOARCH,
Version: version,
GoVersion: runtime.Version(),
GoOS: runtime.GOOS,
GoArch: runtime.GOARCH,
}
err := json.NewEncoder(globalOptions.stdout).Encode(jsonS)

View file

@ -1,16 +1,347 @@
package main
import (
"github.com/restic/restic/internal/archiver"
"bufio"
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/textfile"
"github.com/restic/restic/internal/ui"
"github.com/spf13/pflag"
)
type rejectionCache struct {
m map[string]bool
mtx sync.Mutex
}
// Lock locks the mutex in rc.
func (rc *rejectionCache) Lock() {
if rc != nil {
rc.mtx.Lock()
}
}
// Unlock unlocks the mutex in rc.
func (rc *rejectionCache) Unlock() {
if rc != nil {
rc.mtx.Unlock()
}
}
// Get returns the last stored value for dir and a second boolean that
// indicates whether that value was actually written to the cache. It is the
// callers responsibility to call rc.Lock and rc.Unlock before using this
// method, otherwise data races may occur.
func (rc *rejectionCache) Get(dir string) (bool, bool) {
if rc == nil || rc.m == nil {
return false, false
}
v, ok := rc.m[dir]
return v, ok
}
// Store stores a new value for dir. It is the callers responsibility to call
// rc.Lock and rc.Unlock before using this method, otherwise data races may
// occur.
func (rc *rejectionCache) Store(dir string, rejected bool) {
if rc == nil {
return
}
if rc.m == nil {
rc.m = make(map[string]bool)
}
rc.m[dir] = rejected
}
// RejectByNameFunc is a function that takes a filename of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectByNameFunc func(path string) bool
// RejectFunc is a function that takes a filename and os.FileInfo of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectFunc func(path string, fi os.FileInfo) bool
// rejectByPattern returns a RejectByNameFunc which rejects files that match
// one of the patterns.
func rejectByPattern(patterns []string) RejectByNameFunc {
parsedPatterns := filter.ParsePatterns(patterns)
return func(item string) bool {
matched, err := filter.List(parsedPatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}
if matched {
debug.Log("path %q excluded by an exclude pattern", item)
return true
}
return false
}
}
// Same as `rejectByPattern` but case insensitive.
func rejectByInsensitivePattern(patterns []string) RejectByNameFunc {
for index, path := range patterns {
patterns[index] = strings.ToLower(path)
}
rejFunc := rejectByPattern(patterns)
return func(item string) bool {
return rejFunc(strings.ToLower(item))
}
}
// rejectIfPresent returns a RejectByNameFunc which itself returns whether a path
// should be excluded. The RejectByNameFunc considers a file to be excluded when
// it resides in a directory with an exclusion file, that is specified by
// excludeFileSpec in the form "filename[:content]". The returned error is
// non-nil if the filename component of excludeFileSpec is empty. If rc is
// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation
// of a directory based on previous visits.
func rejectIfPresent(excludeFileSpec string) (RejectByNameFunc, error) {
if excludeFileSpec == "" {
return nil, errors.New("name for exclusion tagfile is empty")
}
colon := strings.Index(excludeFileSpec, ":")
if colon == 0 {
return nil, fmt.Errorf("no name for exclusion tagfile provided")
}
tf, tc := "", ""
if colon > 0 {
tf = excludeFileSpec[:colon]
tc = excludeFileSpec[colon+1:]
} else {
tf = excludeFileSpec
}
debug.Log("using %q as exclusion tagfile", tf)
rc := &rejectionCache{}
fn := func(filename string) bool {
return isExcludedByFile(filename, tf, tc, rc)
}
return fn, nil
}
// isExcludedByFile interprets filename as a path and returns true if that file
// is in an excluded directory. A directory is identified as excluded if it contains a
// tagfile which bears the name specified in tagFilename and starts with
// header. If rc is non-nil, it is used to expedite the evaluation of a
// directory based on previous visits.
func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache) bool {
if tagFilename == "" {
return false
}
dir, base := filepath.Split(filename)
if base == tagFilename {
return false // do not exclude the tagfile itself
}
rc.Lock()
defer rc.Unlock()
rejected, visited := rc.Get(dir)
if visited {
return rejected
}
rejected = isDirExcludedByFile(dir, tagFilename, header)
rc.Store(dir, rejected)
return rejected
}
func isDirExcludedByFile(dir, tagFilename, header string) bool {
tf := filepath.Join(dir, tagFilename)
_, err := fs.Lstat(tf)
if os.IsNotExist(err) {
return false
}
if err != nil {
Warnf("could not access exclusion tagfile: %v", err)
return false
}
// when no signature is given, the mere presence of tf is enough reason
// to exclude filename
if len(header) == 0 {
return true
}
// From this stage, errors mean tagFilename exists but it is malformed.
// Warnings will be generated so that the user is informed that the
// indented ignore-action is not performed.
f, err := os.Open(tf)
if err != nil {
Warnf("could not open exclusion tagfile: %v", err)
return false
}
defer func() {
_ = f.Close()
}()
buf := make([]byte, len(header))
_, err = io.ReadFull(f, buf)
// EOF is handled with a dedicated message, otherwise the warning were too cryptic
if err == io.EOF {
Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf)
return false
}
if err != nil {
Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err)
return false
}
if !bytes.Equal(buf, []byte(header)) {
Warnf("invalid signature in exclusion tagfile %q\n", tf)
return false
}
return true
}
// DeviceMap is used to track allowed source devices for backup. This is used to
// check for crossing mount points during backup (for --one-file-system). It
// maps the name of a source path to its device ID.
type DeviceMap map[string]uint64
// NewDeviceMap creates a new device map from the list of source paths.
func NewDeviceMap(allowedSourcePaths []string) (DeviceMap, error) {
deviceMap := make(map[string]uint64)
for _, item := range allowedSourcePaths {
item, err := filepath.Abs(filepath.Clean(item))
if err != nil {
return nil, err
}
fi, err := fs.Lstat(item)
if err != nil {
return nil, err
}
id, err := fs.DeviceID(fi)
if err != nil {
return nil, err
}
deviceMap[item] = id
}
if len(deviceMap) == 0 {
return nil, errors.New("zero allowed devices")
}
return deviceMap, nil
}
// IsAllowed returns true if the path is located on an allowed device.
func (m DeviceMap) IsAllowed(item string, deviceID uint64) (bool, error) {
for dir := item; ; dir = filepath.Dir(dir) {
debug.Log("item %v, test dir %v", item, dir)
// find a parent directory that is on an allowed device (otherwise
// we would not traverse the directory at all)
allowedID, ok := m[dir]
if !ok {
if dir == filepath.Dir(dir) {
// arrived at root, no allowed device found. this should not happen.
break
}
continue
}
// if the item has a different device ID than the parent directory,
// we crossed a file system boundary
if allowedID != deviceID {
debug.Log("item %v (dir %v) on disallowed device %d", item, dir, deviceID)
return false, nil
}
// item is on allowed device, accept it
debug.Log("item %v allowed", item)
return true, nil
}
return false, fmt.Errorf("item %v (device ID %v) not found, deviceMap: %v", item, deviceID, m)
}
// rejectByDevice returns a RejectFunc that rejects files which are on a
// different file systems than the files/dirs in samples.
func rejectByDevice(samples []string) (RejectFunc, error) {
deviceMap, err := NewDeviceMap(samples)
if err != nil {
return nil, err
}
debug.Log("allowed devices: %v\n", deviceMap)
return func(item string, fi os.FileInfo) bool {
id, err := fs.DeviceID(fi)
if err != nil {
// This should never happen because gatherDevices() would have
// errored out earlier. If it still does that's a reason to panic.
panic(err)
}
allowed, err := deviceMap.IsAllowed(filepath.Clean(item), id)
if err != nil {
// this should not happen
panic(fmt.Sprintf("error checking device ID of %v: %v", item, err))
}
if allowed {
// accept item
return false
}
// reject everything except directories
if !fi.IsDir() {
return true
}
// special case: make sure we keep mountpoints (directories which
// contain a mounted file system). Test this by checking if the parent
// directory would be included.
parentDir := filepath.Dir(filepath.Clean(item))
parentFI, err := fs.Lstat(parentDir)
if err != nil {
debug.Log("item %v: error running lstat() on parent directory: %v", item, err)
// if in doubt, reject
return true
}
parentDeviceID, err := fs.DeviceID(parentFI)
if err != nil {
debug.Log("item %v: getting device ID of parent directory: %v", item, err)
// if in doubt, reject
return true
}
parentAllowed, err := deviceMap.IsAllowed(parentDir, parentDeviceID)
if err != nil {
debug.Log("item %v: error checking parent directory: %v", item, err)
// if in doubt, reject
return true
}
if parentAllowed {
// we found a mount point, so accept the directory
return false
}
// reject everything else
return true
}, nil
}
// rejectResticCache returns a RejectByNameFunc that rejects the restic cache
// directory (if set).
func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc, error) {
func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
if repo.Cache == nil {
return func(string) bool {
return false
@ -31,3 +362,137 @@ func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc,
return false
}, nil
}
func rejectBySize(maxSizeStr string) (RejectFunc, error) {
maxSize, err := ui.ParseBytes(maxSizeStr)
if err != nil {
return nil, err
}
return func(item string, fi os.FileInfo) bool {
// directory will be ignored
if fi.IsDir() {
return false
}
filesize := fi.Size()
if filesize > maxSize {
debug.Log("file %s is oversize: %d", item, filesize)
return true
}
return false
}, nil
}
// readPatternsFromFiles reads all files and returns the list of
// patterns. For each line, leading and trailing white space is removed
// and comment lines are ignored. For each remaining pattern, environment
// variables are resolved. For adding a literal dollar sign ($), write $$ to
// the file.
func readPatternsFromFiles(files []string) ([]string, error) {
getenvOrDollar := func(s string) string {
if s == "$" {
return "$"
}
return os.Getenv(s)
}
var patterns []string
for _, filename := range files {
err := func() (err error) {
data, err := textfile.Read(filename)
if err != nil {
return err
}
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// ignore empty lines
if line == "" {
continue
}
// strip comments
if strings.HasPrefix(line, "#") {
continue
}
line = os.Expand(line, getenvOrDollar)
patterns = append(patterns, line)
}
return scanner.Err()
}()
if err != nil {
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
}
}
return patterns, nil
}
type excludePatternOptions struct {
Excludes []string
InsensitiveExcludes []string
ExcludeFiles []string
InsensitiveExcludeFiles []string
}
func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) {
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
}
func (opts *excludePatternOptions) Empty() bool {
return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
}
func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
var fs []RejectByNameFunc
// add patterns from file
if len(opts.ExcludeFiles) > 0 {
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
if err != nil {
return nil, err
}
if err := filter.ValidatePatterns(excludePatterns); err != nil {
return nil, errors.Fatalf("--exclude-file: %s", err)
}
opts.Excludes = append(opts.Excludes, excludePatterns...)
}
if len(opts.InsensitiveExcludeFiles) > 0 {
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
if err != nil {
return nil, err
}
if err := filter.ValidatePatterns(excludes); err != nil {
return nil, errors.Fatalf("--iexclude-file: %s", err)
}
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
}
if len(opts.InsensitiveExcludes) > 0 {
if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil {
return nil, errors.Fatalf("--iexclude: %s", err)
}
fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
}
if len(opts.Excludes) > 0 {
if err := filter.ValidatePatterns(opts.Excludes); err != nil {
return nil, errors.Fatalf("--exclude: %s", err)
}
fs = append(fs, rejectByPattern(opts.Excludes))
}
return fs, nil
}

View file

@ -1,14 +1,67 @@
package archiver
package main
import (
"os"
"path/filepath"
"testing"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/test"
)
func TestRejectByPattern(t *testing.T) {
var tests = []struct {
filename string
reject bool
}{
{filename: "/home/user/foo.go", reject: true},
{filename: "/home/user/foo.c", reject: false},
{filename: "/home/user/foobar", reject: false},
{filename: "/home/user/foobar/x", reject: true},
{filename: "/home/user/README", reject: false},
{filename: "/home/user/README.md", reject: true},
}
patterns := []string{"*.go", "README.md", "/home/user/foobar/*"}
for _, tc := range tests {
t.Run("", func(t *testing.T) {
reject := rejectByPattern(patterns)
res := reject(tc.filename)
if res != tc.reject {
t.Fatalf("wrong result for filename %v: want %v, got %v",
tc.filename, tc.reject, res)
}
})
}
}
func TestRejectByInsensitivePattern(t *testing.T) {
var tests = []struct {
filename string
reject bool
}{
{filename: "/home/user/foo.GO", reject: true},
{filename: "/home/user/foo.c", reject: false},
{filename: "/home/user/foobar", reject: false},
{filename: "/home/user/FOObar/x", reject: true},
{filename: "/home/user/README", reject: false},
{filename: "/home/user/readme.md", reject: true},
}
patterns := []string{"*.go", "README.md", "/home/user/foobar/*"}
for _, tc := range tests {
t.Run("", func(t *testing.T) {
reject := rejectByInsensitivePattern(patterns)
res := reject(tc.filename)
if res != tc.reject {
t.Fatalf("wrong result for filename %v: want %v, got %v",
tc.filename, tc.reject, res)
}
})
}
}
func TestIsExcludedByFile(t *testing.T) {
const (
tagFilename = "CACHEDIR.TAG"
@ -49,7 +102,7 @@ func TestIsExcludedByFile(t *testing.T) {
if tc.content == "" {
h = ""
}
if got := isExcludedByFile(foo, tagFilename, h, newRejectionCache(), &fs.Local{}, func(msg string, args ...interface{}) { t.Logf(msg, args...) }); tc.want != got {
if got := isExcludedByFile(foo, tagFilename, h, nil); tc.want != got {
t.Fatalf("expected %v, got %v", tc.want, got)
}
})
@ -100,8 +153,8 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
// create two rejection functions, one that tests for the NOFOO file
// and one for the NOBAR file
fooExclude, _ := RejectIfPresent("NOFOO", nil)
barExclude, _ := RejectIfPresent("NOBAR", nil)
fooExclude, _ := rejectIfPresent("NOFOO")
barExclude, _ := rejectIfPresent("NOBAR")
// To mock the archiver scanning walk, we create filepath.WalkFn
// that tests against the two rejection functions and stores
@ -111,8 +164,8 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
if err != nil {
return err
}
excludedByFoo := fooExclude(p, nil, &fs.Local{})
excludedByBar := barExclude(p, nil, &fs.Local{})
excludedByFoo := fooExclude(p)
excludedByBar := barExclude(p)
excluded := excludedByFoo || excludedByBar
// the log message helps debugging in case the test fails
t.Logf("%q: %v || %v = %v", p, excludedByFoo, excludedByBar, excluded)
@ -139,6 +192,9 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
func TestIsExcludedByFileSize(t *testing.T) {
tempDir := test.TempDir(t)
// Max size of file is set to be 1k
maxSizeStr := "1k"
// Create some files in a temporary directory.
// Files in UPPERCASE will be used as exclusion triggers later on.
// We will test the inclusion later, so we add the expected value as
@ -182,7 +238,7 @@ func TestIsExcludedByFileSize(t *testing.T) {
test.OKs(t, errs) // see if anything went wrong during the creation
// create rejection function
sizeExclude, _ := RejectBySize(1024)
sizeExclude, _ := rejectBySize(maxSizeStr)
// To mock the archiver scanning walk, we create filepath.WalkFn
// that tests against the two rejection functions and stores
@ -193,7 +249,7 @@ func TestIsExcludedByFileSize(t *testing.T) {
return err
}
excluded := sizeExclude(p, fs.ExtendedStat(fi), nil)
excluded := sizeExclude(p, fi)
// the log message helps debugging in case the test fails
t.Logf("%q: dir:%t; size:%d; excluded:%v", p, fi.IsDir(), fi.Size(), excluded)
m[p] = !excluded
@ -212,7 +268,7 @@ func TestIsExcludedByFileSize(t *testing.T) {
}
func TestDeviceMap(t *testing.T) {
deviceMap := deviceMap{
deviceMap := DeviceMap{
filepath.FromSlash("/"): 1,
filepath.FromSlash("/usr/local"): 5,
}
@ -243,7 +299,7 @@ func TestDeviceMap(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
res, err := deviceMap.IsAllowed(filepath.FromSlash(test.item), test.deviceID, &fs.Local{})
res, err := deviceMap.IsAllowed(filepath.FromSlash(test.item), test.deviceID)
if err != nil {
t.Fatal(err)
}

View file

@ -24,20 +24,20 @@ func formatNode(path string, n *restic.Node, long bool, human bool) string {
}
switch n.Type {
case restic.NodeTypeFile:
case "file":
mode = 0
case restic.NodeTypeDir:
case "dir":
mode = os.ModeDir
case restic.NodeTypeSymlink:
case "symlink":
mode = os.ModeSymlink
target = fmt.Sprintf(" -> %v", n.LinkTarget)
case restic.NodeTypeDev:
case "dev":
mode = os.ModeDevice
case restic.NodeTypeCharDev:
case "chardev":
mode = os.ModeDevice | os.ModeCharDevice
case restic.NodeTypeFifo:
case "fifo":
mode = os.ModeNamedPipe
case restic.NodeTypeSocket:
case "socket":
mode = os.ModeSocket
}

View file

@ -19,7 +19,7 @@ func TestFormatNode(t *testing.T) {
testPath := "/test/path"
node := restic.Node{
Name: "baz",
Type: restic.NodeTypeFile,
Type: "file",
Size: 14680064,
UID: 1000,
GID: 2000,

View file

@ -29,6 +29,7 @@ import (
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -46,7 +47,7 @@ import (
// to a missing backend storage location or config file
var ErrNoRepository = errors.New("repository does not exist")
var version = "0.17.3-dev (compiled manually)"
var version = "0.17.3"
// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
@ -308,7 +309,7 @@ func readPasswordTerminal(ctx context.Context, in *os.File, out *os.File, prompt
fd := int(out.Fd())
state, err := term.GetState(fd)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
return "", err
}
@ -317,22 +318,16 @@ func readPasswordTerminal(ctx context.Context, in *os.File, out *os.File, prompt
go func() {
defer close(done)
_, err = fmt.Fprint(out, prompt)
if err != nil {
return
}
fmt.Fprint(out, prompt)
buf, err = term.ReadPassword(int(in.Fd()))
if err != nil {
return
}
_, err = fmt.Fprintln(out)
fmt.Fprintln(out)
}()
select {
case <-ctx.Done():
err := term.Restore(fd, state)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
}
return "", ctx.Err()
case <-done:
@ -445,6 +440,26 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
return nil, err
}
report := func(msg string, err error, d time.Duration) {
if d >= 0 {
Warnf("%v returned error, retrying after %v: %v\n", msg, d, err)
} else {
Warnf("%v failed: %v\n", msg, err)
}
}
success := func(msg string, retries int) {
Warnf("%v operation successful after %d retries\n", msg, retries)
}
be = retry.New(be, 15*time.Minute, report, success)
// wrap backend if a test specified a hook
if opts.backendTestHook != nil {
be, err = opts.backendTestHook(be)
if err != nil {
return nil, err
}
}
s, err := repository.New(be, repository.Options{
Compression: opts.Compression,
PackSize: opts.PackSize * 1024 * 1024,
@ -533,7 +548,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
}
for _, item := range oldCacheDirs {
dir := filepath.Join(c.Base, item.Name())
err = os.RemoveAll(dir)
err = fs.RemoveAll(dir)
if err != nil {
Warnf("unable to remove %v: %v\n", dir, err)
}
@ -615,31 +630,12 @@ func innerOpen(ctx context.Context, s string, gopts GlobalOptions, opts options.
}
}
report := func(msg string, err error, d time.Duration) {
if d >= 0 {
Warnf("%v returned error, retrying after %v: %v\n", msg, d, err)
} else {
Warnf("%v failed: %v\n", msg, err)
}
}
success := func(msg string, retries int) {
Warnf("%v operation successful after %d retries\n", msg, retries)
}
be = retry.New(be, 15*time.Minute, report, success)
// wrap backend if a test specified a hook
if gopts.backendTestHook != nil {
be, err = gopts.backendTestHook(be)
if err != nil {
return nil, err
}
}
return be, nil
}
// Open the backend specified by a location config.
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
be, err := innerOpen(ctx, s, gopts, opts, false)
if err != nil {
return nil, err

View file

@ -1,9 +1,10 @@
package filter
package main
import (
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/spf13/pflag"
)
@ -11,21 +12,21 @@ import (
// in the restore process and returns whether it should be included.
type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool)
type IncludePatternOptions struct {
type includePatternOptions struct {
Includes []string
InsensitiveIncludes []string
IncludeFiles []string
InsensitiveIncludeFiles []string
}
func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) {
func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) {
f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
}
func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) {
func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {
var fs []IncludeByNameFunc
if len(opts.IncludeFiles) > 0 {
includePatterns, err := readPatternsFromFiles(opts.IncludeFiles)
@ -33,7 +34,7 @@ func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ..
return nil, err
}
if err := ValidatePatterns(includePatterns); err != nil {
if err := filter.ValidatePatterns(includePatterns); err != nil {
return nil, errors.Fatalf("--include-file: %s", err)
}
@ -46,7 +47,7 @@ func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ..
return nil, err
}
if err := ValidatePatterns(includePatterns); err != nil {
if err := filter.ValidatePatterns(includePatterns); err != nil {
return nil, errors.Fatalf("--iinclude-file: %s", err)
}
@ -54,45 +55,45 @@ func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ..
}
if len(opts.InsensitiveIncludes) > 0 {
if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil {
if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil {
return nil, errors.Fatalf("--iinclude: %s", err)
}
fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf))
fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes))
}
if len(opts.Includes) > 0 {
if err := ValidatePatterns(opts.Includes); err != nil {
if err := filter.ValidatePatterns(opts.Includes); err != nil {
return nil, errors.Fatalf("--include: %s", err)
}
fs = append(fs, IncludeByPattern(opts.Includes, warnf))
fs = append(fs, includeByPattern(opts.Includes))
}
return fs, nil
}
// IncludeByPattern returns a IncludeByNameFunc which includes files that match
// includeByPattern returns a IncludeByNameFunc which includes files that match
// one of the patterns.
func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
parsedPatterns := ParsePatterns(patterns)
func includeByPattern(patterns []string) IncludeByNameFunc {
parsedPatterns := filter.ParsePatterns(patterns)
return func(item string) (matched bool, childMayMatch bool) {
matched, childMayMatch, err := ListWithChild(parsedPatterns, item)
matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item)
if err != nil {
warnf("error for include pattern: %v", err)
Warnf("error for include pattern: %v", err)
}
return matched, childMayMatch
}
}
// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
// one of the patterns, ignoring the casing of the filenames.
func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
func includeByInsensitivePattern(patterns []string) IncludeByNameFunc {
for index, path := range patterns {
patterns[index] = strings.ToLower(path)
}
includeFunc := IncludeByPattern(patterns, warnf)
includeFunc := includeByPattern(patterns)
return func(item string) (matched bool, childMayMatch bool) {
return includeFunc(strings.ToLower(item))
}

View file

@ -1,4 +1,4 @@
package filter
package main
import (
"testing"
@ -21,7 +21,7 @@ func TestIncludeByPattern(t *testing.T) {
for _, tc := range tests {
t.Run(tc.filename, func(t *testing.T) {
includeFunc := IncludeByPattern(patterns, nil)
includeFunc := includeByPattern(patterns)
matched, _ := includeFunc(tc.filename)
if matched != tc.include {
t.Fatalf("wrong result for filename %v: want %v, got %v",
@ -48,7 +48,7 @@ func TestIncludeByInsensitivePattern(t *testing.T) {
for _, tc := range tests {
t.Run(tc.filename, func(t *testing.T) {
includeFunc := IncludeByInsensitivePattern(patterns, nil)
includeFunc := includeByInsensitivePattern(patterns)
matched, _ := includeFunc(tc.filename)
if matched != tc.include {
t.Fatalf("wrong result for filename %v: want %v, got %v",

View file

@ -5,7 +5,6 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/filter"
rtest "github.com/restic/restic/internal/test"
)
@ -18,14 +17,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
var err error
// Test --exclude
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
@ -48,14 +47,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
var err error
// Test --exclude-file:
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude-file
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
@ -71,28 +70,28 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) {
var err error
// Test --exclude
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --include
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iinclude
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
@ -112,22 +111,22 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
t.Fatalf("Could not write include file: %v", fileErr)
}
err := testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())

View file

@ -13,17 +13,17 @@ import (
func (e *dirEntry) equals(out io.Writer, other *dirEntry) bool {
if e.path != other.path {
_, _ = fmt.Fprintf(out, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path)
fmt.Fprintf(out, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path)
return false
}
if e.fi.Mode() != other.fi.Mode() {
_, _ = fmt.Fprintf(out, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode())
fmt.Fprintf(out, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode())
return false
}
if !sameModTime(e.fi, other.fi) {
_, _ = fmt.Fprintf(out, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime())
fmt.Fprintf(out, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime())
return false
}
@ -31,17 +31,17 @@ func (e *dirEntry) equals(out io.Writer, other *dirEntry) bool {
stat2, _ := other.fi.Sys().(*syscall.Stat_t)
if stat.Uid != stat2.Uid {
_, _ = fmt.Fprintf(out, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid)
fmt.Fprintf(out, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid)
return false
}
if stat.Gid != stat2.Gid {
_, _ = fmt.Fprintf(out, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid)
fmt.Fprintf(out, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid)
return false
}
if stat.Nlink != stat2.Nlink {
_, _ = fmt.Fprintf(out, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
fmt.Fprintf(out, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
return false
}

View file

@ -177,47 +177,3 @@ func TestFindListOnce(t *testing.T) {
// the snapshots can only be listed once, if both lists match then the there has been only a single List() call
rtest.Equals(t, thirdSnapshot, snapshotIDs)
}
type failConfigOnceBackend struct {
backend.Backend
failedOnce bool
}
func (be *failConfigOnceBackend) Load(ctx context.Context, h backend.Handle,
length int, offset int64, fn func(rd io.Reader) error) error {
if !be.failedOnce && h.Type == restic.ConfigFile {
be.failedOnce = true
return fmt.Errorf("oops")
}
return be.Backend.Load(ctx, h, length, offset, fn)
}
func (be *failConfigOnceBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if !be.failedOnce && h.Type == restic.ConfigFile {
be.failedOnce = true
return backend.FileInfo{}, fmt.Errorf("oops")
}
return be.Backend.Stat(ctx, h)
}
func TestBackendRetryConfig(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
var wrappedBackend *failConfigOnceBackend
// cause config loading to fail once
env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) {
wrappedBackend = &failConfigOnceBackend{Backend: r}
return wrappedBackend, nil
}
testSetupBackupData(t, env)
rtest.Assert(t, wrappedBackend != nil, "backend not wrapped on init")
rtest.Assert(t, wrappedBackend != nil && wrappedBackend.failedOnce, "config loading was not retried on init")
wrappedBackend = nil
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, BackupOptions{}, env.gopts)
rtest.Assert(t, wrappedBackend != nil, "backend not wrapped on backup")
rtest.Assert(t, wrappedBackend != nil && wrappedBackend.failedOnce, "config loading was not retried on init")
}

View file

@ -4,7 +4,6 @@ import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"log"
"os"
@ -120,30 +119,6 @@ func tweakGoGC() {
}
}
func printExitError(code int, message string) {
if globalOptions.JSON {
type jsonExitError struct {
MessageType string `json:"message_type"` // exit_error
Code int `json:"code"`
Message string `json:"message"`
}
jsonS := jsonExitError{
MessageType: "exit_error",
Code: code,
Message: message,
}
err := json.NewEncoder(globalOptions.stderr).Encode(jsonS)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
return
}
} else {
_, _ = fmt.Fprintf(globalOptions.stderr, "%v\n", message)
}
}
func main() {
tweakGoGC()
// install custom global logger into a buffer, if an error occurs
@ -152,10 +127,10 @@ func main() {
log.SetOutput(logBuffer)
err := feature.Flag.Apply(os.Getenv("RESTIC_FEATURES"), func(s string) {
_, _ = fmt.Fprintln(os.Stderr, s)
fmt.Fprintln(os.Stderr, s)
})
if err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
fmt.Fprintln(os.Stderr, err)
Exit(1)
}
@ -173,24 +148,23 @@ func main() {
err = nil
}
var exitMessage string
switch {
case restic.IsAlreadyLocked(err):
exitMessage = fmt.Sprintf("%v\nthe `unlock` command can be used to remove stale locks", err)
fmt.Fprintf(os.Stderr, "%v\nthe `unlock` command can be used to remove stale locks\n", err)
case err == ErrInvalidSourceData:
exitMessage = fmt.Sprintf("Warning: %v", err)
fmt.Fprintf(os.Stderr, "Warning: %v\n", err)
case errors.IsFatal(err):
exitMessage = err.Error()
fmt.Fprintf(os.Stderr, "%v\n", err)
case errors.Is(err, repository.ErrNoKeyFound):
exitMessage = fmt.Sprintf("Fatal: %v", err)
fmt.Fprintf(os.Stderr, "Fatal: %v\n", err)
case err != nil:
exitMessage = fmt.Sprintf("%+v", err)
fmt.Fprintf(os.Stderr, "%+v\n", err)
if logBuffer.Len() > 0 {
exitMessage += "also, the following messages were logged by a library:\n"
fmt.Fprintf(os.Stderr, "also, the following messages were logged by a library:\n")
sc := bufio.NewScanner(logBuffer)
for sc.Scan() {
exitMessage += fmt.Sprintln(sc.Text())
fmt.Fprintln(os.Stderr, sc.Text())
}
}
}
@ -212,9 +186,5 @@ func main() {
default:
exitCode = 1
}
if exitCode != 0 {
printExitError(exitCode, exitMessage)
}
Exit(exitCode)
}

View file

@ -29,7 +29,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration {
return interval
}
// newGenericProgressMax returns a progress.Counter that prints to stdout or terminal if provided.
// newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided.
func newGenericProgressMax(show bool, max uint64, description string, print func(status string, final bool)) *progress.Counter {
if !show {
return nil

View file

@ -284,7 +284,8 @@ From Source
***********
restic is written in the Go programming language and you need at least
Go version 1.21. Building restic may also work with older versions of Go,
Go version 1.19. Building for Solaris requires at least Go version 1.20.
Building restic may also work with older versions of Go,
but that's not supported. See the `Getting
started <https://go.dev/doc/install>`__ guide of the Go project for
instructions how to install Go.

View file

@ -314,17 +314,9 @@ this command.
S3-compatible Storage
*********************
For an S3-compatible storage service that is not Amazon, you can specify the URL to the server
For an S3-compatible server that is not Amazon, you can specify the URL to the server
like this: ``s3:https://server:port/bucket_name``.
You must also set credentials for authentication to the service.
.. code-block:: console
$ export AWS_ACCESS_KEY_ID=<YOUR-ACCESS-KEY-ID>
$ export AWS_SECRET_ACCESS_KEY=<YOUR-SECRET-ACCESS-KEY>
$ restic -r s3:https://server:port/bucket_name init
If needed, you can manually specify the region to use by either setting the
environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option
parameter like ``-o s3.region="us-east-1"``. If the region is not specified,
@ -568,10 +560,6 @@ The number of concurrent connections to the Azure Blob Storage service can be se
``-o azure.connections=10`` switch. By default, at most five parallel connections are
established.
The access tier of the blobs uploaded to the Azure Blob Storage service can be set with the
``-o azure.access-tier=Cool`` switch. The allowed values are ``Hot``, ``Cool`` or ``Cold``.
If unspecified, the default is inferred from the default configured on the storage account.
Google Cloud Storage
********************

View file

@ -214,8 +214,7 @@ The ``forget`` command accepts the following policy options:
run) and these snapshots will hence not be removed.
.. note:: If there are not enough snapshots to keep one for each duration related
``--keep-{within-,}*`` option, the oldest snapshot is kept additionally and
marked as ``oldest`` in the output (e.g. ``oldest hourly snapshot``).
``--keep-{within-,}*`` option, the oldest snapshot is kept additionally.
.. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only.

View file

@ -87,33 +87,12 @@ JSON output of most restic commands are documented here.
list of allowed values is documented may be extended at any time.
Exit errors
-----------
Fatal errors will result in a final JSON message on ``stderr`` before the process exits.
It will hold the error message and the exit code.
.. note::
Some errors cannot be caught and reported this way,
such as Go runtime errors or command line parsing errors.
+----------------------+-------------------------------------------+
| ``message_type`` | Always "exit_error" |
+----------------------+-------------------------------------------+
| ``code`` | Exit code (see above chart) |
+----------------------+-------------------------------------------+
| ``message`` | Error message |
+----------------------+-------------------------------------------+
Output formats
--------------
Commands print their main JSON output on ``stdout``.
The generated JSON output uses one of the following two formats.
.. note::
Not all messages and errors have been converted to JSON yet.
Feel free to submit a pull request!
Currently only the output on ``stdout`` is JSON formatted. Errors printed on ``stderr``
are still printed as plain text messages. The generated JSON output uses one of the
following two formats.
Single JSON document
^^^^^^^^^^^^^^^^^^^^
@ -161,8 +140,6 @@ Status
Error
^^^^^
These errors are printed on ``stderr``.
+----------------------+-------------------------------------------+
| ``message_type`` | Always "error" |
+----------------------+-------------------------------------------+
@ -226,10 +203,6 @@ Summary is the last output line in a successful backup.
+---------------------------+---------------------------------------------------------+
| ``total_bytes_processed`` | Total number of bytes processed |
+---------------------------+---------------------------------------------------------+
| ``backup_start`` | Time at which the backup was started |
+---------------------------+---------------------------------------------------------+
| ``backup_end`` | Time at which the backup was completed |
+---------------------------+---------------------------------------------------------+
| ``total_duration`` | Total time it took for the operation to complete |
+---------------------------+---------------------------------------------------------+
| ``snapshot_id`` | ID of the new snapshot. Field is omitted if snapshot |
@ -563,8 +536,6 @@ Status
+----------------------+------------------------------------------------------------+
|``files_skipped`` | Files skipped due to overwrite setting |
+----------------------+------------------------------------------------------------+
|``files_deleted`` | Files deleted |
+----------------------+------------------------------------------------------------+
|``total_bytes`` | Total number of bytes in restore set |
+----------------------+------------------------------------------------------------+
|``bytes_restored`` | Number of bytes restored |
@ -575,8 +546,6 @@ Status
Error
^^^^^
These errors are printed on ``stderr``.
+----------------------+-------------------------------------------+
| ``message_type`` | Always "error" |
+----------------------+-------------------------------------------+
@ -617,8 +586,6 @@ Summary
+----------------------+------------------------------------------------------------+
|``files_skipped`` | Files skipped due to overwrite setting |
+----------------------+------------------------------------------------------------+
|``files_deleted`` | Files deleted |
+----------------------+------------------------------------------------------------+
|``total_bytes`` | Total number of bytes in restore set |
+----------------------+------------------------------------------------------------+
|``bytes_restored`` | Number of bytes restored |
@ -728,14 +695,12 @@ version
The version command returns a single JSON object.
+------------------+--------------------+
| ``message_type`` | Always "version" |
+------------------+--------------------+
| ``version`` | restic version |
+------------------+--------------------+
| ``go_version`` | Go compile version |
+------------------+--------------------+
| ``go_os`` | Go OS |
+------------------+--------------------+
| ``go_arch`` | Go architecture |
+------------------+--------------------+
+----------------+--------------------+
| ``version`` | restic version |
+----------------+--------------------+
| ``go_version`` | Go compile version |
+----------------+--------------------+
| ``go_os`` | Go OS |
+----------------+--------------------+
| ``go_arch`` | Go architecture |
+----------------+--------------------+

View file

@ -119,11 +119,16 @@ A local repository can be initialized with the ``restic init`` command, e.g.:
$ restic -r /tmp/restic-repo init
The local and sftp backends will auto-detect and accept all layouts described
in the following sections, so that remote repositories mounted locally e.g. via
fuse can be accessed. The layout auto-detection can be overridden by specifying
the option ``-o local.layout=default``, valid values are ``default`` and
``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the
s3 backend ``s3.layout``.
S3 Legacy Layout (deprecated)
-----------------------------
Restic 0.17 is the last version that supports the legacy layout.
Unfortunately during development the Amazon S3 backend uses slightly different
paths (directory names use singular instead of plural for ``key``,
``lock``, and ``snapshot`` files), and the pack files are stored directly below
@ -147,6 +152,8 @@ the ``data`` directory. The S3 Legacy repository layout looks like this:
/snapshot
└── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
Restic 0.17 is the last version that supports the legacy layout.
Pack Format
===========

90
go.mod
View file

@ -2,11 +2,10 @@ module github.com/restic/restic
require (
cloud.google.com/go/storage v1.43.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
github.com/Backblaze/blazer v0.7.1
github.com/Microsoft/go-winio v0.6.2
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
github.com/Backblaze/blazer v0.6.1
github.com/anacrolix/fuse v0.3.1
github.com/cenkalti/backoff/v4 v4.3.0
github.com/cespare/xxhash/v2 v2.3.0
@ -15,71 +14,76 @@ require (
github.com/google/go-cmp v0.6.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/klauspost/compress v1.17.9
github.com/minio/minio-go/v7 v7.0.77
github.com/ncw/swift/v2 v2.0.3
github.com/minio/minio-go/v7 v7.0.66
github.com/minio/sha256-simd v1.0.1
github.com/ncw/swift/v2 v2.0.2
github.com/peterbourgon/unixtransport v0.0.4
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.7.0
github.com/pkg/sftp v1.13.7
github.com/pkg/sftp v1.13.6
github.com/pkg/xattr v0.4.10
github.com/restic/chunker v0.4.0
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
go.uber.org/automaxprocs v1.6.0
golang.org/x/crypto v0.28.0
golang.org/x/net v0.30.0
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.9.0
golang.org/x/sys v0.27.0
golang.org/x/term v0.25.0
golang.org/x/text v0.20.0
golang.org/x/time v0.7.0
google.golang.org/api v0.204.0
go.uber.org/automaxprocs v1.5.3
golang.org/x/crypto v0.24.0
golang.org/x/net v0.26.0
golang.org/x/oauth2 v0.21.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.22.0
golang.org/x/term v0.22.0
golang.org/x/text v0.16.0
golang.org/x/time v0.5.0
google.golang.org/api v0.187.0
)
require (
cloud.google.com/go v0.116.0 // indirect
cloud.google.com/go/auth v0.10.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
cloud.google.com/go/iam v1.2.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.6.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.8 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/protobuf v1.35.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect
google.golang.org/grpc v1.64.1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go 1.21
go 1.19

217
go.sum
View file

@ -1,40 +1,32 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=
cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=
cloud.google.com/go/auth v0.10.0 h1:tWlkvFAh+wwTOzXIjrwM64karR1iTBZ/GRr0S/DULYo=
cloud.google.com/go/auth v0.10.0/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
cloud.google.com/go/auth/oauth2adapt v0.2.5 h1:2p29+dePqsCHPP1bqDJcKj4qxRyYCcbzKpFyKGt3MTk=
cloud.google.com/go/auth/oauth2adapt v0.2.5/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU=
cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g=
cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc=
cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0=
cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14=
cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU=
cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38=
cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4=
cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0=
cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE=
cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU=
cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/Backblaze/blazer v0.7.1 h1:J43PbFj6hXLg1jvCNr+rQoAsxzKK0IP7ftl1ReCwpcQ=
github.com/Backblaze/blazer v0.7.1/go.mod h1:MhntL1nMpIuoqrPP6TnZu/xTydMgOAe/Xm6KongbjKs=
github.com/Backblaze/blazer v0.6.1 h1:xC9HyC7OcxRzzmtfRiikIEvq4HZYWjU6caFwX2EXw1s=
github.com/Backblaze/blazer v0.6.1/go.mod h1:7/jrGx4O6OKOto6av+hLwelPR8rwZ+PLxQ5ZOiYAjwY=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tcJDk=
github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0=
github.com/anacrolix/fuse v0.3.1 h1:oT8s3B5HFkBdLe/WKJO5MNo9iIyEtc+BhvTZYp4jhDM=
@ -60,8 +52,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dvyukov/go-fuzz v0.0.0-20220726122315-1d375ef9f9f6/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
@ -77,17 +67,13 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -115,32 +101,32 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ=
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA=
github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -156,10 +142,17 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+
github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw=
github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=
github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg=
github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
@ -172,17 +165,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw=
github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw=
github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
@ -190,11 +180,12 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@ -209,34 +200,32 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo=
go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok=
go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4=
golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
@ -247,7 +236,6 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -258,22 +246,21 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -285,35 +272,29 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -323,31 +304,30 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.204.0 h1:3PjmQQEDkR/ENVZZwIYB4W/KzYtN8OrqnNcHWpeR8E4=
google.golang.org/api v0.204.0/go.mod h1:69y8QSoKIbL9F94bWgWAq6wGqGwyjBgi2y8rAK8zLag=
google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo=
google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 h1:Q3nlH8iSQSRUwOskjbcSMcF2jiYMNiQYZ0c2KEJLKKU=
google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38/go.mod h1:xBI+tzfqGGN2JBeSebfKXFSdBpWVQ7sLW40PTupVRm4=
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U=
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls=
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M=
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc=
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA=
google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -357,14 +337,15 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View file

@ -243,15 +243,14 @@ func buildTargets(sourceDir, outputDir string, targets map[string][]string) {
}
var defaultBuildTargets = map[string][]string{
"aix": {"ppc64"},
"darwin": {"amd64", "arm64"},
"dragonfly": {"amd64"},
"freebsd": {"386", "amd64", "arm"},
"linux": {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x"},
"netbsd": {"386", "amd64"},
"openbsd": {"386", "amd64"},
"windows": {"386", "amd64"},
"solaris": {"amd64"},
"aix": {"ppc64"},
"darwin": {"amd64", "arm64"},
"freebsd": {"386", "amd64", "arm"},
"linux": {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x"},
"netbsd": {"386", "amd64"},
"openbsd": {"386", "amd64"},
"windows": {"386", "amd64"},
"solaris": {"amd64"},
}
func downloadModules(sourceDir string) {

View file

@ -25,7 +25,7 @@ type SelectByNameFunc func(item string) bool
// SelectFunc returns true for all items that should be included (files and
// dirs). If false is returned, files are ignored and dirs are not even walked.
type SelectFunc func(item string, fi *fs.ExtendedFileInfo, fs fs.FS) bool
type SelectFunc func(item string, fi os.FileInfo) bool
// ErrorFunc is called when an error during archiving occurs. When nil is
// returned, the archiver continues, otherwise it aborts and passes the error
@ -49,8 +49,6 @@ type ChangeStats struct {
}
type Summary struct {
BackupStart time.Time
BackupEnd time.Time
Files, Dirs ChangeStats
ProcessedBytes uint64
ItemStats
@ -66,11 +64,6 @@ func (s *ItemStats) Add(other ItemStats) {
s.TreeSizeInRepo += other.TreeSizeInRepo
}
// ToNoder returns a restic.Node for a File.
type ToNoder interface {
ToNode(ignoreXattrListError bool) (*restic.Node, error)
}
type archiverRepo interface {
restic.Loader
restic.BlobSaver
@ -82,14 +75,6 @@ type archiverRepo interface {
}
// Archiver saves a directory structure to the repo.
//
// An Archiver has a number of worker goroutines handling saving the different
// data structures to the repository, the details are implemented by the
// fileSaver, blobSaver, and treeSaver types.
//
// The main goroutine (the one calling Snapshot()) traverses the directory tree
// and delegates all work to these worker pools. They return a futureNode which
// can be resolved later, by calling Wait() on it.
type Archiver struct {
Repo archiverRepo
SelectByName SelectByNameFunc
@ -97,9 +82,9 @@ type Archiver struct {
FS fs.FS
Options Options
blobSaver *blobSaver
fileSaver *fileSaver
treeSaver *treeSaver
blobSaver *BlobSaver
fileSaver *FileSaver
treeSaver *TreeSaver
mu sync.Mutex
summary *Summary
@ -175,7 +160,7 @@ func (o Options) ApplyDefaults() Options {
if o.SaveTreeConcurrency == 0 {
// can either wait for a file, wait for a tree, serialize a tree or wait for saveblob
// the last two are cpu-bound and thus mutually exclusive.
// Also allow waiting for FileReadConcurrency files, this is the maximum of files
// Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles
// which currently can be in progress. The main backup loop blocks when trying to queue
// more files to read.
o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.ReadConcurrency
@ -185,12 +170,12 @@ func (o Options) ApplyDefaults() Options {
}
// New initializes a new archiver.
func New(repo archiverRepo, filesystem fs.FS, opts Options) *Archiver {
func New(repo archiverRepo, fs fs.FS, opts Options) *Archiver {
arch := &Archiver{
Repo: repo,
SelectByName: func(_ string) bool { return true },
Select: func(_ string, _ *fs.ExtendedFileInfo, _ fs.FS) bool { return true },
FS: filesystem,
Select: func(_ string, _ os.FileInfo) bool { return true },
FS: fs,
Options: opts.ApplyDefaults(),
CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {},
@ -239,7 +224,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
}
switch current.Type {
case restic.NodeTypeDir:
case "dir":
switch {
case previous == nil:
arch.summary.Dirs.New++
@ -249,7 +234,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
arch.summary.Dirs.Changed++
}
case restic.NodeTypeFile:
case "file":
switch {
case previous == nil:
arch.summary.Files.New++
@ -262,13 +247,14 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
}
// nodeFromFileInfo returns the restic node from an os.FileInfo.
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) {
node, err := meta.ToNode(ignoreXattrListError)
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
mappedFilename := arch.FS.MapFilename(filename)
node, err := restic.NodeFromFileInfo(mappedFilename, fi, ignoreXattrListError)
if !arch.WithAtime {
node.AccessTime = node.ModTime
}
if feature.Flag.Enabled(feature.DeviceIDForHardlinks) {
if node.Links == 1 || node.Type == restic.NodeTypeDir {
if node.Links == 1 || node.Type == "dir" {
// the DeviceID is only necessary for hardlinked files
// when using subvolumes or snapshots their deviceIDs tend to change which causes
// restic to upload new tree blobs
@ -278,7 +264,7 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig
// overwrite name to match that within the snapshot
node.Name = path.Base(snPath)
// do not filter error for nodes of irregular or invalid type
if node.Type != restic.NodeTypeIrregular && node.Type != restic.NodeTypeInvalid && err != nil {
if node.Type != "irregular" && node.Type != "" && err != nil {
err = fmt.Errorf("incomplete metadata for %v: %w", filename, err)
return node, arch.error(filename, err)
}
@ -288,7 +274,7 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig
// loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.
// If there is no node to load, then nil is returned without an error.
func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*restic.Tree, error) {
if node == nil || node.Type != restic.NodeTypeDir || node.Subtree == nil {
if node == nil || node.Type != "dir" || node.Subtree == nil {
return nil, nil
}
@ -313,21 +299,27 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
// saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
debug.Log("%v %v", snPath, dir)
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta)
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false)
if err != nil {
return futureNode{}, err
return FutureNode{}, err
}
nodes := make([]futureNode, 0, len(names))
names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
if err != nil {
return FutureNode{}, err
}
sort.Strings(names)
nodes := make([]FutureNode, 0, len(names))
for _, name := range names {
// test if context has been cancelled
if ctx.Err() != nil {
debug.Log("context has been cancelled, aborting")
return futureNode{}, ctx.Err()
return FutureNode{}, ctx.Err()
}
pathname := arch.FS.Join(dir, name)
@ -343,7 +335,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
continue
}
return futureNode{}, err
return FutureNode{}, err
}
if excluded {
@ -358,34 +350,11 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
return fn, nil
}
func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *restic.Node, names []string, err error) {
err = meta.MakeReadable()
if err != nil {
return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
}
node, err = arch.nodeFromFileInfo(snPath, dir, meta, false)
if err != nil {
return nil, nil, err
}
if node.Type != restic.NodeTypeDir {
return nil, nil, fmt.Errorf("directory %q changed type, refusing to archive", snPath)
}
names, err = meta.Readdirnames(-1)
if err != nil {
return nil, nil, fmt.Errorf("readdirnames %v failed: %w", dir, err)
}
sort.Strings(names)
return node, names, nil
}
// futureNode holds a reference to a channel that returns a FutureNodeResult
// FutureNode holds a reference to a channel that returns a FutureNodeResult
// or a reference to an already existing result. If the result is available
// immediately, then storing a reference directly requires less memory than
// using the indirection via a channel.
type futureNode struct {
type FutureNode struct {
ch <-chan futureNodeResult
res *futureNodeResult
}
@ -398,18 +367,18 @@ type futureNodeResult struct {
err error
}
func newFutureNode() (futureNode, chan<- futureNodeResult) {
func newFutureNode() (FutureNode, chan<- futureNodeResult) {
ch := make(chan futureNodeResult, 1)
return futureNode{ch: ch}, ch
return FutureNode{ch: ch}, ch
}
func newFutureNodeWithResult(res futureNodeResult) futureNode {
return futureNode{
func newFutureNodeWithResult(res futureNodeResult) FutureNode {
return FutureNode{
res: &res,
}
}
func (fn *futureNode) take(ctx context.Context) futureNodeResult {
func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
if fn.res != nil {
res := fn.res
// free result
@ -448,64 +417,38 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
// Errors and completion needs to be handled by the caller.
//
// snPath is the path within the current snapshot.
func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn futureNode, excluded bool, err error) {
func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
start := time.Now()
debug.Log("%v target %q, previous %v", snPath, target, previous)
abstarget, err := arch.FS.Abs(target)
if err != nil {
return futureNode{}, false, err
return FutureNode{}, false, err
}
filterError := func(err error) (futureNode, bool, error) {
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, errors.WithStack(err)
}
return futureNode{}, true, nil
}
filterNotExist := func(err error) error {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
// exclude files by path before running Lstat to reduce number of lstat calls
if !arch.SelectByName(abstarget) {
debug.Log("%v is excluded by path", target)
return futureNode{}, true, nil
return FutureNode{}, true, nil
}
meta, err := arch.FS.OpenFile(target, fs.O_NOFOLLOW, true)
if err != nil {
debug.Log("open metadata for %v returned error: %v", target, err)
// ignore if file disappeared since it was returned by readdir
return filterError(filterNotExist(err))
}
closeFile := true
defer func() {
if closeFile {
cerr := meta.Close()
if err == nil {
err = cerr
}
}
}()
// get file info and run remaining select functions that require file information
fi, err := meta.Stat()
fi, err := arch.FS.Lstat(target)
if err != nil {
debug.Log("lstat() for %v returned error: %v", target, err)
// ignore if file disappeared since it was returned by readdir
return filterError(filterNotExist(err))
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, errors.WithStack(err)
}
return FutureNode{}, true, nil
}
if !arch.Select(abstarget, fi, arch.FS) {
if !arch.Select(abstarget, fi) {
debug.Log("%v is excluded", target)
return futureNode{}, true, nil
return FutureNode{}, true, nil
}
switch {
case fi.Mode.IsRegular():
case fs.IsRegularFile(fi):
debug.Log(" %v regular file", target)
// check if the file has not changed before performing a fopen operation (more expensive, specially
@ -515,9 +458,9 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(previous.Size)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
if err != nil {
return futureNode{}, false, err
return FutureNode{}, false, err
}
// copy list of blobs
@ -536,34 +479,46 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, err
return FutureNode{}, false, err
}
}
// reopen file and do an fstat() on the open file to check it is still
// a file (and has not been exchanged for e.g. a symlink)
err := meta.MakeReadable()
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
if err != nil {
debug.Log("MakeReadable() for %v returned error: %v", target, err)
return filterError(err)
debug.Log("Openfile() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, errors.WithStack(err)
}
return FutureNode{}, true, nil
}
fi, err := meta.Stat()
fi, err = file.Stat()
if err != nil {
debug.Log("stat() on opened file %v returned error: %v", target, err)
return filterError(err)
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, errors.WithStack(err)
}
return FutureNode{}, true, nil
}
// make sure it's still a file
if !fi.Mode.IsRegular() {
err = errors.Errorf("file %q changed type, refusing to archive", target)
return filterError(err)
if !fs.IsRegularFile(fi) {
err = errors.Errorf("file %v changed type, refusing to archive", fi.Name())
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, err
}
return FutureNode{}, true, nil
}
closeFile = false
// Save will close the file, we don't need to do that
fn = arch.fileSaver.Save(ctx, snPath, target, meta, func() {
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
arch.StartFile(snPath)
}, func() {
arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
@ -571,7 +526,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
arch.trackItem(snPath, previous, node, stats, time.Since(start))
})
case fi.Mode.IsDir():
case fi.IsDir():
debug.Log(" %v dir", target)
snItem := snPath + "/"
@ -580,28 +535,28 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
err = arch.error(abstarget, err)
}
if err != nil {
return futureNode{}, false, err
return FutureNode{}, false, err
}
fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree,
fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
func(node *restic.Node, stats ItemStats) {
arch.trackItem(snItem, previous, node, stats, time.Since(start))
})
if err != nil {
debug.Log("SaveDir for %v returned error: %v", snPath, err)
return futureNode{}, false, err
return FutureNode{}, false, err
}
case fi.Mode&os.ModeSocket > 0:
case fi.Mode()&os.ModeSocket > 0:
debug.Log(" %v is a socket, ignoring", target)
return futureNode{}, true, nil
return FutureNode{}, true, nil
default:
debug.Log(" %v other", target)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
if err != nil {
return futureNode{}, false, err
return FutureNode{}, false, err
}
fn = newFutureNodeWithResult(futureNodeResult{
snPath: snPath,
@ -618,26 +573,27 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
// fileChanged tries to detect whether a file's content has changed compared
// to the contents of node, which describes the same path in the parent backup.
// It should only be run for regular files.
func fileChanged(fi *fs.ExtendedFileInfo, node *restic.Node, ignoreFlags uint) bool {
func fileChanged(fi os.FileInfo, node *restic.Node, ignoreFlags uint) bool {
switch {
case node == nil:
return true
case node.Type != restic.NodeTypeFile:
case node.Type != "file":
// We're only called for regular files, so this is a type change.
return true
case uint64(fi.Size) != node.Size:
case uint64(fi.Size()) != node.Size:
return true
case !fi.ModTime.Equal(node.ModTime):
case !fi.ModTime().Equal(node.ModTime):
return true
}
checkCtime := ignoreFlags&ChangeIgnoreCtime == 0
checkInode := ignoreFlags&ChangeIgnoreInode == 0
extFI := fs.ExtendedStat(fi)
switch {
case checkCtime && !fi.ChangeTime.Equal(node.ChangeTime):
case checkCtime && !extFI.ChangeTime.Equal(node.ChangeTime):
return true
case checkInode && node.Inode != fi.Inode:
case checkInode && node.Inode != extFI.Inode:
return true
}
@ -649,20 +605,43 @@ func join(elem ...string) string {
return path.Join(elem...)
}
// statDir returns the file info for the directory. Symbolic links are
// resolved. If the target directory is not a directory, an error is returned.
func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
fi, err := arch.FS.Stat(dir)
if err != nil {
return nil, errors.WithStack(err)
}
tpe := fi.Mode() & (os.ModeType | os.ModeCharDevice)
if tpe != os.ModeDir {
return fi, errors.Errorf("path is not a directory: %v", dir)
}
return fi, nil
}
// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot.
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *restic.Tree, complete fileCompleteFunc) (futureNode, int, error) {
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
var node *restic.Node
if snPath != "/" {
if atree.FileInfoPath == "" {
return futureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath)
return FutureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath)
}
var err error
node, err = arch.dirPathToNode(snPath, atree.FileInfoPath)
fi, err := arch.statDir(atree.FileInfoPath)
if err != nil {
return futureNode{}, 0, err
return FutureNode{}, 0, err
}
debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath)
// in some cases reading xattrs for directories above the backup source is not allowed
// thus ignore errors for such folders.
node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi, true)
if err != nil {
return FutureNode{}, 0, err
}
} else {
// fake root node
@ -671,7 +650,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous)
nodeNames := atree.NodeNames()
nodes := make([]futureNode, 0, len(nodeNames))
nodes := make([]FutureNode, 0, len(nodeNames))
// iterate over the nodes of atree in lexicographic (=deterministic) order
for _, name := range nodeNames {
@ -679,7 +658,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
// test if context has been cancelled
if ctx.Err() != nil {
return futureNode{}, 0, ctx.Err()
return FutureNode{}, 0, ctx.Err()
}
// this is a leaf node
@ -692,11 +671,11 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
// ignore error
continue
}
return futureNode{}, 0, err
return FutureNode{}, 0, err
}
if err != nil {
return futureNode{}, 0, err
return FutureNode{}, 0, err
}
if !excluded {
@ -714,7 +693,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
err = arch.error(join(snPath, name), err)
}
if err != nil {
return futureNode{}, 0, err
return FutureNode{}, 0, err
}
// not a leaf node, archive subtree
@ -722,7 +701,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
arch.trackItem(snItem, oldNode, n, is, time.Since(start))
})
if err != nil {
return futureNode{}, 0, err
return FutureNode{}, 0, err
}
nodes = append(nodes, fn)
}
@ -731,31 +710,6 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
return fn, len(nodes), nil
}
func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, err error) {
meta, err := arch.FS.OpenFile(target, 0, true)
if err != nil {
return nil, err
}
defer func() {
cerr := meta.Close()
if err == nil {
err = cerr
}
}()
debug.Log("%v, reading dir node data from %v", snPath, target)
// in some cases reading xattrs for directories above the backup source is not allowed
// thus ignore errors for such folders.
node, err = arch.nodeFromFileInfo(snPath, target, meta, true)
if err != nil {
return nil, err
}
if node.Type != restic.NodeTypeDir {
return nil, errors.Errorf("path is not a directory: %v", target)
}
return node, err
}
// resolveRelativeTargets replaces targets that only contain relative
// directories ("." or "../../") with the contents of the directory. Each
// element of target is processed with fs.Clean().
@ -827,16 +781,16 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *restic.Snapshot) *
// runWorkers starts the worker pools, which are stopped when the context is cancelled.
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) {
arch.blobSaver = newBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency)
arch.blobSaver = NewBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency)
arch.fileSaver = newFileSaver(ctx, wg,
arch.fileSaver = NewFileSaver(ctx, wg,
arch.blobSaver.Save,
arch.Repo.Config().ChunkerPolynomial,
arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency)
arch.fileSaver.CompleteBlob = arch.CompleteBlob
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo
arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error)
arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error)
}
func (arch *Archiver) stopWorkers() {
@ -850,16 +804,14 @@ func (arch *Archiver) stopWorkers() {
// Snapshot saves several targets and returns a snapshot.
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) {
arch.summary = &Summary{
BackupStart: opts.BackupStart,
}
arch.summary = &Summary{}
cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
if err != nil {
return nil, restic.ID{}, nil, err
}
atree, err := newTree(arch.FS, cleanTargets)
atree, err := NewTree(arch.FS, cleanTargets)
if err != nil {
return nil, restic.ID{}, nil, err
}
@ -935,10 +887,9 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
sn.Parent = opts.ParentSnapshot.ID()
}
sn.Tree = &rootTreeID
arch.summary.BackupEnd = time.Now()
sn.Summary = &restic.SnapshotSummary{
BackupStart: arch.summary.BackupStart,
BackupEnd: arch.summary.BackupEnd,
BackupStart: opts.BackupStart,
BackupEnd: time.Now(),
FilesNew: arch.summary.Files.New,
FilesChanged: arch.summary.Files.Changed,

View file

@ -76,12 +76,17 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS
startCallback = true
}
file, err := arch.FS.OpenFile(filename, fs.O_NOFOLLOW, false)
file, err := arch.FS.OpenFile(filename, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
if err != nil {
t.Fatal(err)
}
res := arch.fileSaver.Save(ctx, "/", filename, file, start, completeReading, complete)
fi, err := file.Stat()
if err != nil {
t.Fatal(err)
}
res := arch.fileSaver.Save(ctx, "/", filename, file, fi, start, completeReading, complete)
fnr := res.take(ctx)
if fnr.err != nil {
@ -516,13 +521,13 @@ func chmodTwice(t testing.TB, name string) {
rtest.OK(t, err)
}
func lstat(t testing.TB, name string) *fs.ExtendedFileInfo {
func lstat(t testing.TB, name string) os.FileInfo {
fi, err := os.Lstat(name)
if err != nil {
t.Fatal(err)
}
return fs.ExtendedStat(fi)
return fi
}
func setTimestamp(t testing.TB, filename string, atime, mtime time.Time) {
@ -551,12 +556,11 @@ func rename(t testing.TB, oldname, newname string) {
}
}
func nodeFromFile(t testing.TB, localFs fs.FS, filename string) *restic.Node {
meta, err := localFs.OpenFile(filename, fs.O_NOFOLLOW, true)
rtest.OK(t, err)
node, err := meta.ToNode(false)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node {
node, err := restic.NodeFromFileInfo(filename, fi, false)
if err != nil {
t.Fatal(err)
}
return node
}
@ -660,7 +664,7 @@ func TestFileChanged(t *testing.T) {
rename(t, filename, tempname)
save(t, filename, defaultContent)
remove(t, tempname)
setTimestamp(t, filename, fi.ModTime, fi.ModTime)
setTimestamp(t, filename, fi.ModTime(), fi.ModTime())
},
ChangeIgnore: ChangeIgnoreCtime | ChangeIgnoreInode,
SameFile: true,
@ -682,10 +686,8 @@ func TestFileChanged(t *testing.T) {
}
save(t, filename, content)
fs := &fs.Local{}
fiBefore, err := fs.Lstat(filename)
rtest.OK(t, err)
node := nodeFromFile(t, fs, filename)
fiBefore := lstat(t, filename)
node := nodeFromFI(t, filename, fiBefore)
if fileChanged(fiBefore, node, 0) {
t.Fatalf("unchanged file detected as changed")
@ -726,8 +728,8 @@ func TestFilChangedSpecialCases(t *testing.T) {
t.Run("type-change", func(t *testing.T) {
fi := lstat(t, filename)
node := nodeFromFile(t, &fs.Local{}, filename)
node.Type = restic.NodeTypeSymlink
node := nodeFromFI(t, filename, fi)
node.Type = "symlink"
if !fileChanged(fi, node, 0) {
t.Fatal("node with changed type detected as unchanged")
}
@ -831,8 +833,7 @@ func TestArchiverSaveDir(t *testing.T) {
wg, ctx := errgroup.WithContext(context.Background())
repo.StartPackUploader(ctx, wg)
testFS := fs.Track{FS: fs.Local{}}
arch := New(repo, testFS, Options{})
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
@ -844,11 +845,15 @@ func TestArchiverSaveDir(t *testing.T) {
back := rtest.Chdir(t, chdir)
defer back()
meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true)
rtest.OK(t, err)
ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, nil)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fi, err := fs.Lstat(test.target)
if err != nil {
t.Fatal(err)
}
ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
if err != nil {
t.Fatal(err)
}
fnr := ft.take(ctx)
node, stats := fnr.node, fnr.stats
@ -910,16 +915,19 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
wg, ctx := errgroup.WithContext(context.TODO())
repo.StartPackUploader(ctx, wg)
testFS := fs.Track{FS: fs.Local{}}
arch := New(repo, testFS, Options{})
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true)
rtest.OK(t, err)
ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, nil)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fi, err := fs.Lstat(tempdir)
if err != nil {
t.Fatal(err)
}
ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
if err != nil {
t.Fatal(err)
}
fnr := ft.take(ctx)
node, stats := fnr.node, fnr.stats
@ -1113,7 +1121,7 @@ func TestArchiverSaveTree(t *testing.T) {
test.prepare(t)
}
atree, err := newTree(testFS, test.targets)
atree, err := NewTree(testFS, test.targets)
if err != nil {
t.Fatal(err)
}
@ -1521,7 +1529,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
},
"other": TestFile{Content: "another file"},
},
selFn: func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
selFn: func(item string, fi os.FileInfo) bool {
return true
},
},
@ -1538,7 +1546,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
},
"other": TestFile{Content: "another file"},
},
selFn: func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
selFn: func(item string, fi os.FileInfo) bool {
return false
},
err: "snapshot is empty",
@ -1565,7 +1573,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
},
"other": TestFile{Content: "another file"},
},
selFn: func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
selFn: func(item string, fi os.FileInfo) bool {
return filepath.Ext(item) != ".txt"
},
},
@ -1589,8 +1597,8 @@ func TestArchiverSnapshotSelect(t *testing.T) {
},
"other": TestFile{Content: "another file"},
},
selFn: func(item string, fi *fs.ExtendedFileInfo, fs fs.FS) bool {
return fs.Base(item) != "subdir"
selFn: func(item string, fi os.FileInfo) bool {
return filepath.Base(item) != "subdir"
},
},
{
@ -1598,8 +1606,8 @@ func TestArchiverSnapshotSelect(t *testing.T) {
src: TestDir{
"foo": TestFile{Content: "foo"},
},
selFn: func(item string, fi *fs.ExtendedFileInfo, fs fs.FS) bool {
return fs.IsAbs(item)
selFn: func(item string, fi os.FileInfo) bool {
return filepath.IsAbs(item)
},
},
}
@ -1656,8 +1664,17 @@ type MockFS struct {
bytesRead map[string]int // tracks bytes read from all opened files
}
func (m *MockFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
f, err := m.FS.OpenFile(name, flag, metadataOnly)
func (m *MockFS) Open(name string) (fs.File, error) {
f, err := m.FS.Open(name)
if err != nil {
return f, err
}
return MockFile{File: f, fs: m, filename: name}, nil
}
func (m *MockFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
f, err := m.FS.OpenFile(name, flag, perm)
if err != nil {
return f, err
}
@ -1683,17 +1700,14 @@ func (f MockFile) Read(p []byte) (int, error) {
}
func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
t.Helper()
rtest.Equals(t, stat.BackupStart, sn.Summary.BackupStart, "BackupStart")
// BackupEnd is set to time.Now() and can't be compared to a fixed value
rtest.Equals(t, stat.Files.New, sn.Summary.FilesNew, "FilesNew")
rtest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged, "FilesChanged")
rtest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified, "FilesUnmodified")
rtest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew, "DirsNew")
rtest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged, "DirsChanged")
rtest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified, "DirsUnmodified")
rtest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed, "TotalBytesProcessed")
rtest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed, "TotalFilesProcessed")
rtest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
rtest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
rtest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
rtest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
rtest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
rtest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
rtest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
rtest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
@ -2047,12 +2061,20 @@ type TrackFS struct {
m sync.Mutex
}
func (m *TrackFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
func (m *TrackFS) Open(name string) (fs.File, error) {
m.m.Lock()
m.opened[name]++
m.m.Unlock()
return m.FS.OpenFile(name, flag, metadataOnly)
return m.FS.Open(name)
}
func (m *TrackFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
m.m.Lock()
m.opened[name]++
m.m.Unlock()
return m.FS.OpenFile(name, flag, perm)
}
type failSaveRepo struct {
@ -2201,51 +2223,48 @@ func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *restic.Snapshot
return snapshot, node
}
type overrideFS struct {
// StatFS allows overwriting what is returned by the Lstat function.
type StatFS struct {
fs.FS
overrideFI *fs.ExtendedFileInfo
resetFIOnRead bool
overrideNode *restic.Node
overrideErr error
OverrideLstat map[string]os.FileInfo
OnlyOverrideStat bool
}
func (m *overrideFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
f, err := m.FS.OpenFile(name, flag, metadataOnly)
if err != nil {
return f, err
func (fs *StatFS) Lstat(name string) (os.FileInfo, error) {
if !fs.OnlyOverrideStat {
if fi, ok := fs.OverrideLstat[fixpath(name)]; ok {
return fi, nil
}
}
if filepath.Base(name) == "testfile" || filepath.Base(name) == "testdir" {
return &overrideFile{f, m}, nil
}
return f, nil
return fs.FS.Lstat(name)
}
type overrideFile struct {
func (fs *StatFS) OpenFile(name string, flags int, perm os.FileMode) (fs.File, error) {
if fi, ok := fs.OverrideLstat[fixpath(name)]; ok {
f, err := fs.FS.OpenFile(name, flags, perm)
if err != nil {
return nil, err
}
wrappedFile := fileStat{
File: f,
fi: fi,
}
return wrappedFile, nil
}
return fs.FS.OpenFile(name, flags, perm)
}
type fileStat struct {
fs.File
ofs *overrideFS
fi os.FileInfo
}
func (f overrideFile) Stat() (*fs.ExtendedFileInfo, error) {
if f.ofs.overrideFI == nil {
return f.File.Stat()
}
return f.ofs.overrideFI, nil
}
func (f overrideFile) MakeReadable() error {
if f.ofs.resetFIOnRead {
f.ofs.overrideFI = nil
}
return f.File.MakeReadable()
}
func (f overrideFile) ToNode(ignoreXattrListError bool) (*restic.Node, error) {
if f.ofs.overrideNode == nil {
return f.File.ToNode(ignoreXattrListError)
}
return f.ofs.overrideNode, f.ofs.overrideErr
func (f fileStat) Stat() (os.FileInfo, error) {
return f.fi, nil
}
// used by wrapFileInfo, use untyped const in order to avoid having a version
@ -2272,19 +2291,17 @@ func TestMetadataChanged(t *testing.T) {
// get metadata
fi := lstat(t, "testfile")
localFS := &fs.Local{}
meta, err := localFS.OpenFile("testfile", fs.O_NOFOLLOW, true)
rtest.OK(t, err)
want, err := meta.ToNode(false)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fs := &overrideFS{
FS: localFS,
overrideFI: fi,
overrideNode: &restic.Node{},
want, err := restic.NodeFromFileInfo("testfile", fi, false)
if err != nil {
t.Fatal(err)
}
fs := &StatFS{
FS: fs.Local{},
OverrideLstat: map[string]os.FileInfo{
"testfile": fi,
},
}
*fs.overrideNode = *want
sn, node2 := snapshot(t, repo, fs, nil, "testfile")
@ -2303,31 +2320,26 @@ func TestMetadataChanged(t *testing.T) {
t.Fatalf("metadata does not match:\n%v", cmp.Diff(want, node2))
}
// modify the mode and UID/GID
modFI := *fi
modFI.Mode = mockFileInfoMode
if runtime.GOOS != "windows" {
modFI.UID = mockFileInfoUID
modFI.GID = mockFileInfoGID
}
fs.overrideFI = &modFI
rtest.Assert(t, !fileChanged(fs.overrideFI, node2, 0), "testfile must not be considered as changed")
// modify the mode by wrapping it in a new struct, uses the consts defined above
fs.OverrideLstat["testfile"] = wrapFileInfo(fi)
// set the override values in the 'want' node which
want.Mode = mockFileInfoMode
want.Mode = 0400
// ignore UID and GID on Windows
if runtime.GOOS != "windows" {
want.UID = mockFileInfoUID
want.GID = mockFileInfoGID
want.UID = 51234
want.GID = 51235
}
// update mock node accordingly
fs.overrideNode.Mode = want.Mode
fs.overrideNode.UID = want.UID
fs.overrideNode.GID = want.GID
// no user and group name
want.User = ""
want.Group = ""
// make another snapshot
_, node3 := snapshot(t, repo, fs, sn, "testfile")
// Override username and group to empty string - in case underlying system has user with UID 51234
// See https://github.com/restic/restic/issues/2372
node3.User = ""
node3.Group = ""
// make sure that metadata was recorded successfully
if !cmp.Equal(want, node3) {
@ -2340,84 +2352,63 @@ func TestMetadataChanged(t *testing.T) {
checker.TestCheckRepo(t, repo, false)
}
func TestRacyFileTypeSwap(t *testing.T) {
func TestRacyFileSwap(t *testing.T) {
files := TestDir{
"testfile": TestFile{
"file": TestFile{
Content: "foo bar test file",
},
"testdir": TestDir{},
}
for _, dirError := range []bool{false, true} {
desc := "file changed type"
if dirError {
desc = "dir changed type"
}
t.Run(desc, func(t *testing.T) {
tempdir, repo := prepareTempdirRepoSrc(t, files)
tempdir, repo := prepareTempdirRepoSrc(t, files)
back := rtest.Chdir(t, tempdir)
defer back()
back := rtest.Chdir(t, tempdir)
defer back()
// get metadata of current folder
var fakeName, realName string
if dirError {
// lstat claims this is a directory, but it's actually a file
fakeName = "testdir"
realName = "testfile"
} else {
fakeName = "testfile"
realName = "testdir"
}
fakeFI := lstat(t, fakeName)
tempfile := filepath.Join(tempdir, realName)
// get metadata of current folder
fi := lstat(t, ".")
tempfile := filepath.Join(tempdir, "file")
statfs := &overrideFS{
FS: fs.Local{},
overrideFI: fakeFI,
resetFIOnRead: true,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wg, ctx := errgroup.WithContext(ctx)
repo.StartPackUploader(ctx, wg)
arch := New(repo, fs.Track{FS: statfs}, Options{})
arch.Error = func(item string, err error) error {
t.Logf("archiver error as expected for %v: %v", item, err)
return err
}
arch.runWorkers(ctx, wg)
// fs.Track will panic if the file was not closed
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "changed type, refusing to archive"), "save() returned wrong error: %v", err)
tpe := "file"
if dirError {
tpe = "directory"
}
rtest.Assert(t, strings.Contains(err.Error(), tpe+" "), "unexpected item type in error: %v", err)
rtest.Assert(t, !excluded, "Save() excluded the node, that's unexpected")
})
statfs := &StatFS{
FS: fs.Local{},
OverrideLstat: map[string]os.FileInfo{
tempfile: fi,
},
OnlyOverrideStat: true,
}
}
type mockToNoder struct {
node *restic.Node
err error
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
func (m *mockToNoder) ToNode(_ bool) (*restic.Node, error) {
return m.node, m.err
wg, ctx := errgroup.WithContext(ctx)
repo.StartPackUploader(ctx, wg)
arch := New(repo, fs.Track{FS: statfs}, Options{})
arch.Error = func(item string, err error) error {
t.Logf("archiver error as expected for %v: %v", item, err)
return err
}
arch.runWorkers(ctx, wg)
// fs.Track will panic if the file was not closed
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
if err == nil {
t.Errorf("Save() should have failed")
}
if excluded {
t.Errorf("Save() excluded the node, that's unexpected")
}
}
func TestMetadataBackupErrorFiltering(t *testing.T) {
tempdir := t.TempDir()
filename := filepath.Join(tempdir, "file")
repo := repository.TestRepository(t)
filename := filepath.Join(tempdir, "file")
rtest.OK(t, os.WriteFile(filename, []byte("example"), 0o600))
fi, err := os.Stat(filename)
rtest.OK(t, err)
arch := New(repo, fs.Local{}, Options{})
var filteredErr error
@ -2427,24 +2418,15 @@ func TestMetadataBackupErrorFiltering(t *testing.T) {
return replacementErr
}
nonExistNoder := &mockToNoder{
node: &restic.Node{Type: restic.NodeTypeFile},
err: fmt.Errorf("not found"),
}
// check that errors from reading extended metadata are properly filtered
node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, false)
node, err := arch.nodeFromFileInfo("file", filename+"invalid", fi, false)
rtest.Assert(t, node != nil, "node is missing")
rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err)
rtest.Assert(t, filteredErr != nil, "missing inner error")
// check that errors from reading irregular file are not filtered
filteredErr = nil
nonExistNoder = &mockToNoder{
node: &restic.Node{Type: restic.NodeTypeIrregular},
err: fmt.Errorf(`unsupported file type "irregular"`),
}
node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, false)
node, err = arch.nodeFromFileInfo("file", filename, wrapIrregularFileInfo(fi), false)
rtest.Assert(t, node != nil, "node is missing")
rtest.Assert(t, filteredErr == nil, "error for irregular node should not have been filtered")
rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err)
@ -2463,22 +2445,18 @@ func TestIrregularFile(t *testing.T) {
tempfile := filepath.Join(tempdir, "testfile")
fi := lstat(t, "testfile")
// patch mode to irregular
fi.Mode = (fi.Mode &^ os.ModeType) | os.ModeIrregular
override := &overrideFS{
FS: fs.Local{},
overrideFI: fi,
overrideNode: &restic.Node{
Type: restic.NodeTypeIrregular,
statfs := &StatFS{
FS: fs.Local{},
OverrideLstat: map[string]os.FileInfo{
tempfile: wrapIrregularFileInfo(fi),
},
overrideErr: fmt.Errorf(`unsupported file type "irregular"`),
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
arch := New(repo, fs.Track{FS: override}, Options{})
arch := New(repo, fs.Track{FS: statfs}, Options{})
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
if err == nil {
t.Fatalf("Save() should have failed")
@ -2489,48 +2467,3 @@ func TestIrregularFile(t *testing.T) {
t.Errorf("Save() excluded the node, that's unexpected")
}
}
type missingFS struct {
fs.FS
errorOnOpen bool
}
func (fs *missingFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
if fs.errorOnOpen {
return nil, os.ErrNotExist
}
return &missingFile{}, nil
}
type missingFile struct {
fs.File
}
func (f *missingFile) Stat() (*fs.ExtendedFileInfo, error) {
return nil, os.ErrNotExist
}
func (f *missingFile) Close() error {
// prevent segfault in test
return nil
}
func TestDisappearedFile(t *testing.T) {
tempdir, repo := prepareTempdirRepoSrc(t, TestDir{})
back := rtest.Chdir(t, tempdir)
defer back()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// depending on the underlying FS implementation a missing file may be detected by OpenFile or
// the subsequent file.Stat() call. Thus test both cases.
for _, errorOnOpen := range []bool{false, true} {
arch := New(repo, fs.Track{FS: &missingFS{FS: &fs.Local{}, errorOnOpen: errorOnOpen}}, Options{})
_, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil)
rtest.OK(t, err)
rtest.Assert(t, excluded, "testfile should have been excluded")
}
}

View file

@ -4,6 +4,8 @@
package archiver
import (
"os"
"syscall"
"testing"
"github.com/restic/restic/internal/feature"
@ -12,9 +14,54 @@ import (
rtest "github.com/restic/restic/internal/test"
)
type wrappedFileInfo struct {
os.FileInfo
sys interface{}
mode os.FileMode
}
func (fi wrappedFileInfo) Sys() interface{} {
return fi.sys
}
func (fi wrappedFileInfo) Mode() os.FileMode {
return fi.mode
}
// wrapFileInfo returns a new os.FileInfo with the mode, owner, and group fields changed.
func wrapFileInfo(fi os.FileInfo) os.FileInfo {
// get the underlying stat_t and modify the values
stat := fi.Sys().(*syscall.Stat_t)
stat.Mode = mockFileInfoMode
stat.Uid = mockFileInfoUID
stat.Gid = mockFileInfoGID
// wrap the os.FileInfo so we can return a modified stat_t
res := wrappedFileInfo{
FileInfo: fi,
sys: stat,
mode: mockFileInfoMode,
}
return res
}
// wrapIrregularFileInfo returns a new os.FileInfo with the mode changed to irregular file
func wrapIrregularFileInfo(fi os.FileInfo) os.FileInfo {
// wrap the os.FileInfo so we can return a modified stat_t
return wrappedFileInfo{
FileInfo: fi,
sys: fi.Sys().(*syscall.Stat_t),
mode: (fi.Mode() &^ os.ModeType) | os.ModeIrregular,
}
}
func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) {
want := nodeFromFile(t, &fs.Local{}, name)
_, node := snapshot(t, repo, &fs.Local{}, nil, name)
fi := lstat(t, name)
want, err := restic.NodeFromFileInfo(name, fi, false)
rtest.OK(t, err)
_, node := snapshot(t, repo, fs.Local{}, nil, name)
return want, node
}

View file

@ -0,0 +1,36 @@
//go:build windows
// +build windows
package archiver
import (
"os"
)
type wrappedFileInfo struct {
os.FileInfo
mode os.FileMode
}
func (fi wrappedFileInfo) Mode() os.FileMode {
return fi.mode
}
// wrapFileInfo returns a new os.FileInfo with the mode, owner, and group fields changed.
func wrapFileInfo(fi os.FileInfo) os.FileInfo {
// wrap the os.FileInfo and return the modified mode, uid and gid are ignored on Windows
res := wrappedFileInfo{
FileInfo: fi,
mode: mockFileInfoMode,
}
return res
}
// wrapIrregularFileInfo returns a new os.FileInfo with the mode changed to irregular file
func wrapIrregularFileInfo(fi os.FileInfo) os.FileInfo {
return wrappedFileInfo{
FileInfo: fi,
mode: (fi.Mode() &^ os.ModeType) | os.ModeIrregular,
}
}

View file

@ -9,22 +9,22 @@ import (
"golang.org/x/sync/errgroup"
)
// saver allows saving a blob.
type saver interface {
// Saver allows saving a blob.
type Saver interface {
SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error)
}
// blobSaver concurrently saves incoming blobs to the repo.
type blobSaver struct {
repo saver
// BlobSaver concurrently saves incoming blobs to the repo.
type BlobSaver struct {
repo Saver
ch chan<- saveBlobJob
}
// newBlobSaver returns a new blob. A worker pool is started, it is stopped
// NewBlobSaver returns a new blob. A worker pool is started, it is stopped
// when ctx is cancelled.
func newBlobSaver(ctx context.Context, wg *errgroup.Group, repo saver, workers uint) *blobSaver {
func NewBlobSaver(ctx context.Context, wg *errgroup.Group, repo Saver, workers uint) *BlobSaver {
ch := make(chan saveBlobJob)
s := &blobSaver{
s := &BlobSaver{
repo: repo,
ch: ch,
}
@ -38,13 +38,13 @@ func newBlobSaver(ctx context.Context, wg *errgroup.Group, repo saver, workers u
return s
}
func (s *blobSaver) TriggerShutdown() {
func (s *BlobSaver) TriggerShutdown() {
close(s.ch)
}
// Save stores a blob in the repo. It checks the index and the known blobs
// before saving anything. It takes ownership of the buffer passed in.
func (s *blobSaver) Save(ctx context.Context, t restic.BlobType, buf *buffer, filename string, cb func(res saveBlobResponse)) {
func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, filename string, cb func(res SaveBlobResponse)) {
select {
case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}:
case <-ctx.Done():
@ -54,26 +54,26 @@ func (s *blobSaver) Save(ctx context.Context, t restic.BlobType, buf *buffer, fi
type saveBlobJob struct {
restic.BlobType
buf *buffer
buf *Buffer
fn string
cb func(res saveBlobResponse)
cb func(res SaveBlobResponse)
}
type saveBlobResponse struct {
type SaveBlobResponse struct {
id restic.ID
length int
sizeInRepo int
known bool
}
func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) {
func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (SaveBlobResponse, error) {
id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false)
if err != nil {
return saveBlobResponse{}, err
return SaveBlobResponse{}, err
}
return saveBlobResponse{
return SaveBlobResponse{
id: id,
length: len(buf),
sizeInRepo: sizeInRepo,
@ -81,7 +81,7 @@ func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte)
}, nil
}
func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error {
func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error {
for {
var job saveBlobJob
var ok bool

View file

@ -38,20 +38,20 @@ func TestBlobSaver(t *testing.T) {
wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{}
b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
var wait sync.WaitGroup
var results []saveBlobResponse
var results []SaveBlobResponse
var lock sync.Mutex
wait.Add(20)
for i := 0; i < 20; i++ {
buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
idx := i
lock.Lock()
results = append(results, saveBlobResponse{})
results = append(results, SaveBlobResponse{})
lock.Unlock()
b.Save(ctx, restic.DataBlob, buf, "file", func(res saveBlobResponse) {
b.Save(ctx, restic.DataBlob, buf, "file", func(res SaveBlobResponse) {
lock.Lock()
results[idx] = res
lock.Unlock()
@ -95,11 +95,11 @@ func TestBlobSaverError(t *testing.T) {
failAt: int32(test.failAt),
}
b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
for i := 0; i < test.blobs; i++ {
buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
b.Save(ctx, restic.DataBlob, buf, "errfile", func(res saveBlobResponse) {})
buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
b.Save(ctx, restic.DataBlob, buf, "errfile", func(res SaveBlobResponse) {})
}
b.TriggerShutdown()

View file

@ -1,14 +1,14 @@
package archiver
// buffer is a reusable buffer. After the buffer has been used, Release should
// Buffer is a reusable buffer. After the buffer has been used, Release should
// be called so the underlying slice is put back into the pool.
type buffer struct {
type Buffer struct {
Data []byte
pool *bufferPool
pool *BufferPool
}
// Release puts the buffer back into the pool it came from.
func (b *buffer) Release() {
func (b *Buffer) Release() {
pool := b.pool
if pool == nil || cap(b.Data) > pool.defaultSize {
return
@ -20,32 +20,32 @@ func (b *buffer) Release() {
}
}
// bufferPool implements a limited set of reusable buffers.
type bufferPool struct {
ch chan *buffer
// BufferPool implements a limited set of reusable buffers.
type BufferPool struct {
ch chan *Buffer
defaultSize int
}
// newBufferPool initializes a new buffer pool. The pool stores at most max
// NewBufferPool initializes a new buffer pool. The pool stores at most max
// items. New buffers are created with defaultSize. Buffers that have grown
// larger are not put back.
func newBufferPool(max int, defaultSize int) *bufferPool {
b := &bufferPool{
ch: make(chan *buffer, max),
func NewBufferPool(max int, defaultSize int) *BufferPool {
b := &BufferPool{
ch: make(chan *Buffer, max),
defaultSize: defaultSize,
}
return b
}
// Get returns a new buffer, either from the pool or newly allocated.
func (pool *bufferPool) Get() *buffer {
func (pool *BufferPool) Get() *Buffer {
select {
case buf := <-pool.ch:
return buf
default:
}
b := &buffer{
b := &Buffer{
Data: make([]byte, pool.defaultSize),
pool: pool,
}

View file

@ -1,3 +1,12 @@
// Package archiver contains the code which reads files, splits them into
// chunks and saves the data to the repository.
//
// An Archiver has a number of worker goroutines handling saving the different
// data structures to the repository, the details are implemented by the
// FileSaver, BlobSaver, and TreeSaver types.
//
// The main goroutine (the one calling Snapshot()) traverses the directory tree
// and delegates all work to these worker pools. They return a type
// (FutureFile, FutureBlob, and FutureTree) which can be resolved later, by
// calling Wait() on it.
package archiver

View file

@ -1,318 +0,0 @@
package archiver
import (
"bytes"
"fmt"
"io"
"os"
"runtime"
"strings"
"sync"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
)
// RejectByNameFunc is a function that takes a filename of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectByNameFunc func(path string) bool
// RejectFunc is a function that takes a filename and os.FileInfo of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectFunc func(path string, fi *fs.ExtendedFileInfo, fs fs.FS) bool
func CombineRejectByNames(funcs []RejectByNameFunc) SelectByNameFunc {
return func(item string) bool {
for _, reject := range funcs {
if reject(item) {
return false
}
}
return true
}
}
func CombineRejects(funcs []RejectFunc) SelectFunc {
return func(item string, fi *fs.ExtendedFileInfo, fs fs.FS) bool {
for _, reject := range funcs {
if reject(item, fi, fs) {
return false
}
}
return true
}
}
type rejectionCache struct {
m map[string]bool
mtx sync.Mutex
}
func newRejectionCache() *rejectionCache {
return &rejectionCache{m: make(map[string]bool)}
}
// Lock locks the mutex in rc.
func (rc *rejectionCache) Lock() {
rc.mtx.Lock()
}
// Unlock unlocks the mutex in rc.
func (rc *rejectionCache) Unlock() {
rc.mtx.Unlock()
}
// Get returns the last stored value for dir and a second boolean that
// indicates whether that value was actually written to the cache. It is the
// callers responsibility to call rc.Lock and rc.Unlock before using this
// method, otherwise data races may occur.
func (rc *rejectionCache) Get(dir string) (bool, bool) {
v, ok := rc.m[dir]
return v, ok
}
// Store stores a new value for dir. It is the callers responsibility to call
// rc.Lock and rc.Unlock before using this method, otherwise data races may
// occur.
func (rc *rejectionCache) Store(dir string, rejected bool) {
rc.m[dir] = rejected
}
// RejectIfPresent returns a RejectByNameFunc which itself returns whether a path
// should be excluded. The RejectByNameFunc considers a file to be excluded when
// it resides in a directory with an exclusion file, that is specified by
// excludeFileSpec in the form "filename[:content]". The returned error is
// non-nil if the filename component of excludeFileSpec is empty. If rc is
// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation
// of a directory based on previous visits.
func RejectIfPresent(excludeFileSpec string, warnf func(msg string, args ...interface{})) (RejectFunc, error) {
if excludeFileSpec == "" {
return nil, errors.New("name for exclusion tagfile is empty")
}
colon := strings.Index(excludeFileSpec, ":")
if colon == 0 {
return nil, fmt.Errorf("no name for exclusion tagfile provided")
}
tf, tc := "", ""
if colon > 0 {
tf = excludeFileSpec[:colon]
tc = excludeFileSpec[colon+1:]
} else {
tf = excludeFileSpec
}
debug.Log("using %q as exclusion tagfile", tf)
rc := newRejectionCache()
return func(filename string, _ *fs.ExtendedFileInfo, fs fs.FS) bool {
return isExcludedByFile(filename, tf, tc, rc, fs, warnf)
}, nil
}
// isExcludedByFile interprets filename as a path and returns true if that file
// is in an excluded directory. A directory is identified as excluded if it contains a
// tagfile which bears the name specified in tagFilename and starts with
// header. If rc is non-nil, it is used to expedite the evaluation of a
// directory based on previous visits.
func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache, fs fs.FS, warnf func(msg string, args ...interface{})) bool {
if tagFilename == "" {
return false
}
if fs.Base(filename) == tagFilename {
return false // do not exclude the tagfile itself
}
rc.Lock()
defer rc.Unlock()
dir := fs.Dir(filename)
rejected, visited := rc.Get(dir)
if visited {
return rejected
}
rejected = isDirExcludedByFile(dir, tagFilename, header, fs, warnf)
rc.Store(dir, rejected)
return rejected
}
func isDirExcludedByFile(dir, tagFilename, header string, fsInst fs.FS, warnf func(msg string, args ...interface{})) bool {
tf := fsInst.Join(dir, tagFilename)
_, err := fsInst.Lstat(tf)
if errors.Is(err, os.ErrNotExist) {
return false
}
if err != nil {
warnf("could not access exclusion tagfile: %v", err)
return false
}
// when no signature is given, the mere presence of tf is enough reason
// to exclude filename
if len(header) == 0 {
return true
}
// From this stage, errors mean tagFilename exists but it is malformed.
// Warnings will be generated so that the user is informed that the
// indented ignore-action is not performed.
f, err := fsInst.OpenFile(tf, fs.O_RDONLY, false)
if err != nil {
warnf("could not open exclusion tagfile: %v", err)
return false
}
defer func() {
_ = f.Close()
}()
buf := make([]byte, len(header))
_, err = io.ReadFull(f, buf)
// EOF is handled with a dedicated message, otherwise the warning were too cryptic
if err == io.EOF {
warnf("invalid (too short) signature in exclusion tagfile %q\n", tf)
return false
}
if err != nil {
warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err)
return false
}
if !bytes.Equal(buf, []byte(header)) {
warnf("invalid signature in exclusion tagfile %q\n", tf)
return false
}
return true
}
// deviceMap is used to track allowed source devices for backup. This is used to
// check for crossing mount points during backup (for --one-file-system). It
// maps the name of a source path to its device ID.
type deviceMap map[string]uint64
// newDeviceMap creates a new device map from the list of source paths.
func newDeviceMap(allowedSourcePaths []string, fs fs.FS) (deviceMap, error) {
if runtime.GOOS == "windows" {
return nil, errors.New("Device IDs are not supported on Windows")
}
deviceMap := make(map[string]uint64)
for _, item := range allowedSourcePaths {
item, err := fs.Abs(fs.Clean(item))
if err != nil {
return nil, err
}
fi, err := fs.Lstat(item)
if err != nil {
return nil, err
}
deviceMap[item] = fi.DeviceID
}
if len(deviceMap) == 0 {
return nil, errors.New("zero allowed devices")
}
return deviceMap, nil
}
// IsAllowed returns true if the path is located on an allowed device.
func (m deviceMap) IsAllowed(item string, deviceID uint64, fs fs.FS) (bool, error) {
for dir := item; ; dir = fs.Dir(dir) {
debug.Log("item %v, test dir %v", item, dir)
// find a parent directory that is on an allowed device (otherwise
// we would not traverse the directory at all)
allowedID, ok := m[dir]
if !ok {
if dir == fs.Dir(dir) {
// arrived at root, no allowed device found. this should not happen.
break
}
continue
}
// if the item has a different device ID than the parent directory,
// we crossed a file system boundary
if allowedID != deviceID {
debug.Log("item %v (dir %v) on disallowed device %d", item, dir, deviceID)
return false, nil
}
// item is on allowed device, accept it
debug.Log("item %v allowed", item)
return true, nil
}
return false, fmt.Errorf("item %v (device ID %v) not found, deviceMap: %v", item, deviceID, m)
}
// RejectByDevice returns a RejectFunc that rejects files which are on a
// different file systems than the files/dirs in samples.
func RejectByDevice(samples []string, filesystem fs.FS) (RejectFunc, error) {
deviceMap, err := newDeviceMap(samples, filesystem)
if err != nil {
return nil, err
}
debug.Log("allowed devices: %v\n", deviceMap)
return func(item string, fi *fs.ExtendedFileInfo, fs fs.FS) bool {
allowed, err := deviceMap.IsAllowed(fs.Clean(item), fi.DeviceID, fs)
if err != nil {
// this should not happen
panic(fmt.Sprintf("error checking device ID of %v: %v", item, err))
}
if allowed {
// accept item
return false
}
// reject everything except directories
if !fi.Mode.IsDir() {
return true
}
// special case: make sure we keep mountpoints (directories which
// contain a mounted file system). Test this by checking if the parent
// directory would be included.
parentDir := fs.Dir(fs.Clean(item))
parentFI, err := fs.Lstat(parentDir)
if err != nil {
debug.Log("item %v: error running lstat() on parent directory: %v", item, err)
// if in doubt, reject
return true
}
parentAllowed, err := deviceMap.IsAllowed(parentDir, parentFI.DeviceID, fs)
if err != nil {
debug.Log("item %v: error checking parent directory: %v", item, err)
// if in doubt, reject
return true
}
if parentAllowed {
// we found a mount point, so accept the directory
return false
}
// reject everything else
return true
}, nil
}
func RejectBySize(maxSize int64) (RejectFunc, error) {
return func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
// directory will be ignored
if fi.Mode.IsDir() {
return false
}
filesize := fi.Size
if filesize > maxSize {
debug.Log("file %s is oversize: %d", item, filesize)
return true
}
return false
}, nil
}

View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"os"
"sync"
"github.com/restic/chunker"
@ -14,13 +15,13 @@ import (
"golang.org/x/sync/errgroup"
)
// saveBlobFn saves a blob to a repo.
type saveBlobFn func(context.Context, restic.BlobType, *buffer, string, func(res saveBlobResponse))
// SaveBlobFn saves a blob to a repo.
type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, string, func(res SaveBlobResponse))
// fileSaver concurrently saves incoming files to the repo.
type fileSaver struct {
saveFilePool *bufferPool
saveBlob saveBlobFn
// FileSaver concurrently saves incoming files to the repo.
type FileSaver struct {
saveFilePool *BufferPool
saveBlob SaveBlobFn
pol chunker.Pol
@ -28,21 +29,21 @@ type fileSaver struct {
CompleteBlob func(bytes uint64)
NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error)
NodeFromFileInfo func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error)
}
// newFileSaver returns a new file saver. A worker pool with fileWorkers is
// NewFileSaver returns a new file saver. A worker pool with fileWorkers is
// started, it is stopped when ctx is cancelled.
func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *fileSaver {
func NewFileSaver(ctx context.Context, wg *errgroup.Group, save SaveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *FileSaver {
ch := make(chan saveFileJob)
debug.Log("new file saver with %v file workers and %v blob workers", fileWorkers, blobWorkers)
poolSize := fileWorkers + blobWorkers
s := &fileSaver{
s := &FileSaver{
saveBlob: save,
saveFilePool: newBufferPool(int(poolSize), chunker.MaxSize),
saveFilePool: NewBufferPool(int(poolSize), chunker.MaxSize),
pol: pol,
ch: ch,
@ -59,23 +60,24 @@ func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol
return s
}
func (s *fileSaver) TriggerShutdown() {
func (s *FileSaver) TriggerShutdown() {
close(s.ch)
}
// fileCompleteFunc is called when the file has been saved.
type fileCompleteFunc func(*restic.Node, ItemStats)
// CompleteFunc is called when the file has been saved.
type CompleteFunc func(*restic.Node, ItemStats)
// Save stores the file f and returns the data once it has been completed. The
// file is closed by Save. completeReading is only called if the file was read
// successfully. complete is always called. If completeReading is called, then
// this will always happen before calling complete.
func (s *fileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, start func(), completeReading func(), complete fileCompleteFunc) futureNode {
func (s *FileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), completeReading func(), complete CompleteFunc) FutureNode {
fn, ch := newFutureNode()
job := saveFileJob{
snPath: snPath,
target: target,
file: file,
fi: fi,
ch: ch,
start: start,
@ -98,15 +100,16 @@ type saveFileJob struct {
snPath string
target string
file fs.File
fi os.FileInfo
ch chan<- futureNodeResult
start func()
completeReading func()
complete fileCompleteFunc
complete CompleteFunc
}
// saveFile stores the file f in the repo, then closes it.
func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, start func(), finishReading func(), finish func(res futureNodeResult)) {
func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func(), finishReading func(), finish func(res futureNodeResult)) {
start()
fnr := futureNodeResult{
@ -153,14 +156,14 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
debug.Log("%v", snPath)
node, err := s.NodeFromFileInfo(snPath, target, f, false)
node, err := s.NodeFromFileInfo(snPath, target, fi, false)
if err != nil {
_ = f.Close()
completeError(err)
return
}
if node.Type != restic.NodeTypeFile {
if node.Type != "file" {
_ = f.Close()
completeError(errors.Errorf("node type %q is wrong", node.Type))
return
@ -202,7 +205,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
node.Content = append(node.Content, restic.ID{})
lock.Unlock()
s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr saveBlobResponse) {
s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr SaveBlobResponse) {
lock.Lock()
if !sbr.known {
fnr.stats.DataBlobs++
@ -243,7 +246,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
completeBlob()
}
func (s *fileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) {
func (s *FileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) {
// a worker has one chunker which is reused for each file (because it contains a rather large buffer)
chnker := chunker.New(nil, s.pol)
@ -259,7 +262,7 @@ func (s *fileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) {
}
}
s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.start, func() {
s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.fi, job.start, func() {
if job.completeReading != nil {
job.completeReading()
}

View file

@ -30,11 +30,11 @@ func createTestFiles(t testing.TB, num int) (files []string) {
return files
}
func startFileSaver(ctx context.Context, t testing.TB, fsInst fs.FS) (*fileSaver, context.Context, *errgroup.Group) {
func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) {
wg, ctx := errgroup.WithContext(ctx)
saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *buffer, _ string, cb func(saveBlobResponse)) {
cb(saveBlobResponse{
saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, _ string, cb func(SaveBlobResponse)) {
cb(SaveBlobResponse{
id: restic.Hash(buf.Data),
length: len(buf.Data),
sizeInRepo: len(buf.Data),
@ -48,9 +48,9 @@ func startFileSaver(ctx context.Context, t testing.TB, fsInst fs.FS) (*fileSaver
t.Fatal(err)
}
s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers)
s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) {
return meta.ToNode(ignoreXattrListError)
s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers)
s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError)
}
return s, ctx, wg
@ -67,17 +67,22 @@ func TestFileSaver(t *testing.T) {
completeFn := func(*restic.Node, ItemStats) {}
testFs := fs.Local{}
s, ctx, wg := startFileSaver(ctx, t, testFs)
s, ctx, wg := startFileSaver(ctx, t)
var results []futureNode
var results []FutureNode
for _, filename := range files {
f, err := testFs.OpenFile(filename, os.O_RDONLY, false)
f, err := testFs.Open(filename)
if err != nil {
t.Fatal(err)
}
ff := s.Save(ctx, filename, filename, f, startFn, completeReadingFn, completeFn)
fi, err := f.Stat()
if err != nil {
t.Fatal(err)
}
ff := s.Save(ctx, filename, filename, f, fi, startFn, completeReadingFn, completeFn)
results = append(results, ff)
}

View file

@ -2,6 +2,8 @@ package archiver
import (
"context"
"os"
"path/filepath"
"sort"
"github.com/restic/restic/internal/debug"
@ -20,11 +22,11 @@ type Scanner struct {
}
// NewScanner initializes a new Scanner.
func NewScanner(filesystem fs.FS) *Scanner {
func NewScanner(fs fs.FS) *Scanner {
return &Scanner{
FS: filesystem,
FS: fs,
SelectByName: func(_ string) bool { return true },
Select: func(_ string, _ *fs.ExtendedFileInfo, _ fs.FS) bool { return true },
Select: func(_ string, _ os.FileInfo) bool { return true },
Error: func(_ string, err error) error { return err },
Result: func(_ string, _ ScanStats) {},
}
@ -36,7 +38,7 @@ type ScanStats struct {
Bytes uint64
}
func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree tree) (ScanStats, error) {
func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree) (ScanStats, error) {
// traverse the path in the file system for all leaf nodes
if tree.Leaf() {
abstarget, err := s.FS.Abs(tree.Path)
@ -81,7 +83,7 @@ func (s *Scanner) Scan(ctx context.Context, targets []string) error {
debug.Log("clean targets %v", cleanTargets)
// we're using the same tree representation as the archiver does
tree, err := newTree(s.FS, cleanTargets)
tree, err := NewTree(s.FS, cleanTargets)
if err != nil {
return err
}
@ -113,15 +115,15 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca
}
// run remaining select functions that require file information
if !s.Select(target, fi, s.FS) {
if !s.Select(target, fi) {
return stats, nil
}
switch {
case fi.Mode.IsRegular():
case fi.Mode().IsRegular():
stats.Files++
stats.Bytes += uint64(fi.Size)
case fi.Mode.IsDir():
stats.Bytes += uint64(fi.Size())
case fi.Mode().IsDir():
names, err := fs.Readdirnames(s.FS, target, fs.O_NOFOLLOW)
if err != nil {
return stats, s.Error(target, err)
@ -129,7 +131,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca
sort.Strings(names)
for _, name := range names {
stats, err = s.scan(ctx, stats, s.FS.Join(target, name))
stats, err = s.scan(ctx, stats, filepath.Join(target, name))
if err != nil {
return stats, err
}

View file

@ -56,8 +56,8 @@ func TestScanner(t *testing.T) {
},
},
},
selFn: func(item string, fi *fs.ExtendedFileInfo, fs fs.FS) bool {
if fi.Mode.IsDir() {
selFn: func(item string, fi os.FileInfo) bool {
if fi.IsDir() {
return true
}

View file

@ -95,17 +95,17 @@ func TestCreateFiles(t testing.TB, target string, dir TestDir) {
t.Fatal(err)
}
case TestSymlink:
err := os.Symlink(filepath.FromSlash(it.Target), targetPath)
err := fs.Symlink(filepath.FromSlash(it.Target), targetPath)
if err != nil {
t.Fatal(err)
}
case TestHardlink:
err := os.Link(filepath.Join(target, filepath.FromSlash(it.Target)), targetPath)
err := fs.Link(filepath.Join(target, filepath.FromSlash(it.Target)), targetPath)
if err != nil {
t.Fatal(err)
}
case TestDir:
err := os.Mkdir(targetPath, 0755)
err := fs.Mkdir(targetPath, 0755)
if err != nil {
t.Fatal(err)
}
@ -157,7 +157,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
// first, test that all items are there
TestWalkFiles(t, target, dir, func(path string, item interface{}) error {
fi, err := os.Lstat(path)
fi, err := fs.Lstat(path)
if err != nil {
return err
}
@ -169,7 +169,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
}
return nil
case TestFile:
if !fi.Mode().IsRegular() {
if !fs.IsRegularFile(fi) {
t.Errorf("is not a regular file: %v", path)
return nil
}
@ -188,7 +188,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
return nil
}
target, err := os.Readlink(path)
target, err := fs.Readlink(path)
if err != nil {
return err
}
@ -208,7 +208,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
})
// then, traverse the directory again, looking for additional files
err := filepath.Walk(target, func(path string, fi os.FileInfo, err error) error {
err := fs.Walk(target, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
@ -289,7 +289,7 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti
switch e := entry.(type) {
case TestDir:
if node.Type != restic.NodeTypeDir {
if node.Type != "dir" {
t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "dir")
return
}
@ -301,13 +301,13 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti
TestEnsureTree(ctx, t, path.Join(prefix, node.Name), repo, *node.Subtree, e)
case TestFile:
if node.Type != restic.NodeTypeFile {
if node.Type != "file" {
t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "file")
}
TestEnsureFileContent(ctx, t, repo, nodePrefix, node, e)
case TestSymlink:
if node.Type != restic.NodeTypeSymlink {
t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "symlink")
if node.Type != "symlink" {
t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "file")
}
if e.Target != node.LinkTarget {

View file

@ -54,7 +54,7 @@ func (t *MockT) Errorf(msg string, args ...interface{}) {
func createFilesAt(t testing.TB, targetdir string, files map[string]interface{}) {
for name, item := range files {
target := filepath.Join(targetdir, filepath.FromSlash(name))
err := os.MkdirAll(filepath.Dir(target), 0700)
err := fs.MkdirAll(filepath.Dir(target), 0700)
if err != nil {
t.Fatal(err)
}
@ -66,7 +66,7 @@ func createFilesAt(t testing.TB, targetdir string, files map[string]interface{})
t.Fatal(err)
}
case TestSymlink:
err := os.Symlink(filepath.FromSlash(it.Target), target)
err := fs.Symlink(filepath.FromSlash(it.Target), target)
if err != nil {
t.Fatal(err)
}
@ -105,7 +105,7 @@ func TestTestCreateFiles(t *testing.T) {
t.Run("", func(t *testing.T) {
tempdir := filepath.Join(tempdir, fmt.Sprintf("test-%d", i))
err := os.MkdirAll(tempdir, 0700)
err := fs.MkdirAll(tempdir, 0700)
if err != nil {
t.Fatal(err)
}
@ -114,7 +114,7 @@ func TestTestCreateFiles(t *testing.T) {
for name, item := range test.files {
targetPath := filepath.Join(tempdir, filepath.FromSlash(name))
fi, err := os.Lstat(targetPath)
fi, err := fs.Lstat(targetPath)
if err != nil {
t.Error(err)
continue
@ -122,7 +122,7 @@ func TestTestCreateFiles(t *testing.T) {
switch node := item.(type) {
case TestFile:
if !fi.Mode().IsRegular() {
if !fs.IsRegularFile(fi) {
t.Errorf("is not regular file: %v", name)
continue
}
@ -142,7 +142,7 @@ func TestTestCreateFiles(t *testing.T) {
continue
}
target, err := os.Readlink(targetPath)
target, err := fs.Readlink(targetPath)
if err != nil {
t.Error(err)
continue
@ -455,7 +455,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
tempdir := rtest.TempDir(t)
targetDir := filepath.Join(tempdir, "target")
err := os.Mkdir(targetDir, 0700)
err := fs.Mkdir(targetDir, 0700)
if err != nil {
t.Fatal(err)
}

View file

@ -9,7 +9,7 @@ import (
"github.com/restic/restic/internal/fs"
)
// tree recursively defines how a snapshot should look like when
// Tree recursively defines how a snapshot should look like when
// archived.
//
// When `Path` is set, this is a leaf node and the contents of `Path` should be
@ -20,8 +20,8 @@ import (
//
// `FileInfoPath` is used to extract metadata for intermediate (=non-leaf)
// trees.
type tree struct {
Nodes map[string]tree
type Tree struct {
Nodes map[string]Tree
Path string // where the files/dirs to be saved are found
FileInfoPath string // where the dir can be found that is not included itself, but its subdirs
Root string // parent directory of the tree
@ -95,13 +95,13 @@ func rootDirectory(fs fs.FS, target string) string {
}
// Add adds a new file or directory to the tree.
func (t *tree) Add(fs fs.FS, path string) error {
func (t *Tree) Add(fs fs.FS, path string) error {
if path == "" {
panic("invalid path (empty string)")
}
if t.Nodes == nil {
t.Nodes = make(map[string]tree)
t.Nodes = make(map[string]Tree)
}
pc, virtualPrefix := pathComponents(fs, path, false)
@ -111,7 +111,7 @@ func (t *tree) Add(fs fs.FS, path string) error {
name := pc[0]
root := rootDirectory(fs, path)
tree := tree{Root: root}
tree := Tree{Root: root}
origName := name
i := 0
@ -152,63 +152,63 @@ func (t *tree) Add(fs fs.FS, path string) error {
}
// add adds a new target path into the tree.
func (t *tree) add(fs fs.FS, target, root string, pc []string) error {
func (t *Tree) add(fs fs.FS, target, root string, pc []string) error {
if len(pc) == 0 {
return errors.Errorf("invalid path %q", target)
}
if t.Nodes == nil {
t.Nodes = make(map[string]tree)
t.Nodes = make(map[string]Tree)
}
name := pc[0]
if len(pc) == 1 {
node, ok := t.Nodes[name]
tree, ok := t.Nodes[name]
if !ok {
t.Nodes[name] = tree{Path: target}
t.Nodes[name] = Tree{Path: target}
return nil
}
if node.Path != "" {
if tree.Path != "" {
return errors.Errorf("path is already set for target %v", target)
}
node.Path = target
t.Nodes[name] = node
tree.Path = target
t.Nodes[name] = tree
return nil
}
node := tree{}
tree := Tree{}
if other, ok := t.Nodes[name]; ok {
node = other
tree = other
}
subroot := fs.Join(root, name)
node.FileInfoPath = subroot
tree.FileInfoPath = subroot
err := node.add(fs, target, subroot, pc[1:])
err := tree.add(fs, target, subroot, pc[1:])
if err != nil {
return err
}
t.Nodes[name] = node
t.Nodes[name] = tree
return nil
}
func (t tree) String() string {
func (t Tree) String() string {
return formatTree(t, "")
}
// Leaf returns true if this is a leaf node, which means Path is set to a
// non-empty string and the contents of Path should be inserted at this point
// in the tree.
func (t tree) Leaf() bool {
func (t Tree) Leaf() bool {
return t.Path != ""
}
// NodeNames returns the sorted list of subtree names.
func (t tree) NodeNames() []string {
func (t Tree) NodeNames() []string {
// iterate over the nodes of atree in lexicographic (=deterministic) order
names := make([]string, 0, len(t.Nodes))
for name := range t.Nodes {
@ -219,7 +219,7 @@ func (t tree) NodeNames() []string {
}
// formatTree returns a text representation of the tree t.
func formatTree(t tree, indent string) (s string) {
func formatTree(t Tree, indent string) (s string) {
for name, node := range t.Nodes {
s += fmt.Sprintf("%v/%v, root %q, path %q, meta %q\n", indent, name, node.Root, node.Path, node.FileInfoPath)
s += formatTree(node, indent+" ")
@ -228,7 +228,7 @@ func formatTree(t tree, indent string) (s string) {
}
// unrollTree unrolls the tree so that only leaf nodes have Path set.
func unrollTree(f fs.FS, t *tree) error {
func unrollTree(f fs.FS, t *Tree) error {
// if the current tree is a leaf node (Path is set) and has additional
// nodes, add the contents of Path to the nodes.
if t.Path != "" && len(t.Nodes) > 0 {
@ -252,7 +252,7 @@ func unrollTree(f fs.FS, t *tree) error {
return errors.Errorf("tree unrollTree: collision on path, node %#v, path %q", node, f.Join(t.Path, entry))
}
t.Nodes[entry] = tree{Path: f.Join(t.Path, entry)}
t.Nodes[entry] = Tree{Path: f.Join(t.Path, entry)}
}
t.Path = ""
}
@ -269,10 +269,10 @@ func unrollTree(f fs.FS, t *tree) error {
return nil
}
// newTree creates a Tree from the target files/directories.
func newTree(fs fs.FS, targets []string) (*tree, error) {
// NewTree creates a Tree from the target files/directories.
func NewTree(fs fs.FS, targets []string) (*Tree, error) {
debug.Log("targets: %v", targets)
tree := &tree{}
tree := &Tree{}
seen := make(map[string]struct{})
for _, target := range targets {
target = fs.Clean(target)

View file

@ -9,20 +9,20 @@ import (
"golang.org/x/sync/errgroup"
)
// treeSaver concurrently saves incoming trees to the repo.
type treeSaver struct {
saveBlob saveBlobFn
// TreeSaver concurrently saves incoming trees to the repo.
type TreeSaver struct {
saveBlob SaveBlobFn
errFn ErrorFunc
ch chan<- saveTreeJob
}
// newTreeSaver returns a new tree saver. A worker pool with treeWorkers is
// NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is
// started, it is stopped when ctx is cancelled.
func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob saveBlobFn, errFn ErrorFunc) *treeSaver {
func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob SaveBlobFn, errFn ErrorFunc) *TreeSaver {
ch := make(chan saveTreeJob)
s := &treeSaver{
s := &TreeSaver{
ch: ch,
saveBlob: saveBlob,
errFn: errFn,
@ -37,12 +37,12 @@ func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, sav
return s
}
func (s *treeSaver) TriggerShutdown() {
func (s *TreeSaver) TriggerShutdown() {
close(s.ch)
}
// Save stores the dir d and returns the data once it has been completed.
func (s *treeSaver) Save(ctx context.Context, snPath string, target string, node *restic.Node, nodes []futureNode, complete fileCompleteFunc) futureNode {
func (s *TreeSaver) Save(ctx context.Context, snPath string, target string, node *restic.Node, nodes []FutureNode, complete CompleteFunc) FutureNode {
fn, ch := newFutureNode()
job := saveTreeJob{
snPath: snPath,
@ -66,13 +66,13 @@ type saveTreeJob struct {
snPath string
target string
node *restic.Node
nodes []futureNode
nodes []FutureNode
ch chan<- futureNodeResult
complete fileCompleteFunc
complete CompleteFunc
}
// save stores the nodes as a tree in the repo.
func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, ItemStats, error) {
func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, ItemStats, error) {
var stats ItemStats
node := job.node
nodes := job.nodes
@ -84,7 +84,7 @@ func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
for i, fn := range nodes {
// fn is a copy, so clear the original value explicitly
nodes[i] = futureNode{}
nodes[i] = FutureNode{}
fnr := fn.take(ctx)
// return the error if it wasn't ignored
@ -128,9 +128,9 @@ func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
return nil, stats, err
}
b := &buffer{Data: buf}
ch := make(chan saveBlobResponse, 1)
s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res saveBlobResponse) {
b := &Buffer{Data: buf}
ch := make(chan SaveBlobResponse, 1)
s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res SaveBlobResponse) {
ch <- res
})
@ -149,7 +149,7 @@ func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
}
}
func (s *treeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error {
func (s *TreeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error {
for {
var job saveTreeJob
var ok bool

View file

@ -12,8 +12,8 @@ import (
"golang.org/x/sync/errgroup"
)
func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *buffer, _ string, cb func(res saveBlobResponse)) {
cb(saveBlobResponse{
func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, _ string, cb func(res SaveBlobResponse)) {
cb(SaveBlobResponse{
id: restic.NewRandomID(),
known: false,
length: len(buf.Data),
@ -21,7 +21,7 @@ func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *buffer, _ string,
})
}
func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() error) {
func setupTreeSaver() (context.Context, context.CancelFunc, *TreeSaver, func() error) {
ctx, cancel := context.WithCancel(context.Background())
wg, ctx := errgroup.WithContext(ctx)
@ -29,7 +29,7 @@ func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() e
return err
}
b := newTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn)
b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn)
shutdown := func() error {
b.TriggerShutdown()
@ -43,7 +43,7 @@ func TestTreeSaver(t *testing.T) {
ctx, cancel, b, shutdown := setupTreeSaver()
defer cancel()
var results []futureNode
var results []FutureNode
for i := 0; i < 20; i++ {
node := &restic.Node{
@ -83,13 +83,13 @@ func TestTreeSaverError(t *testing.T) {
ctx, cancel, b, shutdown := setupTreeSaver()
defer cancel()
var results []futureNode
var results []FutureNode
for i := 0; i < test.trees; i++ {
node := &restic.Node{
Name: fmt.Sprintf("file-%d", i),
}
nodes := []futureNode{
nodes := []FutureNode{
newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
Name: fmt.Sprintf("child-%d", i),
}}),
@ -128,7 +128,7 @@ func TestTreeSaverDuplicates(t *testing.T) {
node := &restic.Node{
Name: "file",
}
nodes := []futureNode{
nodes := []FutureNode{
newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
Name: "child",
}}),

View file

@ -12,7 +12,7 @@ import (
)
// debug.Log requires Tree.String.
var _ fmt.Stringer = tree{}
var _ fmt.Stringer = Tree{}
func TestPathComponents(t *testing.T) {
var tests = []struct {
@ -142,20 +142,20 @@ func TestTree(t *testing.T) {
var tests = []struct {
targets []string
src TestDir
want tree
want Tree
unix bool
win bool
mustError bool
}{
{
targets: []string{"foo"},
want: tree{Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Path: "foo", Root: "."},
}},
},
{
targets: []string{"foo", "bar", "baz"},
want: tree{Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Path: "foo", Root: "."},
"bar": {Path: "bar", Root: "."},
"baz": {Path: "baz", Root: "."},
@ -163,8 +163,8 @@ func TestTree(t *testing.T) {
},
{
targets: []string{"foo/user1", "foo/user2", "foo/other"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/user1")},
"user2": {Path: filepath.FromSlash("foo/user2")},
"other": {Path: filepath.FromSlash("foo/other")},
@ -173,9 +173,9 @@ func TestTree(t *testing.T) {
},
{
targets: []string{"foo/work/user1", "foo/work/user2"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/work/user1")},
"user2": {Path: filepath.FromSlash("foo/work/user2")},
}},
@ -184,50 +184,50 @@ func TestTree(t *testing.T) {
},
{
targets: []string{"foo/user1", "bar/user1", "foo/other"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/user1")},
"other": {Path: filepath.FromSlash("foo/other")},
}},
"bar": {Root: ".", FileInfoPath: "bar", Nodes: map[string]tree{
"bar": {Root: ".", FileInfoPath: "bar", Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("bar/user1")},
}},
}},
},
{
targets: []string{"../work"},
want: tree{Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"work": {Root: "..", Path: filepath.FromSlash("../work")},
}},
},
{
targets: []string{"../work/other"},
want: tree{Nodes: map[string]tree{
"work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]Tree{
"other": {Path: filepath.FromSlash("../work/other")},
}},
}},
},
{
targets: []string{"foo/user1", "../work/other", "foo/user2"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/user1")},
"user2": {Path: filepath.FromSlash("foo/user2")},
}},
"work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]tree{
"work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]Tree{
"other": {Path: filepath.FromSlash("../work/other")},
}},
}},
},
{
targets: []string{"foo/user1", "../foo/other", "foo/user2"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/user1")},
"user2": {Path: filepath.FromSlash("foo/user2")},
}},
"foo-1": {Root: "..", FileInfoPath: filepath.FromSlash("../foo"), Nodes: map[string]tree{
"foo-1": {Root: "..", FileInfoPath: filepath.FromSlash("../foo"), Nodes: map[string]Tree{
"other": {Path: filepath.FromSlash("../foo/other")},
}},
}},
@ -240,11 +240,11 @@ func TestTree(t *testing.T) {
},
},
targets: []string{"foo", "foo/work"},
want: tree{Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {
Root: ".",
FileInfoPath: "foo",
Nodes: map[string]tree{
Nodes: map[string]Tree{
"file": {Path: filepath.FromSlash("foo/file")},
"work": {Path: filepath.FromSlash("foo/work")},
},
@ -261,11 +261,11 @@ func TestTree(t *testing.T) {
},
},
targets: []string{"foo/work", "foo"},
want: tree{Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {
Root: ".",
FileInfoPath: "foo",
Nodes: map[string]tree{
Nodes: map[string]Tree{
"file": {Path: filepath.FromSlash("foo/file")},
"work": {Path: filepath.FromSlash("foo/work")},
},
@ -282,11 +282,11 @@ func TestTree(t *testing.T) {
},
},
targets: []string{"foo/work", "foo/work/user2"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"work": {
FileInfoPath: filepath.FromSlash("foo/work"),
Nodes: map[string]tree{
Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/work/user1")},
"user2": {Path: filepath.FromSlash("foo/work/user2")},
},
@ -304,10 +304,10 @@ func TestTree(t *testing.T) {
},
},
targets: []string{"foo/work/user2", "foo/work"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"),
Nodes: map[string]tree{
Nodes: map[string]Tree{
"user1": {Path: filepath.FromSlash("foo/work/user1")},
"user2": {Path: filepath.FromSlash("foo/work/user2")},
},
@ -332,12 +332,12 @@ func TestTree(t *testing.T) {
},
},
targets: []string{"foo/work/user2/data/secret", "foo"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"other": {Path: filepath.FromSlash("foo/other")},
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{
"user2": {FileInfoPath: filepath.FromSlash("foo/work/user2"), Nodes: map[string]tree{
"data": {FileInfoPath: filepath.FromSlash("foo/work/user2/data"), Nodes: map[string]tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{
"user2": {FileInfoPath: filepath.FromSlash("foo/work/user2"), Nodes: map[string]Tree{
"data": {FileInfoPath: filepath.FromSlash("foo/work/user2/data"), Nodes: map[string]Tree{
"secret": {
Path: filepath.FromSlash("foo/work/user2/data/secret"),
},
@ -368,10 +368,10 @@ func TestTree(t *testing.T) {
},
unix: true,
targets: []string{"mnt/driveA", "mnt/driveA/work/driveB"},
want: tree{Nodes: map[string]tree{
"mnt": {Root: ".", FileInfoPath: filepath.FromSlash("mnt"), Nodes: map[string]tree{
"driveA": {FileInfoPath: filepath.FromSlash("mnt/driveA"), Nodes: map[string]tree{
"work": {FileInfoPath: filepath.FromSlash("mnt/driveA/work"), Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"mnt": {Root: ".", FileInfoPath: filepath.FromSlash("mnt"), Nodes: map[string]Tree{
"driveA": {FileInfoPath: filepath.FromSlash("mnt/driveA"), Nodes: map[string]Tree{
"work": {FileInfoPath: filepath.FromSlash("mnt/driveA/work"), Nodes: map[string]Tree{
"driveB": {
Path: filepath.FromSlash("mnt/driveA/work/driveB"),
},
@ -384,9 +384,9 @@ func TestTree(t *testing.T) {
},
{
targets: []string{"foo/work/user", "foo/work/user"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{
"user": {Path: filepath.FromSlash("foo/work/user")},
}},
}},
@ -394,9 +394,9 @@ func TestTree(t *testing.T) {
},
{
targets: []string{"./foo/work/user", "foo/work/user"},
want: tree{Nodes: map[string]tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{
"work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{
"user": {Path: filepath.FromSlash("foo/work/user")},
}},
}},
@ -405,10 +405,10 @@ func TestTree(t *testing.T) {
{
win: true,
targets: []string{`c:\users\foobar\temp`},
want: tree{Nodes: map[string]tree{
"c": {Root: `c:\`, FileInfoPath: `c:\`, Nodes: map[string]tree{
"users": {FileInfoPath: `c:\users`, Nodes: map[string]tree{
"foobar": {FileInfoPath: `c:\users\foobar`, Nodes: map[string]tree{
want: Tree{Nodes: map[string]Tree{
"c": {Root: `c:\`, FileInfoPath: `c:\`, Nodes: map[string]Tree{
"users": {FileInfoPath: `c:\users`, Nodes: map[string]Tree{
"foobar": {FileInfoPath: `c:\users\foobar`, Nodes: map[string]Tree{
"temp": {Path: `c:\users\foobar\temp`},
}},
}},
@ -445,7 +445,7 @@ func TestTree(t *testing.T) {
back := rtest.Chdir(t, tempdir)
defer back()
tree, err := newTree(fs.Local{}, test.targets)
tree, err := NewTree(fs.Local{}, test.targets)
if test.mustError {
if err == nil {
t.Fatal("expected error, got nil")

View file

@ -37,8 +37,6 @@ type Backend struct {
prefix string
listMaxItems int
layout.Layout
accessTier blob.AccessTier
}
const saveLargeSize = 256 * 1024 * 1024
@ -62,11 +60,6 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
} else {
endpointSuffix = "core.windows.net"
}
if cfg.AccountName == "" {
return nil, errors.Fatalf("unable to open Azure backend: Account name ($AZURE_ACCOUNT_NAME) is empty")
}
url := fmt.Sprintf("https://%s.blob.%s/%s", cfg.AccountName, endpointSuffix, cfg.Container)
opts := &azContainer.ClientOptions{
ClientOptions: azcore.ClientOptions{
@ -131,33 +124,20 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
}
var accessTier blob.AccessTier
// if the access tier is not supported, then we will not set the access tier; during the upload process,
// the value will be inferred from the default configured on the storage account.
for _, tier := range supportedAccessTiers() {
if strings.EqualFold(string(tier), cfg.AccessTier) {
accessTier = tier
debug.Log(" - using access tier %v", accessTier)
break
}
}
be := &Backend{
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: &layout.DefaultLayout{
Path: cfg.Prefix,
Join: path.Join,
},
listMaxItems: defaultListMaxItems,
accessTier: accessTier,
}
return be, nil
}
func supportedAccessTiers() []blob.AccessTier {
return []blob.AccessTier{blob.AccessTierHot, blob.AccessTierCool, blob.AccessTierCold, blob.AccessTierArchive}
}
// Open opens the Azure backend at specified container.
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
return open(cfg, rt)
@ -217,6 +197,11 @@ func (be *Backend) IsPermanentError(err error) bool {
return false
}
// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
return path.Join(p...)
}
func (be *Backend) Connections() uint {
return be.connections
}
@ -236,39 +221,25 @@ func (be *Backend) Path() string {
return be.prefix
}
// useAccessTier determines whether to apply the configured access tier to a given file.
// For archive access tier, only data files are stored using that class; metadata
// must remain instantly accessible.
func (be *Backend) useAccessTier(h backend.Handle) bool {
notArchiveClass := !strings.EqualFold(be.cfg.AccessTier, "archive")
isDataFile := h.Type == backend.PackFile && !h.IsMetadata
return isDataFile || notArchiveClass
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
var accessTier blob.AccessTier
if be.useAccessTier(h) {
accessTier = be.accessTier
}
var err error
if rd.Length() < saveLargeSize {
// if it's smaller than 256miB, then just create the file directly from the reader
err = be.saveSmall(ctx, objName, rd, accessTier)
err = be.saveSmall(ctx, objName, rd)
} else {
// otherwise use the more complicated method
err = be.saveLarge(ctx, objName, rd, accessTier)
err = be.saveLarge(ctx, objName, rd)
}
return err
}
func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader, accessTier blob.AccessTier) error {
func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
// upload it as a new "block", use the base64 hash for the ID
@ -289,13 +260,11 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.Rew
}
blocks := []string{id}
_, err = blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{
Tier: &accessTier,
})
_, err = blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{})
return errors.Wrap(err, "CommitBlockList")
}
func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader, accessTier blob.AccessTier) error {
func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
buf := make([]byte, 100*1024*1024)
@ -342,9 +311,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.Rew
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length())
}
_, err := blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{
Tier: &accessTier,
})
_, err := blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{})
debug.Log("uploaded %d parts: %v", len(blocks), blocks)
return errors.Wrap(err, "CommitBlockList")

View file

@ -22,8 +22,7 @@ type Config struct {
Container string
Prefix string
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
AccessTier string `option:"access-tier" help:"set the access tier for the blob storage (default: inferred from the storage account defaults)"`
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
}
// NewConfig returns a new Config with the default values filled in.

View file

@ -107,10 +107,13 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backen
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
client: client,
bucket: bucket,
cfg: cfg,
Layout: &layout.DefaultLayout{
Join: path.Join,
Path: cfg.Prefix,
},
listMaxItems: defaultListMaxItems,
canDelete: true,
}
@ -140,10 +143,13 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Back
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
client: client,
bucket: bucket,
cfg: cfg,
Layout: &layout.DefaultLayout{
Join: path.Join,
Path: cfg.Prefix,
},
listMaxItems: defaultListMaxItems,
}
return be, nil

View file

@ -12,6 +12,7 @@ import (
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
)
@ -53,7 +54,7 @@ const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"
func writeCachedirTag(dir string) error {
tagfile := filepath.Join(dir, "CACHEDIR.TAG")
f, err := os.OpenFile(tagfile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, fileMode)
f, err := fs.OpenFile(tagfile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, fileMode)
if err != nil {
if errors.Is(err, os.ErrExist) {
return nil
@ -84,7 +85,7 @@ func New(id string, basedir string) (c *Cache, err error) {
}
}
err = os.MkdirAll(basedir, dirMode)
err = fs.MkdirAll(basedir, dirMode)
if err != nil {
return nil, errors.WithStack(err)
}
@ -112,7 +113,7 @@ func New(id string, basedir string) (c *Cache, err error) {
case errors.Is(err, os.ErrNotExist):
// Create the repo cache dir. The parent exists, so Mkdir suffices.
err := os.Mkdir(cachedir, dirMode)
err := fs.Mkdir(cachedir, dirMode)
switch {
case err == nil:
created = true
@ -133,7 +134,7 @@ func New(id string, basedir string) (c *Cache, err error) {
}
for _, p := range cacheLayoutPaths {
if err = os.MkdirAll(filepath.Join(cachedir, p), dirMode); err != nil {
if err = fs.MkdirAll(filepath.Join(cachedir, p), dirMode); err != nil {
return nil, errors.WithStack(err)
}
}
@ -151,7 +152,7 @@ func New(id string, basedir string) (c *Cache, err error) {
// directory d to the current time.
func updateTimestamp(d string) error {
t := time.Now()
return os.Chtimes(d, t, t)
return fs.Chtimes(d, t, t)
}
// MaxCacheAge is the default age (30 days) after which cache directories are considered old.
@ -164,7 +165,7 @@ func validCacheDirName(s string) bool {
// listCacheDirs returns the list of cache directories.
func listCacheDirs(basedir string) ([]os.FileInfo, error) {
f, err := os.Open(basedir)
f, err := fs.Open(basedir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
err = nil

View file

@ -12,6 +12,7 @@ import (
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
)
@ -43,7 +44,7 @@ func (c *Cache) load(h backend.Handle, length int, offset int64) (io.ReadCloser,
return nil, false, errors.New("cannot be cached")
}
f, err := os.Open(c.filename(h))
f, err := fs.Open(c.filename(h))
if err != nil {
return nil, false, errors.WithStack(err)
}
@ -90,7 +91,7 @@ func (c *Cache) save(h backend.Handle, rd io.Reader) error {
finalname := c.filename(h)
dir := filepath.Dir(finalname)
err := os.Mkdir(dir, 0700)
err := fs.Mkdir(dir, 0700)
if err != nil && !errors.Is(err, os.ErrExist) {
return err
}
@ -105,26 +106,26 @@ func (c *Cache) save(h backend.Handle, rd io.Reader) error {
n, err := io.Copy(f, rd)
if err != nil {
_ = f.Close()
_ = os.Remove(f.Name())
_ = fs.Remove(f.Name())
return errors.Wrap(err, "Copy")
}
if n <= int64(crypto.CiphertextLength(0)) {
_ = f.Close()
_ = os.Remove(f.Name())
_ = fs.Remove(f.Name())
debug.Log("trying to cache truncated file %v, removing", h)
return nil
}
// Close, then rename. Windows doesn't like the reverse order.
if err = f.Close(); err != nil {
_ = os.Remove(f.Name())
_ = fs.Remove(f.Name())
return errors.WithStack(err)
}
err = os.Rename(f.Name(), finalname)
err = fs.Rename(f.Name(), finalname)
if err != nil {
_ = os.Remove(f.Name())
_ = fs.Remove(f.Name())
}
if runtime.GOOS == "windows" && errors.Is(err, os.ErrPermission) {
// On Windows, renaming over an existing file is ok
@ -161,7 +162,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
return false, nil
}
err := os.Remove(c.filename(h))
err := fs.Remove(c.filename(h))
removed := err == nil
if errors.Is(err, os.ErrNotExist) {
err = nil
@ -188,7 +189,7 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
}
// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
@ -239,6 +240,6 @@ func (c *Cache) Has(h backend.Handle) bool {
return false
}
_, err := os.Stat(c.filename(h))
_, err := fs.Stat(c.filename(h))
return err == nil
}

View file

@ -12,16 +12,17 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet {
ids := restic.NewIDSet()
for i := 0; i < random.Intn(15)+10; i++ {
buf := rtest.Random(random.Int(), 1<<19)
for i := 0; i < rand.Intn(15)+10; i++ {
buf := rtest.Random(rand.Int(), 1<<19)
id := restic.Hash(buf)
h := backend.Handle{Type: tpe, Name: id.String()}
@ -87,7 +88,7 @@ func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet)
func TestFiles(t *testing.T) {
seed := time.Now().Unix()
t.Logf("seed is %v", seed)
random := rand.New(rand.NewSource(seed))
rand.Seed(seed)
c := TestNewCache(t)
@ -99,7 +100,7 @@ func TestFiles(t *testing.T) {
for _, tpe := range tests {
t.Run(tpe.String(), func(t *testing.T) {
ids := generateRandomFiles(t, random, tpe, c)
ids := generateRandomFiles(t, tpe, c)
id := randomID(ids)
h := backend.Handle{Type: tpe, Name: id.String()}
@ -139,12 +140,12 @@ func TestFiles(t *testing.T) {
func TestFileLoad(t *testing.T) {
seed := time.Now().Unix()
t.Logf("seed is %v", seed)
random := rand.New(rand.NewSource(seed))
rand.Seed(seed)
c := TestNewCache(t)
// save about 5 MiB of data in the cache
data := rtest.Random(random.Int(), 5234142)
data := rtest.Random(rand.Int(), 5234142)
id := restic.ID{}
copy(id[:], data)
h := backend.Handle{
@ -222,10 +223,6 @@ func TestFileSaveConcurrent(t *testing.T) {
t.Skip("may not work due to FILE_SHARE_DELETE issue")
}
seed := time.Now().Unix()
t.Logf("seed is %v", seed)
random := rand.New(rand.NewSource(seed))
const nproc = 40
var (
@ -234,8 +231,7 @@ func TestFileSaveConcurrent(t *testing.T) {
g errgroup.Group
id restic.ID
)
random.Read(id[:])
rand.Read(id[:])
h := backend.Handle{
Type: restic.PackFile,
@ -277,7 +273,7 @@ func TestFileSaveConcurrent(t *testing.T) {
func TestFileSaveAfterDamage(t *testing.T) {
c := TestNewCache(t)
rtest.OK(t, os.RemoveAll(c.path))
rtest.OK(t, fs.RemoveAll(c.path))
// save a few bytes of data in the cache
data := rtest.Random(123456789, 42)

View file

@ -105,14 +105,17 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
be := &Backend{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix,
Layout: &layout.DefaultLayout{
Path: cfg.Prefix,
Join: path.Join,
},
listMaxItems: defaultListMaxItems,
}
@ -186,6 +189,11 @@ func (be *Backend) IsPermanentError(err error) bool {
return false
}
// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
return path.Join(p...)
}
func (be *Backend) Connections() uint {
return be.connections
}

View file

@ -1,7 +1,18 @@
package layout
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
)
// Layout computes paths for file name storage.
@ -12,3 +23,159 @@ type Layout interface {
Paths() []string
Name() string
}
// Filesystem is the abstraction of a file system used for a backend.
type Filesystem interface {
Join(...string) string
ReadDir(context.Context, string) ([]os.FileInfo, error)
IsNotExist(error) bool
}
// ensure statically that *LocalFilesystem implements Filesystem.
var _ Filesystem = &LocalFilesystem{}
// LocalFilesystem implements Filesystem in a local path.
type LocalFilesystem struct {
}
// ReadDir returns all entries of a directory.
func (l *LocalFilesystem) ReadDir(_ context.Context, dir string) ([]os.FileInfo, error) {
f, err := fs.Open(dir)
if err != nil {
return nil, err
}
entries, err := f.Readdir(-1)
if err != nil {
return nil, errors.Wrap(err, "Readdir")
}
err = f.Close()
if err != nil {
return nil, errors.Wrap(err, "Close")
}
return entries, nil
}
// Join combines several path components to one.
func (l *LocalFilesystem) Join(paths ...string) string {
return filepath.Join(paths...)
}
// IsNotExist returns true for errors that are caused by not existing files.
func (l *LocalFilesystem) IsNotExist(err error) bool {
return os.IsNotExist(err)
}
var backendFilenameLength = len(restic.ID{}) * 2
var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength))
func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(ctx, dir)
if err != nil && fs.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "ReadDir")
}
for _, e := range entries {
if backendFilename.MatchString(e.Name()) {
return true, nil
}
}
return false, nil
}
// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
// cannot be detected automatically.
var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
var ErrLegacyLayoutFound = errors.New("detected legacy S3 layout. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your repository")
// DetectLayout tries to find out which layout is used in a local (or sftp)
// filesystem at the given path. If repo is nil, an instance of LocalFilesystem
// is used.
func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, error) {
debug.Log("detect layout at %v", dir)
if repo == nil {
repo = &LocalFilesystem{}
}
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
if foundKeysFile && !foundKeyFile {
debug.Log("found default layout at %v", dir)
return &DefaultLayout{
Path: dir,
Join: repo.Join,
}, nil
}
if foundKeyFile && !foundKeysFile {
if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
return nil, ErrLegacyLayoutFound
}
debug.Log("found s3 layout at %v", dir)
return &S3LegacyLayout{
Path: dir,
Join: repo.Join,
}, nil
}
debug.Log("layout detection failed")
return nil, ErrLayoutDetectionFailed
}
// ParseLayout parses the config string and returns a Layout. When layout is
// the empty string, DetectLayout is used. If that fails, defaultLayout is used.
func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
debug.Log("parse layout string %q for backend at %v", layout, path)
switch layout {
case "default":
l = &DefaultLayout{
Path: path,
Join: repo.Join,
}
case "s3legacy":
if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
return nil, ErrLegacyLayoutFound
}
l = &S3LegacyLayout{
Path: path,
Join: repo.Join,
}
case "":
l, err = DetectLayout(ctx, repo, path)
// use the default layout if auto detection failed
if errors.Is(err, ErrLayoutDetectionFailed) && defaultLayout != "" {
debug.Log("error: %v, use default layout %v", err, defaultLayout)
return ParseLayout(ctx, repo, defaultLayout, "", path)
}
if err != nil {
return nil, err
}
debug.Log("layout detected: %v", l)
default:
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout)
}
return l, nil
}

View file

@ -11,8 +11,8 @@ import (
// subdirs, two characters each (taken from the first two characters of the
// file name).
type DefaultLayout struct {
path string
join func(...string) string
Path string
Join func(...string) string
}
var defaultLayoutPaths = map[backend.FileType]string{
@ -23,13 +23,6 @@ var defaultLayoutPaths = map[backend.FileType]string{
backend.KeyFile: "keys",
}
func NewDefaultLayout(path string, join func(...string) string) *DefaultLayout {
return &DefaultLayout{
path: path,
join: join,
}
}
func (l *DefaultLayout) String() string {
return "<DefaultLayout>"
}
@ -44,32 +37,32 @@ func (l *DefaultLayout) Dirname(h backend.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == backend.PackFile && len(h.Name) > 2 {
p = l.join(p, h.Name[:2]) + "/"
p = l.Join(p, h.Name[:2]) + "/"
}
return l.join(l.path, p) + "/"
return l.Join(l.Path, p) + "/"
}
// Filename returns a path to a file, including its name.
func (l *DefaultLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == backend.ConfigFile {
return l.join(l.path, "config")
return l.Join(l.Path, "config")
}
return l.join(l.Dirname(h), name)
return l.Join(l.Dirname(h), name)
}
// Paths returns all directory names needed for a repo.
func (l *DefaultLayout) Paths() (dirs []string) {
for _, p := range defaultLayoutPaths {
dirs = append(dirs, l.join(l.path, p))
dirs = append(dirs, l.Join(l.Path, p))
}
// also add subdirs
for i := 0; i < 256; i++ {
subdir := hex.EncodeToString([]byte{byte(i)})
dirs = append(dirs, l.join(l.path, defaultLayoutPaths[backend.PackFile], subdir))
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir))
}
return dirs
@ -81,6 +74,6 @@ func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs boo
subdirs = true
}
dirname = l.join(l.path, defaultLayoutPaths[t])
dirname = l.Join(l.Path, defaultLayoutPaths[t])
return
}

View file

@ -1,24 +1,18 @@
package layout
import (
"path"
"github.com/restic/restic/internal/backend"
)
// RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct {
url string
URL string
Path string
Join func(...string) string
}
var restLayoutPaths = defaultLayoutPaths
func NewRESTLayout(url string) *RESTLayout {
return &RESTLayout{
url: url,
}
}
func (l *RESTLayout) String() string {
return "<RESTLayout>"
}
@ -31,10 +25,10 @@ func (l *RESTLayout) Name() string {
// Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.url + "/"
return l.URL + l.Join(l.Path, "/")
}
return l.url + path.Join("/", restLayoutPaths[h.Type]) + "/"
return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
@ -45,18 +39,18 @@ func (l *RESTLayout) Filename(h backend.Handle) string {
name = "config"
}
return l.url + path.Join("/", restLayoutPaths[h.Type], name)
return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *RESTLayout) Paths() (dirs []string) {
for _, p := range restLayoutPaths {
dirs = append(dirs, l.url+path.Join("/", p))
dirs = append(dirs, l.URL+l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.url + path.Join("/", restLayoutPaths[t]), false
return l.URL + l.Join(l.Path, restLayoutPaths[t]), false
}

View file

@ -0,0 +1,79 @@
package layout
import (
"github.com/restic/restic/internal/backend"
)
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
type S3LegacyLayout struct {
URL string
Path string
Join func(...string) string
}
var s3LayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshot",
backend.IndexFile: "index",
backend.LockFile: "lock",
backend.KeyFile: "key",
}
func (l *S3LegacyLayout) String() string {
return "<S3LegacyLayout>"
}
// Name returns the name for this layout.
func (l *S3LegacyLayout) Name() string {
return "s3legacy"
}
// join calls Join with the first empty elements removed.
func (l *S3LegacyLayout) join(url string, items ...string) string {
for len(items) > 0 && items[0] == "" {
items = items[1:]
}
path := l.Join(items...)
if path == "" || path[0] != '/' {
if url != "" && url[len(url)-1] != '/' {
url += "/"
}
}
return url + path
}
// Dirname returns the directory path for a given file type and name.
func (l *S3LegacyLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *S3LegacyLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == backend.ConfigFile {
name = "config"
}
return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *S3LegacyLayout) Paths() (dirs []string) {
for _, p := range s3LayoutPaths {
dirs = append(dirs, l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.Join(l.Path, s3LayoutPaths[t]), false
}

View file

@ -1,15 +1,16 @@
package layout
import (
"context"
"fmt"
"path"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/feature"
rtest "github.com/restic/restic/internal/test"
)
@ -98,8 +99,8 @@ func TestDefaultLayout(t *testing.T) {
t.Run("Paths", func(t *testing.T) {
l := &DefaultLayout{
path: tempdir,
join: filepath.Join,
Path: tempdir,
Join: filepath.Join,
}
dirs := l.Paths()
@ -127,8 +128,8 @@ func TestDefaultLayout(t *testing.T) {
for _, test := range tests {
t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
l := &DefaultLayout{
path: test.path,
join: test.join,
Path: test.path,
Join: test.join,
}
filename := l.Filename(test.Handle)
@ -140,7 +141,7 @@ func TestDefaultLayout(t *testing.T) {
}
func TestRESTLayout(t *testing.T) {
url := `https://hostname.foo`
path := rtest.TempDir(t)
var tests = []struct {
backend.Handle
@ -148,43 +149,44 @@ func TestRESTLayout(t *testing.T) {
}{
{
backend.Handle{Type: backend.PackFile, Name: "0123456"},
strings.Join([]string{url, "data", "0123456"}, "/"),
filepath.Join(path, "data", "0123456"),
},
{
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
strings.Join([]string{url, "config"}, "/"),
filepath.Join(path, "config"),
},
{
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
strings.Join([]string{url, "snapshots", "123456"}, "/"),
filepath.Join(path, "snapshots", "123456"),
},
{
backend.Handle{Type: backend.IndexFile, Name: "123456"},
strings.Join([]string{url, "index", "123456"}, "/"),
filepath.Join(path, "index", "123456"),
},
{
backend.Handle{Type: backend.LockFile, Name: "123456"},
strings.Join([]string{url, "locks", "123456"}, "/"),
filepath.Join(path, "locks", "123456"),
},
{
backend.Handle{Type: backend.KeyFile, Name: "123456"},
strings.Join([]string{url, "keys", "123456"}, "/"),
filepath.Join(path, "keys", "123456"),
},
}
l := &RESTLayout{
url: url,
Path: path,
Join: filepath.Join,
}
t.Run("Paths", func(t *testing.T) {
dirs := l.Paths()
want := []string{
strings.Join([]string{url, "data"}, "/"),
strings.Join([]string{url, "snapshots"}, "/"),
strings.Join([]string{url, "index"}, "/"),
strings.Join([]string{url, "locks"}, "/"),
strings.Join([]string{url, "keys"}, "/"),
filepath.Join(path, "data"),
filepath.Join(path, "snapshots"),
filepath.Join(path, "index"),
filepath.Join(path, "locks"),
filepath.Join(path, "keys"),
}
sort.Strings(want)
@ -213,23 +215,59 @@ func TestRESTLayoutURLs(t *testing.T) {
dir string
}{
{
&RESTLayout{url: "https://hostname.foo"},
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&RESTLayout{url: "https://hostname.foo:1234/prefix/repo"},
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&RESTLayout{url: "https://hostname.foo:1234/prefix/repo"},
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"/config",
"/",
},
}
for _, test := range tests {
@ -246,3 +284,165 @@ func TestRESTLayoutURLs(t *testing.T) {
})
}
}
func TestS3LegacyLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
backend.Handle
filename string
}{
{
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"),
},
{
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"),
},
{
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"),
},
}
l := &S3LegacyLayout{
Path: path,
Join: filepath.Join,
}
t.Run("Paths", func(t *testing.T) {
dirs := l.Paths()
want := []string{
filepath.Join(path, "data"),
filepath.Join(path, "snapshot"),
filepath.Join(path, "index"),
filepath.Join(path, "lock"),
filepath.Join(path, "key"),
}
sort.Strings(want)
sort.Strings(dirs)
if !reflect.DeepEqual(dirs, want) {
t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs)
}
})
for _, test := range tests {
t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
filename := l.Filename(test.Handle)
if filename != test.filename {
t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
}
})
}
}
func TestDetectLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
filename string
want string
}{
{"repo-layout-default.tar.gz", "*layout.DefaultLayout"},
{"repo-layout-s3legacy.tar.gz", "*layout.S3LegacyLayout"},
}
var fs = &LocalFilesystem{}
for _, test := range tests {
for _, fs := range []Filesystem{fs, nil} {
t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
rtest.SetupTarTestFixture(t, path, filepath.Join("../testdata", test.filename))
layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
if layout == nil {
t.Fatal("wanted some layout, but detect returned nil")
}
layoutName := fmt.Sprintf("%T", layout)
if layoutName != test.want {
t.Fatalf("want layout %v, got %v", test.want, layoutName)
}
rtest.RemoveAll(t, filepath.Join(path, "repo"))
})
}
}
}
func TestParseLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
layoutName string
defaultLayoutName string
want string
}{
{"default", "", "*layout.DefaultLayout"},
{"s3legacy", "", "*layout.S3LegacyLayout"},
{"", "", "*layout.DefaultLayout"},
}
rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", "repo-layout-default.tar.gz"))
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {
layout, err := ParseLayout(context.TODO(), &LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
if layout == nil {
t.Fatal("wanted some layout, but detect returned nil")
}
// test that the functions work (and don't panic)
_ = layout.Dirname(backend.Handle{Type: backend.PackFile})
_ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"})
_ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout)
if layoutName != test.want {
t.Fatalf("want layout %v, got %v", test.want, layoutName)
}
})
}
}
func TestParseLayoutInvalid(t *testing.T) {
path := rtest.TempDir(t)
var invalidNames = []string{
"foo", "bar", "local",
}
for _, name := range invalidNames {
t.Run(name, func(t *testing.T) {
layout, err := ParseLayout(context.TODO(), nil, name, "", path)
if err == nil {
t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout)
}
})
}
}

View file

@ -9,7 +9,8 @@ import (
// Config holds all information needed to open a local repository.
type Config struct {
Path string
Path string
Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"`
Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"`
}

View file

@ -6,22 +6,30 @@ import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/feature"
rtest "github.com/restic/restic/internal/test"
)
func TestLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
filename string
layout string
failureExpected bool
packfiles map[string]bool
}{
{"repo-layout-default.tar.gz", false, map[string]bool{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
}
for _, test := range tests {
@ -31,6 +39,7 @@ func TestLayout(t *testing.T) {
repo := filepath.Join(path, "repo")
be, err := Open(context.TODO(), Config{
Path: repo,
Layout: test.layout,
Connections: 2,
})
if err != nil {

View file

@ -37,10 +37,15 @@ func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
}
func open(cfg Config) (*Local, error) {
l := layout.NewDefaultLayout(cfg.Path, filepath.Join)
const defaultLayout = "default"
fi, err := os.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
func open(ctx context.Context, cfg Config) (*Local, error) {
l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@ -52,30 +57,30 @@ func open(cfg Config) (*Local, error) {
}
// Open opens the local backend as specified by config.
func Open(_ context.Context, cfg Config) (*Local, error) {
debug.Log("open local backend at %v", cfg.Path)
return open(cfg)
func Open(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
return open(ctx, cfg)
}
// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(_ context.Context, cfg Config) (*Local, error) {
debug.Log("create local backend at %v", cfg.Path)
func Create(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
be, err := open(cfg)
be, err := open(ctx, cfg)
if err != nil {
return nil, err
}
// test if config file already exists
_, err = os.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile}))
_, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
// create paths for data and refs
for _, d := range be.Paths() {
err := os.MkdirAll(d, be.Modes.Dir)
err := fs.MkdirAll(d, be.Modes.Dir)
if err != nil {
return nil, errors.WithStack(err)
}
@ -127,7 +132,7 @@ func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReade
debug.Log("error %v: creating dir", err)
// error is caused by a missing directory, try to create it
mkdirErr := os.MkdirAll(dir, b.Modes.Dir)
mkdirErr := fs.MkdirAll(dir, b.Modes.Dir)
if mkdirErr != nil {
debug.Log("error creating dir %v: %v", dir, mkdirErr)
} else {
@ -147,7 +152,7 @@ func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReade
// temporary's name and no other goroutine will get the same data to
// Save, so the temporary name should never be reused by another
// goroutine.
_ = os.Remove(f.Name())
_ = fs.Remove(f.Name())
}
}(f)
@ -211,7 +216,7 @@ func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset i
}
func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := os.Open(b.Filename(h))
f, err := fs.Open(b.Filename(h))
if err != nil {
return nil, err
}
@ -245,7 +250,7 @@ func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offs
// Stat returns information about a blob.
func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
fi, err := os.Stat(b.Filename(h))
fi, err := fs.Stat(b.Filename(h))
if err != nil {
return backend.FileInfo{}, errors.WithStack(err)
}
@ -258,12 +263,12 @@ func (b *Local) Remove(_ context.Context, h backend.Handle) error {
fn := b.Filename(h)
// reset read-only flag
err := os.Chmod(fn, 0666)
err := fs.Chmod(fn, 0666)
if err != nil && !os.IsPermission(err) {
return errors.WithStack(err)
}
return os.Remove(fn)
return fs.Remove(fn)
}
// List runs fn for each file in the backend which has the type t. When an
@ -289,7 +294,7 @@ func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.Fi
// Also, visitDirs assumes it sees a directory full of directories, while
// visitFiles wants a directory full or regular files.
func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error {
d, err := os.Open(dir)
d, err := fs.Open(dir)
if err != nil {
return err
}
@ -316,7 +321,7 @@ func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error)
}
func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error {
d, err := os.Open(dir)
d, err := fs.Open(dir)
if err != nil {
return err
}
@ -362,7 +367,7 @@ func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error
// Delete removes the repository and all files.
func (b *Local) Delete(_ context.Context) error {
return os.RemoveAll(b.Path)
return fs.RemoveAll(b.Path)
}
// Close closes all open files.

View file

@ -8,6 +8,8 @@ import (
"os"
"runtime"
"syscall"
"github.com/restic/restic/internal/fs"
)
// fsyncDir flushes changes to the directory dir.
@ -43,5 +45,5 @@ func isMacENOTTY(err error) bool {
// set file to readonly
func setFileReadonly(f string, mode os.FileMode) error {
return os.Chmod(f, mode&^0222)
return fs.Chmod(f, mode&^0222)
}

View file

@ -94,7 +94,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru
err = errW
}
if err != nil {
if errors.Is(err, exec.ErrDot) {
if util.IsErrDot(err) {
return nil, nil, nil, nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o rclone.program=./<program> to override", cmd.Path)
}
return nil, nil, nil, nil, err

View file

@ -8,6 +8,7 @@ import (
"io"
"net/http"
"net/url"
"path"
"strings"
"github.com/restic/restic/internal/backend"
@ -65,7 +66,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error)
be := &Backend{
url: cfg.URL,
client: http.Client{Transport: rt},
Layout: layout.NewRESTLayout(url),
Layout: &layout.RESTLayout{URL: url, Join: path.Join},
connections: cfg.Connections,
}

View file

@ -1,3 +1,6 @@
//go:build go1.20
// +build go1.20
package rest_test
import (
@ -106,7 +109,7 @@ func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string)
matched = true
}
}
_, _ = fmt.Fprintln(os.Stdout, line) // print all output to console
fmt.Fprintln(os.Stdout, line) // print all output to console
}
}()

Some files were not shown because too many files have changed in this diff Show more