Merge pull request #5125 from restic/patch-release-cherrypicks

Prepare patch release
This commit is contained in:
Michael Eischer 2024-11-03 22:24:49 +01:00 committed by GitHub
commit fb4d9b3232
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 281 additions and 69 deletions

View file

@ -0,0 +1,9 @@
Bugfix: Fix unusable `mount` on macOS Sonoma
On macOS Sonoma when using fuse-t, it was not possible to access files in
a mounted repository.
This issue has been resolved.
https://github.com/restic/restic/issues/4971
https://github.com/restic/restic/pull/5048

View file

@ -0,0 +1,14 @@
Bugfix: fix metadata errors during backup of removable disks on Windows
Since restic 0.17.0, backups of removable disks on Windows could report
errors with retrieving metadata like shown below.
```
error: incomplete metadata for d:\filename: get named security info failed with: Access is denied.
```
This has now been fixed.
https://github.com/restic/restic/issues/5003
https://github.com/restic/restic/pull/5123
https://forum.restic.net/t/backing-up-a-folder-from-a-veracrypt-volume-brings-up-errors-since-restic-v17-0/8444

View file

@ -0,0 +1,7 @@
Enhancement: Allow prune dry-run without lock
The `prune --dry-run --no-lock` now allows performing a dry-run without
taking a lock. If the repository is modified concurrently, `prune` may
return inaccurate statistics or errors.
https://github.com/restic/restic/pull/5096

View file

@ -0,0 +1,9 @@
Bugfix: Do not retry load/list operation is SFTP connection is broken
When using restic with the SFTP backend, backend operations that load
a file or list files were retried even if the SFTP connection is broken.
This has been fixed now.
https://github.com/restic/restic/pull/5101
https://forum.restic.net/t/restic-hanging-on-backup/8559/2

View file

@ -149,7 +149,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
}
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if gopts.NoLock && !opts.DryRun {
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for prune command")
}
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
if err != nil {
return err
}

View file

@ -2,7 +2,6 @@ package main
import (
"context"
"fmt"
"time"
"github.com/spf13/cobra"
@ -140,7 +139,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
if selectByName(path) {
return node
}
Verbosef(fmt.Sprintf("excluding %s\n", path))
Verbosef("excluding %s\n", path)
return nil
}

View file

@ -132,6 +132,10 @@ options will be deleted. For example, the command
``restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo --delete``
would only delete files within ``/tmp/restore-work/foo``.
When using ``--target / --delete`` then the ``restore`` command only works if either an ``--include``
or ``--exclude`` option is also specified. This ensures that one cannot accidentaly delete
the whole system.
Dry run
-------

View file

@ -191,9 +191,9 @@ Summary is the last output line in a successful backup.
+---------------------------+---------------------------------------------------------+
| ``dirs_unmodified`` | Number of directories that did not change |
+---------------------------+---------------------------------------------------------+
| ``data_blobs`` | Number of data blobs |
| ``data_blobs`` | Number of data blobs added |
+---------------------------+---------------------------------------------------------+
| ``tree_blobs`` | Number of tree blobs |
| ``tree_blobs`` | Number of tree blobs added |
+---------------------------+---------------------------------------------------------+
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
+---------------------------+---------------------------------------------------------+
@ -651,9 +651,9 @@ was created.
+---------------------------+---------------------------------------------------------+
| ``dirs_unmodified`` | Number of directories that did not change |
+---------------------------+---------------------------------------------------------+
| ``data_blobs`` | Number of data blobs |
| ``data_blobs`` | Number of data blobs added |
+---------------------------+---------------------------------------------------------+
| ``tree_blobs`` | Number of tree blobs |
| ``tree_blobs`` | Number of tree blobs added |
+---------------------------+---------------------------------------------------------+
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
+---------------------------+---------------------------------------------------------+

View file

@ -31,7 +31,7 @@ var opts = struct {
var versionRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
func init() {
pflag.BoolVar(&opts.IgnoreBranchName, "ignore-branch-name", false, "allow releasing from other branches as 'master'")
pflag.BoolVar(&opts.IgnoreBranchName, "ignore-branch-name", false, "allow releasing from other branches than 'master'")
pflag.BoolVar(&opts.IgnoreUncommittedChanges, "ignore-uncommitted-changes", false, "allow uncommitted changes")
pflag.BoolVar(&opts.IgnoreChangelogVersion, "ignore-changelog-version", false, "ignore missing entry in CHANGELOG.md")
pflag.BoolVar(&opts.IgnoreChangelogReleaseDate, "ignore-changelog-release-date", false, "ignore missing subdir with date in changelog/")
@ -128,17 +128,22 @@ func uncommittedChanges(dirs ...string) string {
return string(changes)
}
func preCheckBranchMaster() {
if opts.IgnoreBranchName {
return
}
func getBranchName() string {
branch, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
if err != nil {
die("error running 'git': %v", err)
}
if strings.TrimSpace(string(branch)) != "master" {
return strings.TrimSpace(string(branch))
}
func preCheckBranchMaster() {
if opts.IgnoreBranchName {
return
}
branch := getBranchName()
if branch != "master" {
die("wrong branch: %s", branch)
}
}
@ -449,6 +454,7 @@ func main() {
}
preCheckBranchMaster()
branch := getBranchName()
preCheckUncommittedChanges()
preCheckVersionExists()
preCheckDockerBuilderGoVersion()
@ -485,5 +491,5 @@ func main() {
msg("done, output dir is %v", opts.OutputDir)
msg("now run:\n\ngit push --tags origin master\n%s\n\nrm -rf %q", dockerCmds, sourceDir)
msg("now run:\n\ngit push --tags origin %s\n%s\n\nrm -rf %q", branch, dockerCmds, sourceDir)
}

View file

@ -421,6 +421,10 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if err := r.clientError(); err != nil {
return err
}
return util.DefaultLoad(ctx, h, length, offset, r.openReader, func(rd io.Reader) error {
if length == 0 || !feature.Flag.Enabled(feature.BackendErrorRedesign) {
return fn(rd)
@ -490,6 +494,10 @@ func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
if err := r.clientError(); err != nil {
return err
}
basedir, subdirs := r.Basedir(t)
walker := r.c.Walk(basedir)
for {

View file

@ -54,6 +54,15 @@ func GetSecurityDescriptor(filePath string) (securityDescriptor *[]byte, err err
sd, err = getNamedSecurityInfoLow(filePath)
} else {
sd, err = getNamedSecurityInfoHigh(filePath)
// Fallback to the low privilege version when receiving an access denied error.
// For some reason the ERROR_PRIVILEGE_NOT_HELD error is not returned for removable media
// but instead an access denied error is returned. Workaround that by just retrying with
// the low privilege version, but don't switch privileges as we cannot distinguish this
// case from actual access denied errors.
// see https://github.com/restic/restic/issues/5003#issuecomment-2452314191 for details
if err != nil && isAccessDeniedError(err) {
sd, err = getNamedSecurityInfoLow(filePath)
}
}
if err != nil {
if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) {
@ -114,6 +123,10 @@ func SetSecurityDescriptor(filePath string, securityDescriptor *[]byte) error {
err = setNamedSecurityInfoLow(filePath, dacl)
} else {
err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl)
// See corresponding fallback in getSecurityDescriptor for an explanation
if err != nil && isAccessDeniedError(err) {
err = setNamedSecurityInfoLow(filePath, dacl)
}
}
if err != nil {
@ -174,6 +187,15 @@ func isHandlePrivilegeNotHeldError(err error) bool {
return false
}
// isAccessDeniedError checks if the error is ERROR_ACCESS_DENIED
func isAccessDeniedError(err error) bool {
if errno, ok := err.(syscall.Errno); ok {
// Compare the error code to the expected value
return errno == windows.ERROR_ACCESS_DENIED
}
return false
}
// SecurityDescriptorBytesToStruct converts the security descriptor bytes representation
// into a pointer to windows SECURITY_DESCRIPTOR.
func SecurityDescriptorBytesToStruct(sd []byte) (*windows.SECURITY_DESCRIPTOR, error) {

View file

@ -20,29 +20,36 @@ import (
// Statically ensure that *dir implement those interface
var _ = fs.HandleReadDirAller(&dir{})
var _ = fs.NodeForgetter(&dir{})
var _ = fs.NodeGetxattrer(&dir{})
var _ = fs.NodeListxattrer(&dir{})
var _ = fs.NodeStringLookuper(&dir{})
type dir struct {
root *Root
forget forgetFn
items map[string]*restic.Node
inode uint64
parentInode uint64
node *restic.Node
m sync.Mutex
cache treeCache
}
func cleanupNodeName(name string) string {
return filepath.Base(name)
}
func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, error) {
func newDir(root *Root, forget forgetFn, inode, parentInode uint64, node *restic.Node) (*dir, error) {
debug.Log("new dir for %v (%v)", node.Name, node.Subtree)
return &dir{
root: root,
forget: forget,
node: node,
inode: inode,
parentInode: parentInode,
cache: *newTreeCache(),
}, nil
}
@ -75,10 +82,11 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *rest
return tree.Nodes, nil
}
func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
debug.Log("new dir for snapshot %v (%v)", snapshot.ID(), snapshot.Tree)
return &dir{
root: root,
forget: forget,
node: &restic.Node{
AccessTime: snapshot.Time,
ModTime: snapshot.Time,
@ -87,6 +95,7 @@ func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*d
Subtree: snapshot.Tree,
},
inode: inode,
cache: *newTreeCache(),
}, nil
}
@ -208,6 +217,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
return nil, err
}
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
node, ok := d.items[name]
if !ok {
debug.Log(" Lookup(%v) -> not found", name)
@ -216,17 +226,18 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
inode := inodeFromNode(d.inode, node)
switch node.Type {
case "dir":
return newDir(d.root, inode, d.inode, node)
return newDir(d.root, forget, inode, d.inode, node)
case "file":
return newFile(d.root, inode, node)
return newFile(d.root, forget, inode, node)
case "symlink":
return newLink(d.root, inode, node)
return newLink(d.root, forget, inode, node)
case "dev", "chardev", "fifo", "socket":
return newOther(d.root, inode, node)
return newOther(d.root, forget, inode, node)
default:
debug.Log(" node %v has unknown type %v", name, node.Type)
return nil, syscall.ENOENT
}
})
}
func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
@ -237,3 +248,7 @@ func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fus
func (d *dir) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return nodeGetXattr(d.node, req, resp)
}
func (d *dir) Forget() {
d.forget()
}

View file

@ -20,12 +20,14 @@ const blockSize = 512
// Statically ensure that *file and *openFile implement the given interfaces
var _ = fs.HandleReader(&openFile{})
var _ = fs.NodeListxattrer(&file{})
var _ = fs.NodeForgetter(&file{})
var _ = fs.NodeGetxattrer(&file{})
var _ = fs.NodeListxattrer(&file{})
var _ = fs.NodeOpener(&file{})
type file struct {
root *Root
forget forgetFn
node *restic.Node
inode uint64
}
@ -36,10 +38,11 @@ type openFile struct {
cumsize []uint64
}
func newFile(root *Root, inode uint64, node *restic.Node) (fusefile *file, err error) {
func newFile(root *Root, forget forgetFn, inode uint64, node *restic.Node) (fusefile *file, err error) {
debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
return &file{
inode: inode,
forget: forget,
root: root,
node: node,
}, nil
@ -172,3 +175,7 @@ func (f *file) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
func (f *file) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return nodeGetXattr(f.node, req, resp)
}
func (f *file) Forget() {
f.forget()
}

View file

@ -119,7 +119,7 @@ func TestFuseFile(t *testing.T) {
root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
inode := inodeFromNode(1, node)
f, err := newFile(root, inode, node)
f, err := newFile(root, func() {}, inode, node)
rtest.OK(t, err)
of, err := f.Open(context.TODO(), nil, nil)
rtest.OK(t, err)
@ -162,7 +162,7 @@ func TestFuseDir(t *testing.T) {
}
parentInode := inodeFromName(0, "parent")
inode := inodeFromName(1, "foo")
d, err := newDir(root, inode, parentInode, node)
d, err := newDir(root, func() {}, inode, parentInode, node)
rtest.OK(t, err)
// don't open the directory as that would require setting up a proper tree blob
@ -217,6 +217,34 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui
rtest.Equals(t, uint32(0), attr.Gid)
}
// The Lookup method must return the same Node object unless it was forgotten in the meantime
func testStableLookup(t *testing.T, node fs.Node, path string) fs.Node {
t.Helper()
result, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), path)
rtest.OK(t, err)
result2, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), path)
rtest.OK(t, err)
rtest.Assert(t, result == result2, "%v are not the same object", path)
result2.(fs.NodeForgetter).Forget()
result2, err = node.(fs.NodeStringLookuper).Lookup(context.TODO(), path)
rtest.OK(t, err)
rtest.Assert(t, result != result2, "object for %v should change after forget", path)
return result
}
func TestStableNodeObjects(t *testing.T) {
repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 2)
root := NewRoot(repo, Config{})
idsdir := testStableLookup(t, root, "ids")
snapID := loadFirstSnapshot(t, repo).ID().Str()
snapshotdir := testStableLookup(t, idsdir, snapID)
dir := testStableLookup(t, snapshotdir, "dir-0")
testStableLookup(t, dir, "file-2")
}
// Test reporting of fuse.Attr.Blocks in multiples of 512.
func TestBlocks(t *testing.T) {
root := &Root{}
@ -276,7 +304,7 @@ func TestLink(t *testing.T) {
{Name: "foo", Value: []byte("bar")},
}}
lnk, err := newLink(&Root{}, 42, node)
lnk, err := newLink(&Root{}, func() {}, 42, node)
rtest.OK(t, err)
target, err := lnk.Readlink(context.TODO(), nil)
rtest.OK(t, err)

View file

@ -12,16 +12,20 @@ import (
)
// Statically ensure that *link implements the given interface
var _ = fs.NodeForgetter(&link{})
var _ = fs.NodeGetxattrer(&link{})
var _ = fs.NodeListxattrer(&link{})
var _ = fs.NodeReadlinker(&link{})
type link struct {
root *Root
forget forgetFn
node *restic.Node
inode uint64
}
func newLink(root *Root, inode uint64, node *restic.Node) (*link, error) {
return &link{root: root, inode: inode, node: node}, nil
func newLink(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*link, error) {
return &link{root: root, forget: forget, inode: inode, node: node}, nil
}
func (l *link) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@ -55,3 +59,7 @@ func (l *link) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
func (l *link) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return nodeGetXattr(l.node, req, resp)
}
func (l *link) Forget() {
l.forget()
}

View file

@ -7,17 +7,23 @@ import (
"context"
"github.com/anacrolix/fuse"
"github.com/anacrolix/fuse/fs"
"github.com/restic/restic/internal/restic"
)
// Statically ensure that *other implements the given interface
var _ = fs.NodeForgetter(&other{})
var _ = fs.NodeReadlinker(&other{})
type other struct {
root *Root
forget forgetFn
node *restic.Node
inode uint64
}
func newOther(root *Root, inode uint64, node *restic.Node) (*other, error) {
return &other{root: root, inode: inode, node: node}, nil
func newOther(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*other, error) {
return &other{root: root, forget: forget, inode: inode, node: node}, nil
}
func (l *other) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@ -40,3 +46,7 @@ func (l *other) Attr(_ context.Context, a *fuse.Attr) error {
return nil
}
func (l *other) Forget() {
l.forget()
}

View file

@ -66,7 +66,7 @@ func NewRoot(repo restic.Repository, cfg Config) *Root {
}
}
root.SnapshotsDir = NewSnapshotsDir(root, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
root.SnapshotsDir = NewSnapshotsDir(root, func() {}, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
return root
}

View file

@ -19,25 +19,30 @@ import (
// It uses the saved prefix to select the corresponding MetaDirData.
type SnapshotsDir struct {
root *Root
forget forgetFn
inode uint64
parentInode uint64
dirStruct *SnapshotsDirStructure
prefix string
cache treeCache
}
// ensure that *SnapshotsDir implements these interfaces
var _ = fs.HandleReadDirAller(&SnapshotsDir{})
var _ = fs.NodeForgetter(&SnapshotsDir{})
var _ = fs.NodeStringLookuper(&SnapshotsDir{})
// NewSnapshotsDir returns a new directory structure containing snapshots and "latest" links
func NewSnapshotsDir(root *Root, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
func NewSnapshotsDir(root *Root, forget forgetFn, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
debug.Log("create snapshots dir, inode %d", inode)
return &SnapshotsDir{
root: root,
forget: forget,
inode: inode,
parentInode: parentInode,
dirStruct: dirStruct,
prefix: prefix,
cache: *newTreeCache(),
}
}
@ -107,33 +112,41 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
return nil, syscall.ENOENT
}
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
entry := meta.names[name]
if entry != nil {
inode := inodeFromName(d.inode, name)
if entry.linkTarget != "" {
return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
} else if entry.snapshot != nil {
return newDirFromSnapshot(d.root, inode, entry.snapshot)
}
return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
if entry == nil {
return nil, syscall.ENOENT
}
return nil, syscall.ENOENT
inode := inodeFromName(d.inode, name)
if entry.linkTarget != "" {
return newSnapshotLink(d.root, forget, inode, entry.linkTarget, entry.snapshot)
} else if entry.snapshot != nil {
return newDirFromSnapshot(d.root, forget, inode, entry.snapshot)
}
return NewSnapshotsDir(d.root, forget, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
})
}
func (d *SnapshotsDir) Forget() {
d.forget()
}
// SnapshotLink
type snapshotLink struct {
root *Root
forget forgetFn
inode uint64
target string
snapshot *restic.Snapshot
}
var _ = fs.NodeForgetter(&snapshotLink{})
var _ = fs.NodeReadlinker(&snapshotLink{})
// newSnapshotLink
func newSnapshotLink(root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
func newSnapshotLink(root *Root, forget forgetFn, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
return &snapshotLink{root: root, forget: forget, inode: inode, target: target, snapshot: snapshot}, nil
}
// Readlink
@ -157,3 +170,7 @@ func (l *snapshotLink) Attr(_ context.Context, a *fuse.Attr) error {
return nil
}
func (l *snapshotLink) Forget() {
l.forget()
}

View file

@ -0,0 +1,45 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package fuse
import (
"sync"
"github.com/anacrolix/fuse/fs"
)
type treeCache struct {
nodes map[string]fs.Node
m sync.Mutex
}
type forgetFn func()
func newTreeCache() *treeCache {
return &treeCache{
nodes: map[string]fs.Node{},
}
}
func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
t.m.Lock()
defer t.m.Unlock()
if node, ok := t.nodes[name]; ok {
return node, nil
}
node, err := create(func() {
t.m.Lock()
defer t.m.Unlock()
delete(t.nodes, name)
})
if err != nil {
return nil, err
}
t.nodes[name] = node
return node, nil
}