From 94de87d4b7d6b12ffbe3726954ed4b02d4360a3a Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:57:00 -0700 Subject: [PATCH 1/7] Add CombineErrors helper function --- internal/errors/errors.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 0327ea0da..3c669f861 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -2,6 +2,7 @@ package errors import ( stderrors "errors" + "fmt" "github.com/pkg/errors" ) @@ -22,12 +23,42 @@ var Wrap = errors.Wrap // nil, Wrapf returns nil. var Wrapf = errors.Wrapf +// WithStack annotates err with a stack trace at the point WithStack was called. +// If err is nil, WithStack returns nil. var WithStack = errors.WithStack // Go 1.13-style error handling. +// As finds the first error in err's tree that matches target, and if one is found, +// sets target to that error value and returns true. Otherwise, it returns false. func As(err error, tgt interface{}) bool { return stderrors.As(err, tgt) } +// Is reports whether any error in err's tree matches target. func Is(x, y error) bool { return stderrors.Is(x, y) } +// Unwrap returns the result of calling the Unwrap method on err, if err's type contains +// an Unwrap method returning error. Otherwise, Unwrap returns nil. +// +// Unwrap only calls a method of the form "Unwrap() error". In particular Unwrap does not +// unwrap errors returned by [Join]. func Unwrap(err error) error { return stderrors.Unwrap(err) } + +// CombineErrors combines multiple errors into a single error. +func CombineErrors(errors ...error) error { + var combinedErrorMsg string + + for _, err := range errors { + if err != nil { + if combinedErrorMsg != "" { + combinedErrorMsg += "; " // Separate error messages with a delimiter + } + combinedErrorMsg += err.Error() + } + } + + if combinedErrorMsg == "" { + return nil // No errors, return nil + } + + return fmt.Errorf("multiple errors occurred: [%s]", combinedErrorMsg) +} From 62a8a599f1f4f36c55e17106a8e40bd956430058 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:58:12 -0700 Subject: [PATCH 2/7] Add optional messages for Equals helper --- internal/test/helpers.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/test/helpers.go b/internal/test/helpers.go index 65e3e36ec..242da6079 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -3,6 +3,7 @@ package test import ( "compress/bzip2" "compress/gzip" + "fmt" "io" "os" "os/exec" @@ -47,10 +48,22 @@ func OKs(tb testing.TB, errs []error) { } // Equals fails the test if exp is not equal to act. -func Equals(tb testing.TB, exp, act interface{}) { +// msg is optional message to be printed, first param being format string and rest being arguments. +func Equals(tb testing.TB, exp, act interface{}, msgs ...string) { tb.Helper() if !reflect.DeepEqual(exp, act) { - tb.Fatalf("\033[31m\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) + var msgString string + length := len(msgs) + if length == 1 { + msgString = msgs[0] + } else if length > 1 { + args := make([]interface{}, length-1) + for i, msg := range msgs[1:] { + args[i] = msg + } + msgString = fmt.Sprintf(msgs[0], args...) + } + tb.Fatalf("\033[31m\n\n\t"+msgString+"\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) } } From 0962917974952531347f82dffc0d6e82b1edf5f5 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:31:20 -0700 Subject: [PATCH 3/7] Support windows metadata using generic attribs Add new generic_attributes attribute in Node. Use the generic attributes to add support for creation time and file attributes like hidden, readonly, encrypted in windows. Handle permission errors for readonly files in windows. Handle backup and restore of encrypted attributes using windows system calls. --- cmd/restic/cmd_find.go | 1 + internal/fs/file.go | 14 ++ internal/fs/file_windows.go | 26 ++++ internal/restic/node.go | 247 +++++++++++++++++++++++++++++-- internal/restic/node_aix.go | 17 ++- internal/restic/node_netbsd.go | 24 ++- internal/restic/node_openbsd.go | 24 ++- internal/restic/node_test.go | 35 +++-- internal/restic/node_windows.go | 224 +++++++++++++++++++++++++++- internal/restic/node_xattr.go | 11 ++ internal/restorer/fileswriter.go | 28 ++-- 11 files changed, 591 insertions(+), 60 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 04e6ae3dd..7ea7c425a 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -126,6 +126,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { // Make the following attributes disappear Name byte `json:"name,omitempty"` ExtendedAttributes byte `json:"extended_attributes,omitempty"` + GenericAttributes byte `json:"generic_attributes,omitempty"` Device byte `json:"device,omitempty"` Content byte `json:"content,omitempty"` Subtree byte `json:"subtree,omitempty"` diff --git a/internal/fs/file.go b/internal/fs/file.go index f35901c06..4a236ea09 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -124,3 +124,17 @@ func RemoveIfExists(filename string) error { func Chtimes(name string, atime time.Time, mtime time.Time) error { return os.Chtimes(fixpath(name), atime, mtime) } + +// IsAccessDenied checks if the error is due to permission error. +func IsAccessDenied(err error) bool { + return os.IsPermission(err) +} + +// ResetPermissions resets the permissions of the file at the specified path +func ResetPermissions(path string) error { + // Set the default file permissions + if err := os.Chmod(path, 0600); err != nil { + return err + } + return nil +} diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index d19a744e1..2f0969804 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -77,3 +77,29 @@ func TempFile(dir, prefix string) (f *os.File, err error) { func Chmod(name string, mode os.FileMode) error { return os.Chmod(fixpath(name), mode) } + +// ClearSystem removes the system attribute from the file. +func ClearSystem(path string) error { + return ClearAttribute(path, windows.FILE_ATTRIBUTE_SYSTEM) +} + +// ClearAttribute removes the specified attribute from the file. +func ClearAttribute(path string, attribute uint32) error { + ptr, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + fileAttributes, err := windows.GetFileAttributes(ptr) + if err != nil { + return err + } + if fileAttributes&attribute != 0 { + // Clear the attribute + fileAttributes &= ^uint32(attribute) + err = windows.SetFileAttributes(ptr, fileAttributes) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/restic/node.go b/internal/restic/node.go index 1d5bb51af..44ca52b0c 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "os/user" + "reflect" "strconv" + "strings" "sync" "syscall" "time" @@ -20,12 +22,53 @@ import ( "github.com/restic/restic/internal/fs" ) -// ExtendedAttribute is a tuple storing the xattr name and value. +// ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. type ExtendedAttribute struct { Name string `json:"name"` Value []byte `json:"value"` } +// GenericAttributeType can be used for OS specific functionalities by defining specific types +// in node.go to be used by the specific node_xx files. +// OS specific attribute types should follow the convention Attributes. +// GenericAttributeTypes should follow the convention . +// The attributes in OS specific attribute types must be pointers as we want to distinguish nil values +// and not create GenericAttributes for them. +type GenericAttributeType string + +// OSType is the type created to represent each specific OS +type OSType string + +const ( + // When new GenericAttributeType are defined, they must be added in the init function as well. + + // Below are windows specific attributes. + + // TypeCreationTime is the GenericAttributeType used for storing creation time for windows files within the generic attributes map. + TypeCreationTime GenericAttributeType = "windows.creation_time" + // TypeFileAttributes is the GenericAttributeType used for storing file attributes for windows files within the generic attributes map. + TypeFileAttributes GenericAttributeType = "windows.file_attributes" + + // Generic Attributes for other OS types should be defined here. +) + +// init is called when the package is initialized. Any new GenericAttributeTypes being created must be added here as well. +func init() { + storeGenericAttributeType(TypeCreationTime, TypeFileAttributes) +} + +// genericAttributesForOS maintains a map of known genericAttributesForOS to the OSType +var genericAttributesForOS = map[GenericAttributeType]OSType{} + +// storeGenericAttributeType adds and entry in genericAttributesForOS map +func storeGenericAttributeType(attributeTypes ...GenericAttributeType) { + for _, attributeType := range attributeTypes { + // Get the OS attribute type from the GenericAttributeType + osAttributeName := strings.Split(string(attributeType), ".")[0] + genericAttributesForOS[attributeType] = OSType(osAttributeName) + } +} + // Node is a file, directory or other item in a backup. type Node struct { Name string `json:"name"` @@ -47,11 +90,12 @@ type Node struct { // This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems, // as LinkTarget without breaking backwards-compatibility. // Must only be set of the linktarget cannot be encoded as valid utf8. - LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` - ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` - Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev - Content IDs `json:"content"` - Subtree *ID `json:"subtree,omitempty"` + LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` + ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` + GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"` + Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev + Content IDs `json:"content"` + Subtree *ID `json:"subtree,omitempty"` Error string `json:"error,omitempty"` @@ -203,14 +247,6 @@ func (node Node) restoreMetadata(path string) error { } } - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr != nil { - firsterr = errors.WithStack(err) - } - } - } - if err := node.RestoreTimestamps(path); err != nil { debug.Log("error restoring timestamps for dir %v: %v", path, err) if firsterr != nil { @@ -225,6 +261,24 @@ func (node Node) restoreMetadata(path string) error { } } + if err := node.restoreGenericAttributes(path); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr != nil { + firsterr = err + } + } + + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := fs.Chmod(path, node.Mode); err != nil { + if firsterr != nil { + firsterr = errors.WithStack(err) + } + } + } + return firsterr } @@ -438,6 +492,9 @@ func (node Node) Equals(other Node) bool { if !node.sameExtendedAttributes(other) { return false } + if !node.sameGenericAttributes(other) { + return false + } if node.Subtree != nil { if other.Subtree == nil { return false @@ -480,8 +537,13 @@ func (node Node) sameContent(other Node) bool { } func (node Node) sameExtendedAttributes(other Node) bool { - if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) { + ln := len(node.ExtendedAttributes) + lo := len(other.ExtendedAttributes) + if ln != lo { return false + } else if ln == 0 { + // This means lo is also of length 0 + return true } // build a set of all attributes that node has @@ -525,6 +587,33 @@ func (node Node) sameExtendedAttributes(other Node) bool { return true } +func (node Node) sameGenericAttributes(other Node) bool { + return deepEqual(node.GenericAttributes, other.GenericAttributes) +} + +func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { + // Check if the maps have the same number of keys + if len(map1) != len(map2) { + return false + } + + // Iterate over each key-value pair in map1 + for key, value1 := range map1 { + // Check if the key exists in map2 + value2, ok := map2[key] + if !ok { + return false + } + + // Check if the JSON.RawMessage values are equal byte by byte + if !bytes.Equal(value1, value2) { + return false + } + } + + return true +} + func (node *Node) fillUser(stat *statT) { uid, gid := stat.uid(), stat.gid() node.UID, node.GID = uid, gid @@ -627,7 +716,17 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { return errors.Errorf("unsupported file type %q", node.Type) } - return node.fillExtendedAttributes(path) + allowExtended, err := node.fillGenericAttributes(path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + errEx := node.fillExtendedAttributes(path) + if err == nil { + err = errEx + } else { + debug.Log("Error filling extended attributes for %v at %v : %v", node.Name, path, errEx) + } + } + return err } func (node *Node) fillExtendedAttributes(path string) error { @@ -665,3 +764,119 @@ func (node *Node) fillTimes(stat *statT) { node.ChangeTime = time.Unix(ctim.Unix()) node.AccessTime = time.Unix(atim.Unix()) } + +// HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType) { + for _, unknownAttrib := range unknownAttribs { + handleUnknownGenericAttributeFound(unknownAttrib) + } +} + +// handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType) { + if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) { + // Print the unique error only once for a given execution + os, exists := genericAttributesForOS[genericAttributeType] + + if exists { + // If genericAttributesForOS contains an entry but we still got here, it means the specific node_xx.go for the current OS did not handle it and the repository may have been originally created on a different OS. + // The fact that node.go knows about the attribute, means it is not a new attribute. This may be a common situation if a repo is used across OSs. + debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os) + } else { + // If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType. + debug.Log("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType) + } + } +} + +// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. +// This is not used on windows currently because windows has handling for generic attributes. +// nolint:unused +func (node Node) handleAllUnknownGenericAttributesFound() error { + for name := range node.GenericAttributes { + handleUnknownGenericAttributeFound(name) + } + return nil +} + +var unknownGenericAttributesHandlingHistory sync.Map + +// checkGenericAttributeNameNotHandledAndPut checks if the GenericAttributeType name entry +// already exists and puts it in the map if not. +func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool { + // If Key doesn't exist, put the value and return true because it is not already handled + _, exists := unknownGenericAttributesHandlingHistory.LoadOrStore(value, "") + // Key exists, then it is already handled so return false + return !exists +} + +// The functions below are common helper functions which can be used for generic attributes support +// across different OS. + +// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection +// nolint:unused +func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { + attributeValue := *attributeValuePtr + + for key, rawMsg := range attrs { + found := false + for i := 0; i < attributeType.NumField(); i++ { + if getFQKeyByIndex(attributeType, i, keyPrefix) == key { + found = true + fieldValue := attributeValue.Field(i) + // For directly supported types, use json.Unmarshal directly + if err := json.Unmarshal(rawMsg, fieldValue.Addr().Interface()); err != nil { + return unknownAttribs, errors.Wrap(err, "Unmarshal") + } + break + } + } + if !found { + unknownAttribs = append(unknownAttribs, key) + } + } + return unknownAttribs, nil +} + +// getFQKey gets the fully qualified key for the field +// nolint:unused +func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType { + return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic"))) +} + +// getFQKeyByIndex gets the fully qualified key for the field index +// nolint:unused +func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType { + return getFQKey(attributeType.Field(index), keyPrefix) +} + +// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection +// nolint:unused +func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { + attributeValue := *attributeValuePtr + attrs = make(map[GenericAttributeType]json.RawMessage) + + // Iterate over the fields of the struct + for i := 0; i < attributeType.NumField(); i++ { + field := attributeType.Field(i) + + // Get the field value using reflection + fieldValue := attributeValue.FieldByName(field.Name) + + // Check if the field is nil + if fieldValue.IsNil() { + // If it's nil, skip this field + continue + } + + // Marshal the field value into a json.RawMessage + var fieldBytes []byte + if fieldBytes, err = json.Marshal(fieldValue.Interface()); err != nil { + return attrs, errors.Wrap(err, "Marshal") + } + + // Insert the field into the map + attrs[getFQKey(field, keyPrefix)] = json.RawMessage(fieldBytes) + } + return attrs, nil +} diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 572e33a65..4d8c248de 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -3,9 +3,12 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -34,3 +37,13 @@ func Listxattr(path string) ([]string, error) { func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on AIX. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op on AIX. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index 0eade2f37..be4afa3ae 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec } func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } -// Getxattr retrieves extended attribute data associated with path. +// Getxattr is a no-op on netbsd. func Getxattr(path, name string) ([]byte, error) { return nil, nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. +// Listxattr is a no-op on netbsd. func Listxattr(path string) ([]string, error) { return nil, nil } -// Setxattr associates name and data together as an attribute of path. +// Setxattr is a no-op on netbsd. func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on netbsd. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op on netbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index a4ccc7211..bfff8f8aa 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atim } func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } -// Getxattr retrieves extended attribute data associated with path. +// Getxattr is a no-op on openbsd. func Getxattr(path, name string) ([]byte, error) { return nil, nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. +// Listxattr is a no-op on openbsd. func Listxattr(path string) ([]string, error) { return nil, nil } -// Setxattr associates name and data together as an attribute of path. +// Setxattr is a no-op on openbsd. func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on openbsd. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op on openbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index aae010421..c2c7306b7 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -1,4 +1,4 @@ -package restic_test +package restic import ( "context" @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -32,7 +31,7 @@ func BenchmarkNodeFillUser(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi) rtest.OK(t, err) } @@ -56,7 +55,7 @@ func BenchmarkNodeFromFileInfo(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi) if err != nil { t.Fatal(err) } @@ -75,11 +74,11 @@ func parseTime(s string) time.Time { return t.Local() } -var nodeTests = []restic.Node{ +var nodeTests = []Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -90,7 +89,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetuid, @@ -101,7 +100,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile2", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetgid, @@ -112,7 +111,7 @@ var nodeTests = []restic.Node{ { Name: "testSticky", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSticky, @@ -148,7 +147,7 @@ var nodeTests = []restic.Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -170,14 +169,14 @@ var nodeTests = []restic.Node{ { Name: "testXattrFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -191,7 +190,7 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -228,7 +227,7 @@ func TestNodeRestoreAt(t *testing.T) { fi, err := os.Lstat(nodePath) rtest.OK(t, err) - n2, err := restic.NodeFromFileInfo(nodePath, fi) + n2, err := NodeFromFileInfo(nodePath, fi) rtest.OK(t, err) rtest.Assert(t, test.Name == n2.Name, @@ -330,7 +329,7 @@ func TestFixTime(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - res := restic.FixTime(test.src) + res := FixTime(test.src) if !res.Equal(test.want) { t.Fatalf("wrong result for %v, want:\n %v\ngot:\n %v", test.src, test.want, res) } @@ -343,12 +342,12 @@ func TestSymlinkSerialization(t *testing.T) { "válîd \t Üñi¢òde \n śẗŕinǵ", string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc}), } { - n := restic.Node{ + n := Node{ LinkTarget: link, } ser, err := json.Marshal(n) test.OK(t, err) - var n2 restic.Node + var n2 Node err = json.Unmarshal(ser, &n2) test.OK(t, err) fmt.Println(string(ser)) @@ -365,7 +364,7 @@ func TestSymlinkSerializationFormat(t *testing.T) { {`{"linktarget":"test"}`, "test"}, {`{"linktarget":"\u0000\u0001\u0002\ufffd\ufffd\ufffd","linktarget_raw":"AAEC+vv8"}`, string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc})}, } { - var n2 restic.Node + var n2 Node err := json.Unmarshal([]byte(d.ser), &n2) test.OK(t, err) test.Equals(t, d.linkTarget, n2.LinkTarget) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index fc6439b40..a2b8c75e5 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -1,21 +1,47 @@ package restic import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" "syscall" + "unsafe" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "golang.org/x/sys/windows" +) + +// WindowsAttributes are the genericAttributes for Windows OS +type WindowsAttributes struct { + // CreationTime is used for storing creation time for windows files. + CreationTime *syscall.Filetime `generic:"creation_time"` + // FileAttributes is used for storing file attributes for windows files. + FileAttributes *uint32 `generic:"file_attributes"` +} + +var ( + modAdvapi32 = syscall.NewLazyDLL("advapi32.dll") + procEncryptFile = modAdvapi32.NewProc("EncryptFileW") + procDecryptFile = modAdvapi32.NewProc("DecryptFileW") ) // mknod is not supported on Windows. -func mknod(path string, mode uint32, dev uint64) (err error) { +func mknod(_ string, mode uint32, dev uint64) (err error) { return errors.New("device nodes cannot be created on windows") } // Windows doesn't need lchown -func lchown(path string, uid int, gid int) (err error) { +func lchown(_ string, uid int, gid int) (err error) { return nil } +// restoreSymlinkTimestamps restores timestamps for symlinks func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(path) @@ -28,7 +54,14 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe if e != nil { return e } - defer syscall.Close(h) + + defer func() { + err := syscall.Close(h) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + a := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[0])) w := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[1])) return syscall.SetFileTime(h, nil, &a, &w) @@ -83,3 +116,188 @@ func (s statT) ctim() syscall.Timespec { // Windows does not have the concept of a "change time" in the sense Unix uses it, so we're using the LastWriteTime here. return syscall.NsecToTimespec(s.LastWriteTime.Nanoseconds()) } + +// restoreGenericAttributes restores generic attributes for Windows +func (node Node) restoreGenericAttributes(path string) (err error) { + if len(node.GenericAttributes) == 0 { + return nil + } + var errs []error + windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + if err != nil { + return fmt.Errorf("error parsing generic attribute for: %s : %v", path, err) + } + if windowsAttributes.CreationTime != nil { + if err := restoreCreationTime(path, windowsAttributes.CreationTime); err != nil { + errs = append(errs, fmt.Errorf("error restoring creation time for: %s : %v", path, err)) + } + } + if windowsAttributes.FileAttributes != nil { + if err := restoreFileAttributes(path, windowsAttributes.FileAttributes); err != nil { + errs = append(errs, fmt.Errorf("error restoring file attributes for: %s : %v", path, err)) + } + } + + HandleUnknownGenericAttributesFound(unknownAttribs) + return errors.CombineErrors(errs...) +} + +// genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unkown attributes that it could not convert. +func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { + waValue := reflect.ValueOf(&windowsAttributes).Elem() + unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + return windowsAttributes, unknownAttribs, err +} + +// restoreCreationTime gets the creation time from the data and sets it to the file/folder at +// the specified path. +func restoreCreationTime(path string, creationTime *syscall.Filetime) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + handle, err := syscall.CreateFile(pathPointer, + syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil, + syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return err + } + defer func() { + if err := syscall.Close(handle); err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + return syscall.SetFileTime(handle, creationTime, nil, nil) +} + +// restoreFileAttributes gets the File Attributes from the data and sets them to the file/folder +// at the specified path. +func restoreFileAttributes(path string, fileAttributes *uint32) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + err = fixEncryptionAttribute(path, fileAttributes, pathPointer) + if err != nil { + debug.Log("Could not change encryption attribute for path: %s: %v", path, err) + } + return syscall.SetFileAttributes(pathPointer, *fileAttributes) +} + +// fixEncryptionAttribute checks if a file needs to be marked encrypted and is not already encrypted, it sets +// the FILE_ATTRIBUTE_ENCRYPTED. Conversely, if the file needs to be marked unencrypted and it is already +// marked encrypted, it removes the FILE_ATTRIBUTE_ENCRYPTED. +func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (err error) { + if *attrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should be encrypted. + err = encryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) { + // If existing file already has readonly or system flag, encrypt file call fails. + // We have already cleared readonly flag, clearing system flag if needed. + // The readonly and system flags will be set again at the end of this func if they are needed. + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) + } + err = encryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } + } else { + existingAttrs, err := windows.GetFileAttributes(pathPointer) + if err != nil { + return fmt.Errorf("failed to get file attributes for existing file: %s : %v", path, err) + } + if existingAttrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should not be encrypted, but its already encrypted. Decrypt it. + err = decryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) { + // If existing file already has readonly or system flag, decrypt file call fails. + // We have already cleared readonly flag, clearing system flag if needed. + // The readonly and system flags will be set again after this func if they are needed. + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) + } + err = decryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } + } + } + return err +} + +// encryptFile set the encrypted flag on the file. +func encryptFile(pathPointer *uint16) error { + // Call EncryptFile function + ret, _, err := procEncryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// decryptFile removes the encrypted flag from the file. +func decryptFile(pathPointer *uint16) error { + // Call DecryptFile function + ret, _, err := procDecryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// fillGenericAttributes fills in the generic attributes for windows like File Attributes, +// Created time etc. +func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { + if strings.Contains(filepath.Base(path), ":") { + //Do not process for Alternate Data Streams in Windows + // Also do not allow processing of extended attributes for ADS. + return false, nil + } + if !strings.HasSuffix(filepath.Clean(path), `\`) { + // Do not process file attributes and created time for windows directories like + // C:, D: + // Filepath.Clean(path) ends with '\' for Windows root drives only. + + // Add Windows attributes + node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{ + CreationTime: getCreationTime(fi, path), + FileAttributes: &stat.FileAttributes, + }) + } + return true, err +} + +// windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { + // Get the value of the WindowsAttributes + windowsAttributesValue := reflect.ValueOf(windowsAttributes) + return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) +} + +// getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. +// The value is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (UTC) +// split into two 32-bit parts: the low-order DWORD and the high-order DWORD for efficiency and interoperability. +// The low-order DWORD represents the number of 100-nanosecond intervals elapsed since January 1, 1601, modulo +// 2^32. The high-order DWORD represents the number of times the low-order DWORD has overflowed. +func getCreationTime(fi os.FileInfo, path string) (creationTimeAttribute *syscall.Filetime) { + attrib, success := fi.Sys().(*syscall.Win32FileAttributeData) + if success && attrib != nil { + return &attrib.CreationTime + } else { + debug.Log("Could not get create time for path: %s", path) + return nil + } +} diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index ea9eafe94..826b8b74a 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -4,6 +4,7 @@ package restic import ( + "os" "syscall" "github.com/restic/restic/internal/errors" @@ -47,3 +48,13 @@ func handleXattrErr(err error) error { return errors.WithStack(e) } } + +// restoreGenericAttributes is no-op. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 589aa502a..cbe89c30c 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -50,16 +50,26 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.files[path].users++ return wr, nil } - - var flags int + var f *os.File + var err error if createSize >= 0 { - flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY - } else { - flags = os.O_WRONLY - } - - f, err := os.OpenFile(path, flags, 0600) - if err != nil { + if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(path); err != nil { + return nil, err + } + if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + return nil, err + } + } else { + return nil, err + } + } + } else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil { return nil, err } From eeb1aa5388693c8e6b693cb3941045588080ee69 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:52:26 -0700 Subject: [PATCH 4/7] Add ability to report warnings to terminal Report warnings to terminal when unrecognized generic attributes are found in the repository. --- cmd/restic/cmd_restore.go | 3 +++ internal/restic/node.go | 20 ++++++++++---------- internal/restic/node_aix.go | 4 ++-- internal/restic/node_netbsd.go | 4 ++-- internal/restic/node_openbsd.go | 4 ++-- internal/restic/node_test.go | 2 +- internal/restic/node_windows.go | 4 ++-- internal/restic/node_xattr.go | 4 ++-- internal/restorer/restorer.go | 3 ++- 9 files changed, 26 insertions(+), 22 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 37d304672..58f257541 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -178,6 +178,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, totalErrors++ return nil } + res.Warn = func(message string) { + msg.E("Warning: %s\n", message) + } excludePatterns := filter.ParsePatterns(opts.Exclude) insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) diff --git a/internal/restic/node.go b/internal/restic/node.go index 44ca52b0c..cbe9ef363 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -224,8 +224,8 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) er } // RestoreMetadata restores node metadata -func (node Node) RestoreMetadata(path string) error { - err := node.restoreMetadata(path) +func (node Node) RestoreMetadata(path string, warn func(msg string)) error { + err := node.restoreMetadata(path, warn) if err != nil { debug.Log("restoreMetadata(%s) error %v", path, err) } @@ -233,7 +233,7 @@ func (node Node) RestoreMetadata(path string) error { return err } -func (node Node) restoreMetadata(path string) error { +func (node Node) restoreMetadata(path string, warn func(msg string)) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { @@ -261,7 +261,7 @@ func (node Node) restoreMetadata(path string) error { } } - if err := node.restoreGenericAttributes(path); err != nil { + if err := node.restoreGenericAttributes(path, warn); err != nil { debug.Log("error restoring generic attributes for %v: %v", path, err) if firsterr != nil { firsterr = err @@ -766,14 +766,14 @@ func (node *Node) fillTimes(stat *statT) { } // HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories -func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType) { +func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { for _, unknownAttrib := range unknownAttribs { - handleUnknownGenericAttributeFound(unknownAttrib) + handleUnknownGenericAttributeFound(unknownAttrib, warn) } } // handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories -func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType) { +func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType, warn func(msg string)) { if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) { // Print the unique error only once for a given execution os, exists := genericAttributesForOS[genericAttributeType] @@ -784,7 +784,7 @@ func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeTyp debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os) } else { // If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType. - debug.Log("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType) + warn(fmt.Sprintf("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType)) } } } @@ -792,9 +792,9 @@ func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeTyp // handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. // This is not used on windows currently because windows has handling for generic attributes. // nolint:unused -func (node Node) handleAllUnknownGenericAttributesFound() error { +func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error { for name := range node.GenericAttributes { - handleUnknownGenericAttributeFound(name) + handleUnknownGenericAttributeFound(name, warn) } return nil } diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 4d8c248de..def46bd60 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -39,8 +39,8 @@ func Setxattr(path, name string, data []byte) error { } // restoreGenericAttributes is no-op on AIX. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op on AIX. diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index be4afa3ae..1a47299be 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -29,8 +29,8 @@ func Setxattr(path, name string, data []byte) error { } // restoreGenericAttributes is no-op on netbsd. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op on netbsd. diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index bfff8f8aa..e60eb9dc8 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -29,8 +29,8 @@ func Setxattr(path, name string, data []byte) error { } // restoreGenericAttributes is no-op on openbsd. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op on openbsd. diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index c2c7306b7..d9fa02ac8 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -218,7 +218,7 @@ func TestNodeRestoreAt(t *testing.T) { nodePath = filepath.Join(tempdir, test.Name) } rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath)) + rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) if test.Type == "dir" { rtest.OK(t, test.RestoreTimestamps(nodePath)) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index a2b8c75e5..5875c3ccd 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -118,7 +118,7 @@ func (s statT) ctim() syscall.Timespec { } // restoreGenericAttributes restores generic attributes for Windows -func (node Node) restoreGenericAttributes(path string) (err error) { +func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -138,7 +138,7 @@ func (node Node) restoreGenericAttributes(path string) (err error) { } } - HandleUnknownGenericAttributesFound(unknownAttribs) + HandleUnknownGenericAttributesFound(unknownAttribs, warn) return errors.CombineErrors(errs...) } diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index 826b8b74a..0b2d5d552 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -50,8 +50,8 @@ func handleXattrErr(err error) error { } // restoreGenericAttributes is no-op. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op. diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 3f4fb32e3..0aeb636d0 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -24,6 +24,7 @@ type Restorer struct { progress *restoreui.Progress Error func(location string, err error) error + Warn func(message string) SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) } @@ -178,7 +179,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := node.RestoreMetadata(target) + err := node.RestoreMetadata(target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } From d4be734c73565f822a4965c835bf302330631464 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:54:43 -0700 Subject: [PATCH 5/7] Handle readonly empty files in windows --- internal/restorer/restorer.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 0aeb636d0..9f41f5cf2 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -205,11 +205,19 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error { wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return err + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(target); err != nil { + return err + } + if wr, err = os.OpenFile(target, os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + return err + } } - err = wr.Close() - if err != nil { + if err = wr.Close(); err != nil { return err } From 4bbd25a37fb4554b01af7b567ac7b4f16147a588 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:55:50 -0700 Subject: [PATCH 6/7] Add tests for generic attribute changes --- internal/restic/node_windows_test.go | 210 +++++++++ internal/restorer/restorer_test.go | 86 ++-- internal/restorer/restorer_unix_test.go | 4 +- internal/restorer/restorer_windows_test.go | 506 +++++++++++++++++++++ 4 files changed, 770 insertions(+), 36 deletions(-) create mode 100644 internal/restic/node_windows_test.go diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go new file mode 100644 index 000000000..501d5a98a --- /dev/null +++ b/internal/restic/node_windows_test.go @@ -0,0 +1,210 @@ +//go:build windows +// +build windows + +package restic + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" + "golang.org/x/sys/windows" +) + +func TestRestoreCreationTime(t *testing.T) { + t.Parallel() + path := t.TempDir() + fi, err := os.Lstat(path) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", path)) + creationTimeAttribute := getCreationTime(fi, path) + test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) + //Using the temp dir creation time as the test creation time for the test file and folder + runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) +} + +func TestRestoreFileAttributes(t *testing.T) { + t.Parallel() + genericAttributeName := TypeFileAttributes + tempDir := t.TempDir() + normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) + hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) + system := uint32(syscall.FILE_ATTRIBUTE_SYSTEM) + archive := uint32(syscall.FILE_ATTRIBUTE_ARCHIVE) + encrypted := uint32(windows.FILE_ATTRIBUTE_ENCRYPTED) + fileAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, fileAttr := range fileAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testfile%d", i), + Type: "file", + Mode: 0655, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, fileAttr, false) + } + normal = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY) + hidden = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | syscall.FILE_ATTRIBUTE_HIDDEN) + system = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_SYSTEM) + archive = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ARCHIVE) + encrypted = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ENCRYPTED) + folderAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, folderAttr := range folderAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testdirectory%d", i), + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, folderAttr, false) + } +} + +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) +} +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected) + rawMessage := node.GenericAttributes[genericAttr] + genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + rawMessageExpected := genericAttrsExpected[genericAttr] + test.Equals(t, rawMessageExpected, rawMessage, "Generic attribute: %s got from NodeFromFileInfo not equal for path: %s", string(genericAttr), testPath) + } +} + +func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) { + testPath := filepath.Join(tempDir, "001", testNode.Name) + err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) + + if testNode.Type == "file" { + + testFile, err := os.Create(testPath) + test.OK(t, errors.Wrapf(err, "Failed to create test file: %s", testPath)) + testFile.Close() + } else if testNode.Type == "dir" { + + err := os.Mkdir(testPath, testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) + } + + err = testNode.RestoreMetadata(testPath, func(msg string) { + if warningExpected { + test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg) + } else { + // If warning is not expected, this code should not get triggered. + test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg)) + } + }) + test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath)) + + fi, err := os.Lstat(testPath) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath)) + + nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi) + test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath)) + + return testPath, nodeFromFileInfo +} + +const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" + +func TestNewGenericAttributeType(t *testing.T) { + t.Parallel() + + newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") + + tempDir := t.TempDir() + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + } + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, true) + _, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + test.OK(t, err) + // Since this GenericAttribute is unknown to this version of the software, it will not get set on the file. + test.Assert(t, len(ua) == 0, "Unkown attributes: %s found for path: %s", ua, testPath) + } +} diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index c33214bc3..5742d7663 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -3,6 +3,7 @@ package restorer import ( "bytes" "context" + "encoding/json" "io" "math" "os" @@ -27,17 +28,27 @@ type Snapshot struct { } type File struct { - Data string - Links uint64 - Inode uint64 - Mode os.FileMode - ModTime time.Time + Data string + Links uint64 + Inode uint64 + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes } type Dir struct { - Nodes map[string]Node - Mode os.FileMode - ModTime time.Time + Nodes map[string]Node + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes +} + +type FileAttributes struct { + ReadOnly bool + Hidden bool + System bool + Archive bool + Encrypted bool } func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID { @@ -52,7 +63,7 @@ func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID { return id } -func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64) restic.ID { +func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -78,20 +89,21 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u mode = 0644 } err := tree.Insert(&restic.Node{ - Type: "file", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Content: fc, - Size: uint64(len(n.(File).Data)), - Inode: fi, - Links: lc, + Type: "file", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Content: fc, + Size: uint64(len(n.(File).Data)), + Inode: fi, + Links: lc, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) case Dir: - id := saveDir(t, repo, node.Nodes, inode) + id := saveDir(t, repo, node.Nodes, inode, getGenericAttributes) mode := node.Mode if mode == 0 { @@ -99,13 +111,14 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u } err := tree.Insert(&restic.Node{ - Type: "dir", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Subtree: &id, + Type: "dir", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Subtree: &id, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) default: @@ -121,13 +134,13 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u return id } -func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*restic.Snapshot, restic.ID) { +func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) (*restic.Snapshot, restic.ID) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - treeID := saveDir(t, repo, snapshot.Nodes, 1000) + treeID := saveDir(t, repo, snapshot.Nodes, 1000, getGenericAttributes) err := repo.Flush(ctx) if err != nil { t.Fatal(err) @@ -147,6 +160,11 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res return sn, id } +var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + // No-op + return nil +} + func TestRestorer(t *testing.T) { var tests = []struct { Snapshot @@ -322,7 +340,7 @@ func TestRestorer(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, false, nil) @@ -439,7 +457,7 @@ func TestRestorerRelative(t *testing.T) { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, false, nil) @@ -669,7 +687,7 @@ func TestRestorerTraverseTree(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, test.Snapshot) + sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -745,7 +763,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -800,7 +818,7 @@ func TestVerifyCancel(t *testing.T) { } repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, snapshot) + sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index 2c30a6b64..0cbfefa92 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -29,7 +29,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -95,7 +95,7 @@ func TestRestorerProgressBar(t *testing.T) { }, "file2": File{Links: 1, Inode: 2, Data: "example"}, }, - }) + }, noopGetGenericAttributes) mock := &printerMock{} progress := restoreui.NewProgress(mock, 0) diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 3ec4b1f11..684d51ace 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -4,11 +4,20 @@ package restorer import ( + "context" + "encoding/json" "math" + "os" + "path" "syscall" "testing" + "time" "unsafe" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -33,3 +42,500 @@ func getBlockCount(t *testing.T, filename string) int64 { return int64(math.Ceil(float64(result) / 512)) } + +type DataStreamInfo struct { + name string + data string +} + +type NodeInfo struct { + DataStreamInfo + parentDir string + attributes FileAttributes + Exists bool + IsDirectory bool +} + +func TestFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, false) +} + +func TestEmptyFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, true) +} + +func testFileAttributeCombination(t *testing.T, isEmpty bool) { + t.Parallel() + //Generate combination of 5 attributes. + attributeCombinations := generateCombinations(5, []bool{}) + + fileName := "TestFile.txt" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required file information + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: false, + } + + //Get the current test name + testName := getCombinationTestName(fileInfo, fileName, fileInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, fileInfo.attributes) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } +} + +func generateCombinations(n int, prefix []bool) [][]bool { + if n == 0 { + // Return a slice containing the current permutation + return [][]bool{append([]bool{}, prefix...)} + } + + // Generate combinations with True + prefixTrue := append(prefix, true) + permsTrue := generateCombinations(n-1, prefixTrue) + + // Generate combinations with False + prefixFalse := append(prefix, false) + permsFalse := generateCombinations(n-1, prefixFalse) + + // Combine combinations with True and False + return append(permsTrue, permsFalse...) +} + +func getDataStreamInfo(isEmpty bool, fileName string) DataStreamInfo { + var dataStreamInfo DataStreamInfo + if isEmpty { + dataStreamInfo = DataStreamInfo{ + name: fileName, + } + } else { + dataStreamInfo = DataStreamInfo{ + name: fileName, + data: "Main file data stream.", + } + } + return dataStreamInfo +} + +func getFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + ReadOnly: values[0], + Hidden: values[1], + System: values[2], + Archive: values[3], + Encrypted: values[4], + } +} + +func getCombinationTestName(fi NodeInfo, fileName string, overwriteAttr FileAttributes) string { + if fi.attributes.ReadOnly { + fileName += "-ReadOnly" + } + if fi.attributes.Hidden { + fileName += "-Hidden" + } + if fi.attributes.System { + fileName += "-System" + } + if fi.attributes.Archive { + fileName += "-Archive" + } + if fi.attributes.Encrypted { + fileName += "-Encrypted" + } + if fi.Exists { + fileName += "-Overwrite" + if overwriteAttr.ReadOnly { + fileName += "-R" + } + if overwriteAttr.Hidden { + fileName += "-H" + } + if overwriteAttr.System { + fileName += "-S" + } + if overwriteAttr.Archive { + fileName += "-A" + } + if overwriteAttr.Encrypted { + fileName += "-E" + } + } + return fileName +} + +func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAttributes) string { + testDir := t.TempDir() + res, _ := setupWithFileAttributes(t, fileInfo, testDir, existingFileAttr) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, testDir) + rtest.OK(t, err) + + mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name) + //Verify restore + verifyFileAttributes(t, mainFilePath, fileInfo.attributes) + return mainFilePath +} + +func setupWithFileAttributes(t *testing.T, nodeInfo NodeInfo, testDir string, existingFileAttr FileAttributes) (*Restorer, []int) { + t.Helper() + if nodeInfo.Exists { + if !nodeInfo.IsDirectory { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir), os.ModeDir) + rtest.OK(t, err) + filepath := path.Join(testDir, nodeInfo.parentDir, nodeInfo.name) + if existingFileAttr.Encrypted { + err := createEncryptedFileWriteData(filepath, nodeInfo) + rtest.OK(t, err) + } else { + // Write the data to the file + file, err := os.OpenFile(path.Clean(filepath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + rtest.OK(t, err) + _, err = file.Write([]byte(nodeInfo.data)) + rtest.OK(t, err) + + err = file.Close() + rtest.OK(t, err) + } + } else { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name), os.ModeDir) + rtest.OK(t, err) + } + + pathPointer, err := syscall.UTF16PtrFromString(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name)) + rtest.OK(t, err) + syscall.SetFileAttributes(pathPointer, getAttributeValue(&existingFileAttr)) + } + + index := 0 + + order := []int{} + streams := []DataStreamInfo{} + if !nodeInfo.IsDirectory { + order = append(order, index) + index++ + streams = append(streams, nodeInfo.DataStreamInfo) + } + return setup(t, getNodes(nodeInfo.parentDir, nodeInfo.name, order, streams, nodeInfo.IsDirectory, &nodeInfo.attributes)), order +} + +func createEncryptedFileWriteData(filepath string, fileInfo NodeInfo) (err error) { + var ptr *uint16 + if ptr, err = windows.UTF16PtrFromString(filepath); err != nil { + return err + } + var handle windows.Handle + //Create the file with encrypted flag + if handle, err = windows.CreateFile(ptr, uint32(windows.GENERIC_READ|windows.GENERIC_WRITE), uint32(windows.FILE_SHARE_READ), nil, uint32(windows.CREATE_ALWAYS), windows.FILE_ATTRIBUTE_ENCRYPTED, 0); err != nil { + return err + } + //Write data to file + if _, err = windows.Write(handle, []byte(fileInfo.data)); err != nil { + return err + } + //Close handle + return windows.CloseHandle(handle) +} + +func setup(t *testing.T, nodesMap map[string]Node) *Restorer { + repo := repository.TestRepository(t) + getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + if attr == nil { + return + } + + fileattr := getAttributeValue(attr) + + if isDir { + //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes + fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY + } + attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + test.OK(t, err) + return attrs + } + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: nodesMap, + }, getFileAttributes) + res := NewRestorer(repo, sn, false, nil) + return res +} + +func getAttributeValue(attr *FileAttributes) uint32 { + var fileattr uint32 + if attr.ReadOnly { + fileattr |= windows.FILE_ATTRIBUTE_READONLY + } + if attr.Hidden { + fileattr |= windows.FILE_ATTRIBUTE_HIDDEN + } + if attr.Encrypted { + fileattr |= windows.FILE_ATTRIBUTE_ENCRYPTED + } + if attr.Archive { + fileattr |= windows.FILE_ATTRIBUTE_ARCHIVE + } + if attr.System { + fileattr |= windows.FILE_ATTRIBUTE_SYSTEM + } + return fileattr +} + +func getNodes(dir string, mainNodeName string, order []int, streams []DataStreamInfo, isDirectory bool, attributes *FileAttributes) map[string]Node { + var mode os.FileMode + if isDirectory { + mode = os.FileMode(2147484159) + } else { + if attributes != nil && attributes.ReadOnly { + mode = os.FileMode(0o444) + } else { + mode = os.FileMode(0o666) + } + } + + getFileNodes := func() map[string]Node { + nodes := map[string]Node{} + if isDirectory { + //Add a directory node at the same level as the other streams + nodes[mainNodeName] = Dir{ + ModTime: time.Now(), + attributes: attributes, + Mode: mode, + } + } + + if len(streams) > 0 { + for _, index := range order { + stream := streams[index] + + var attr *FileAttributes = nil + if mainNodeName == stream.name { + attr = attributes + } else if attributes != nil && attributes.Encrypted { + //Set encrypted attribute + attr = &FileAttributes{Encrypted: true} + } + + nodes[stream.name] = File{ + ModTime: time.Now(), + Data: stream.data, + Mode: mode, + attributes: attr, + } + } + } + return nodes + } + + return map[string]Node{ + dir: Dir{ + Mode: normalizeFileMode(0750 | mode), + ModTime: time.Now(), + Nodes: getFileNodes(), + }, + } +} + +func verifyFileAttributes(t *testing.T, mainFilePath string, attr FileAttributes) { + ptr, err := windows.UTF16PtrFromString(mainFilePath) + rtest.OK(t, err) + //Get file attributes using syscall + fileAttributes, err := syscall.GetFileAttributes(ptr) + rtest.OK(t, err) + //Test positive and negative scenarios + if attr.ReadOnly { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY != 0, "Expected read only attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY == 0, "Unexpected read only attibute.") + } + if attr.Hidden { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN != 0, "Expected hidden attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN == 0, "Unexpected hidden attibute.") + } + if attr.System { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM != 0, "Expected system attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM == 0, "Unexpected system attibute.") + } + if attr.Archive { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE != 0, "Expected archive attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE == 0, "Unexpected archive attibute.") + } + if attr.Encrypted { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED != 0, "Expected encrypted attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED == 0, "Unexpected encrypted attibute.") + } +} + +func verifyFileRestores(isEmpty bool, mainFilePath string, t *testing.T, fileInfo NodeInfo) { + if isEmpty { + _, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + } else { + + verifyMainFileRestore(t, mainFilePath, fileInfo) + } +} + +func verifyMainFileRestore(t *testing.T, mainFilePath string, fileInfo NodeInfo) { + fi, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + + size := fi.Size() + rtest.Assert(t, size > 0, "The file "+fileInfo.name+" exists but is empty") + + content, err := os.ReadFile(mainFilePath) + rtest.OK(t, err) + rtest.Assert(t, string(content) == fileInfo.data, "The file "+fileInfo.name+" exists but the content is not overwritten") +} + +func TestDirAttributeCombination(t *testing.T) { + t.Parallel() + attributeCombinations := generateCombinations(4, []bool{}) + + dirName := "TestDir" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required directory information + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: false, + IsDirectory: true, + } + + //Get the current test name + testName := getCombinationTestName(dirInfo, dirName, dirInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } +} + +func getDirFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + // readonly not valid for directories + Hidden: values[0], + System: values[1], + Archive: values[2], + Encrypted: values[3], + } +} + +func TestFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, false) +} + +func TestEmptyFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, true) +} + +func testFileAttributeCombinationsOverwrite(t *testing.T, isEmpty bool) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(5, []bool{}) + //Get overwrite file attribute combinations + overwriteCombinations := generateCombinations(5, []bool{}) + + fileName := "TestOverwriteFile" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: true, + } + + overwriteFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteFileAttributes = append(overwriteFileAttributes, getFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combination + for _, overwriteFileAttr := range overwriteFileAttributes { + //Get the test name + testName := getCombinationTestName(fileInfo, fileName, overwriteFileAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, overwriteFileAttr) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } + } +} + +func TestDirAttributeCombinationsOverwrite(t *testing.T) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(4, []bool{}) + //Get overwrite dir attribute combinations + overwriteCombinations := generateCombinations(4, []bool{}) + + dirName := "TestOverwriteDir" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: true, + IsDirectory: true, + } + + overwriteDirFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteDirFileAttributes = append(overwriteDirFileAttributes, getDirFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combinations + for _, overwriteDirAttr := range overwriteDirFileAttributes { + //Get the test name + testName := getCombinationTestName(dirInfo, dirName, overwriteDirAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } + } +} From e8211cb64a80fc4f949b5088c167d51dc4ef45a1 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:59:56 -0700 Subject: [PATCH 7/7] Add changelog and update docs for windows attr --- changelog/unreleased/pull-4611 | 7 +++++++ doc/040_backup.rst | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/pull-4611 diff --git a/changelog/unreleased/pull-4611 b/changelog/unreleased/pull-4611 new file mode 100644 index 000000000..940de9c26 --- /dev/null +++ b/changelog/unreleased/pull-4611 @@ -0,0 +1,7 @@ +Enhancement: Back up windows created time and file attributes like hidden flag + +Restic did not back up windows-specific meta-data like created time and file attributes like hidden flag. +Restic now backs up file created time and file attributes like hidden, readonly and encrypted flag when backing up files and folders on windows. + +https://github.com/restic/restic/pull/4611 + diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 550957eeb..d0bd4b2e2 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -487,7 +487,6 @@ particular note are: * File creation date on Unix platforms * Inode flags on Unix platforms * File ownership and ACLs on Windows -* The "hidden" flag on Windows Reading data from a command ***************************