From 5d5f3de62feab51f04b47c4a357959134051f876 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:08:07 +0200 Subject: [PATCH 01/24] Output EXE file on Windows platform. --- build.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build.go b/build.go index 765432f0c..94d245c4c 100644 --- a/build.go +++ b/build.go @@ -10,6 +10,7 @@ import ( "os/exec" "path" "path/filepath" + "runtime" "strings" "time" ) @@ -261,8 +262,14 @@ func main() { args := []string{ "-tags", strings.Join(buildTags, " "), "-ldflags", fmt.Sprintf(`-s -X main.version %q -X main.compiledAt %q`, version, compileTime), - "-o", "restic", "github.com/restic/restic/cmd/restic", } + if runtime.GOOS != "windows" { + args = append(args, "-o", "restic", "github.com/restic/restic/cmd/restic") + + } else { + args = append(args, "-o", "restic.exe", "github.com/restic/restic/cmd/restic") + } + err = build(gopath, args...) if err != nil { fmt.Fprintf(os.Stderr, "build failed: %v\n", err) From 3504ea39925cc26a71926bb34667806e44542fdb Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:13:11 +0200 Subject: [PATCH 02/24] Skip UID/GID and SIGHUP lock-check on Windows On Windows, User ID/Group ID is not a number, so we ignore any parsing error that may come from attempting to convert it to an integer. It will simply be '0'. Don't defer the process Release until we have checked the error. Skip the SIGHUP connection attempt on Windows. If the process exists, we assume it is running. --- lock.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lock.go b/lock.go index c662f1718..767f21511 100644 --- a/lock.go +++ b/lock.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/backend" "github.com/restic/restic/debug" "github.com/restic/restic/repository" + "runtime" ) // Lock represents a process locking the repository for an operation. @@ -116,16 +117,12 @@ func (l *Lock) fillUserInfo() error { } l.Username = usr.Username - uid, err := strconv.ParseInt(usr.Uid, 10, 32) - if err != nil { - return err - } + // We ignore the error. On Windows Uid is not a number + uid, _ := strconv.ParseInt(usr.Uid, 10, 32) l.UID = uint32(uid) - gid, err := strconv.ParseInt(usr.Gid, 10, 32) - if err != nil { - return err - } + // We ignore the error. On Windows Gid is not a number + gid, _ := strconv.ParseInt(usr.Gid, 10, 32) l.GID = uint32(gid) return nil @@ -207,17 +204,20 @@ func (l *Lock) Stale() bool { } proc, err := os.FindProcess(l.PID) - defer proc.Release() if err != nil { debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) return true } + defer proc.Release() - debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) - err = proc.Signal(syscall.SIGHUP) - if err != nil { - debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) - return true + // Windows does not have SIGHUP + if runtime.GOOS != "windows" { + debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) + err = proc.Signal(syscall.SIGHUP) + if err != nil { + debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) + return true + } } debug.Log("Lock.Stale", "lock not stale\n") From 3804bc7493711782d022164f9dc95b8e1c79a700 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:19:45 +0200 Subject: [PATCH 03/24] Use bigger offset in test to avoid random test failures. Since Windows cannot to a SIGHUP test to check if a process is live, the test will fail because the lock isn't detected as stale if a process with that ID exists. Process IDs are re-used agressively on Windows, so add 500000 makes the test extremely unlikely (if not impossible) to fail. --- lock_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lock_test.go b/lock_test.go index a2b5cb2d3..c07abb239 100644 --- a/lock_test.go +++ b/lock_test.go @@ -124,7 +124,7 @@ var staleLockTests = []struct { { timestamp: time.Now(), stale: true, - pid: os.Getpid() + 500, + pid: os.Getpid() + 500000, }, } @@ -158,7 +158,7 @@ func TestLockWithStaleLock(t *testing.T) { id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) OK(t, err) - id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000) OK(t, err) OK(t, restic.RemoveStaleLocks(repo)) From 7c84d810d33a6bb9f542e4f49f8c0ebc46ce7a8f Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:30:36 +0200 Subject: [PATCH 04/24] Make sure we can delete files on Windows. Files must be closed on Windows before they can be deleted. Therefore we keep track of all open files, and closes them before we delete them. Also we don't set finished blobs to read-only on Windows, since that prevents us from deleting them. --- backend/local/local.go | 72 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index cc95bb5b4..f63a71743 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -7,7 +7,9 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "sort" + "sync" "github.com/restic/restic/backend" ) @@ -15,7 +17,9 @@ import ( var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") type Local struct { - p string + p string + mu sync.Mutex + open map[string][]*os.File // Contains open files. Guarded by 'mu'. } // Open opens the local backend at dir. @@ -37,7 +41,7 @@ func Open(dir string) (*Local, error) { } } - return &Local{p: dir}, nil + return &Local{p: dir, open: make(map[string][]*os.File)}, nil } // Create creates all the necessary files and directories for a new local @@ -143,7 +147,16 @@ func (lb *localBlob) Finalize(t backend.Type, name string) error { return err } - return os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222))) + // set file to readonly, except on Windows, + // otherwise deletion will fail. + if runtime.GOOS != "windows" { + err = os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222))) + if err != nil { + return err + } + } + + return nil } // Create creates a new Blob. The data is available only after Finalize() @@ -162,6 +175,11 @@ func (b *Local) Create() (backend.Blob, error) { basedir: b.p, } + b.mu.Lock() + open, _ := b.open["blobs"] + b.open["blobs"] = append(open, file) + b.mu.Unlock() + return &blob, nil } @@ -198,7 +216,15 @@ func dirname(base string, t backend.Type, name string) string { // Get returns a reader that yields the content stored under the given // name. The reader should be closed after draining it. func (b *Local) Get(t backend.Type, name string) (io.ReadCloser, error) { - return os.Open(filename(b.p, t, name)) + file, err := os.Open(filename(b.p, t, name)) + if err != nil { + return nil, err + } + b.mu.Lock() + open, _ := b.open[filename(b.p, t, name)] + b.open[filename(b.p, t, name)] = append(open, file) + b.mu.Unlock() + return file, nil } // GetReader returns an io.ReadCloser for the Blob with the given name of @@ -209,6 +235,11 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io. return nil, err } + b.mu.Lock() + open, _ := b.open[filename(b.p, t, name)] + b.open[filename(b.p, t, name)] = append(open, f) + b.mu.Unlock() + _, err = f.Seek(int64(offset), 0) if err != nil { return nil, err @@ -236,7 +267,17 @@ func (b *Local) Test(t backend.Type, name string) (bool, error) { // Remove removes the blob with the given name and type. func (b *Local) Remove(t backend.Type, name string) error { - return os.Remove(filename(b.p, t, name)) + // close all open files we may have. + fn := filename(b.p, t, name) + b.mu.Lock() + open, _ := b.open[fn] + for _, file := range open { + file.Close() + } + b.open[fn] = nil + b.mu.Unlock() + + return os.Remove(fn) } // List returns a channel that yields all names of blobs of type t. A @@ -283,7 +324,22 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { } // Delete removes the repository and all files. -func (b *Local) Delete() error { return os.RemoveAll(b.p) } +func (b *Local) Delete() error { + b.Close() + return os.RemoveAll(b.p) +} -// Close does nothing -func (b *Local) Close() error { return nil } +// Close closes all open files. +// They may have been closed already, +// so we ignore all errors. +func (b *Local) Close() error { + b.mu.Lock() + for _, open := range b.open { + for _, file := range open { + file.Close() + } + } + b.open = make(map[string][]*os.File) + b.mu.Unlock() + return nil +} From 2dcb52782824856a92beaef071ef2d7805479df3 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:32:07 +0200 Subject: [PATCH 05/24] Only use Setsid on Unix. Setsid is not a part of syscall.SysProcAttr on Windows, so we only set that on systems that have it. --- backend/sftp/sftp.go | 4 +++- backend/sftp/sftp_unix.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 backend/sftp/sftp_unix.go diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 71c874f0b..08641fd5e 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -28,6 +28,8 @@ type SFTP struct { cmd *exec.Cmd } +var sysProcAttr syscall.SysProcAttr + func startClient(program string, args ...string) (*SFTP, error) { // Connect to a remote host and request the sftp subsystem via the 'ssh' // command. This assumes that passwordless login is correctly configured. @@ -37,7 +39,7 @@ func startClient(program string, args ...string) (*SFTP, error) { cmd.Stderr = os.Stderr // ignore signals sent to the parent (e.g. SIGINT) - cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + cmd.SysProcAttr = &sysProcAttr // get stdin and stdout wr, err := cmd.StdinPipe() diff --git a/backend/sftp/sftp_unix.go b/backend/sftp/sftp_unix.go new file mode 100644 index 000000000..577649426 --- /dev/null +++ b/backend/sftp/sftp_unix.go @@ -0,0 +1,12 @@ +// +build !windows + +package sftp + +import ( + "syscall" +) + +func init() { + // ignore signals sent to the parent (e.g. SIGINT) + sysProcAttr = syscall.SysProcAttr{Setsid: true} +} From fb3778abb6f19c1cb2e142049671994081b508c7 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:33:11 +0200 Subject: [PATCH 06/24] Disable mount command on Windows. FUSE does not support Windows, so we disable it on Windows. --- cmd/restic/cmd_mount.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index dc2ad673d..b8d8cb277 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -1,4 +1,5 @@ // +build !openbsd +// +build !windows package main From 4dc746dac260898974f13ea694515eeb4916afc5 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:39:16 +0200 Subject: [PATCH 07/24] Change repository type detection to first check if path is a directory that exists. The method of determining if a repository exists doesn't work on Windows, since the "url.Scheme" will contain the drive letter - "c" in "c:\backup", so as a first step we check if the URL can be opened as a file, and if so, we assume it is a 'local' type repository. --- cmd/restic/global.go | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index b71f7aa25..46ee2e011 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -132,7 +132,14 @@ func (o GlobalOptions) OpenRepository() (*repository.Repository, error) { // * s3://region/bucket -> amazon s3 bucket // * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar // * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup +// * c:\temp -> local repository at c:\temp - the path must exist func open(u string) (backend.Backend, error) { + // check if the url is a directory that exists + fi, err := os.Stat(u) + if err == nil && fi.IsDir() { + return local.Open(u) + } + url, err := url.Parse(u) if err != nil { return nil, err @@ -140,7 +147,13 @@ func open(u string) (backend.Backend, error) { if url.Scheme == "" { return local.Open(url.Path) - } else if url.Scheme == "s3" { + } + + if len(url.Path) < 1 { + return nil, fmt.Errorf("unable to parse url %v", url) + } + + if url.Scheme == "s3" { return s3.Open(url.Host, url.Path[1:]) } @@ -156,6 +169,12 @@ func open(u string) (backend.Backend, error) { // Create the backend specified by URI. func create(u string) (backend.Backend, error) { + // check if the url is a directory that exists + fi, err := os.Stat(u) + if err == nil && fi.IsDir() { + return local.Create(u) + } + url, err := url.Parse(u) if err != nil { return nil, err @@ -163,7 +182,13 @@ func create(u string) (backend.Backend, error) { if url.Scheme == "" { return local.Create(url.Path) - } else if url.Scheme == "s3" { + } + + if len(url.Path) < 1 { + return nil, fmt.Errorf("unable to parse url %v", url) + } + + if url.Scheme == "s3" { return s3.Open(url.Host, url.Path[1:]) } From 2e7b40baca102b239108e9505336533f71269c07 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:50:14 +0200 Subject: [PATCH 08/24] Make UID/GID '0' on Windows. We ignore parser errors on Uid/Gid, since they are not numbers on Windows. UID/GID is usually 'root' on Linux, so I am not sure if 0/0 is a good idea. Maybe if the type of the fields were changed from uint32 to int, we could set it to -1 to indicate "no value". --- snapshot.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/snapshot.go b/snapshot.go index b1477c316..1693d2b75 100644 --- a/snapshot.go +++ b/snapshot.go @@ -76,16 +76,12 @@ func (sn *Snapshot) fillUserInfo() error { } sn.Username = usr.Username - uid, err := strconv.ParseInt(usr.Uid, 10, 32) - if err != nil { - return err - } + // We ignore the error. On Windows Uid is not a number + uid, _ := strconv.ParseInt(usr.Uid, 10, 32) sn.UID = uint32(uid) - gid, err := strconv.ParseInt(usr.Gid, 10, 32) - if err != nil { - return err - } + // We ignore the error. On Windows Gid is not a number + gid, _ := strconv.ParseInt(usr.Gid, 10, 32) sn.GID = uint32(gid) return nil From dfe232cf4693ab54addf91302c940d52c50fa34d Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 14 Aug 2015 15:57:47 +0200 Subject: [PATCH 09/24] Add Windows node support. The syscall.Stat_t doesn't exist on Windows, so it is replaced by an interface, which Windows can fill out, and field access is replaced by function calls. Common Unix functionality is put into "node_unix.go", so there is less boilerplate. Symlinks are skipped on Windows, since they require admin privileges. --- node.go | 78 ++++++++++++++++++++++++++++++++++++++----------- node_darwin.go | 14 +++------ node_freebsd.go | 14 +++------ node_linux.go | 14 +++------ node_openbsd.go | 14 +++------ node_test.go | 19 +++++++----- node_unix.go | 34 +++++++++++++++++++++ node_windows.go | 66 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 189 insertions(+), 64 deletions(-) create mode 100644 node_unix.go create mode 100644 node_windows.go diff --git a/node.go b/node.go index e607bc8b1..974f83834 100644 --- a/node.go +++ b/node.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/debug" "github.com/restic/restic/pack" "github.com/restic/restic/repository" + "runtime" ) // Node is a file, directory or other item in a backup. @@ -148,7 +149,7 @@ func (node *Node) CreateAt(path string, repo *repository.Repository) error { func (node Node) restoreMetadata(path string) error { var err error - err = os.Lchown(path, int(node.UID), int(node.GID)) + err = lchown(path, int(node.UID), int(node.GID)) if err != nil { return errors.Annotate(err, "Lchown") } @@ -236,6 +237,10 @@ func (node Node) createFileAt(path string, repo *repository.Repository) error { } func (node Node) createSymlinkAt(path string) error { + // Windows does not allow non-admins to create soft links. + if runtime.GOOS == "windows" { + return nil + } err := os.Symlink(node.LinkTarget, path) if err != nil { return errors.Annotate(err, "Symlink") @@ -245,15 +250,15 @@ func (node Node) createSymlinkAt(path string) error { } func (node *Node) createDevAt(path string) error { - return syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device)) + return mknod(path, syscall.S_IFBLK|0600, int(node.Device)) } func (node *Node) createCharDevAt(path string) error { - return syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device)) + return mknod(path, syscall.S_IFCHR|0600, int(node.Device)) } func (node *Node) createFifoAt(path string) error { - return syscall.Mkfifo(path, 0600) + return mkfifo(path, 0600) } func (node Node) MarshalJSON() ([]byte, error) { @@ -381,9 +386,19 @@ func (node *Node) isNewer(path string, fi os.FileInfo) bool { return true } - extendedStat := fi.Sys().(*syscall.Stat_t) - inode := extendedStat.Ino - size := uint64(extendedStat.Size) + size := uint64(fi.Size()) + + extendedStat, ok := toStatT(fi.Sys()) + if !ok { + if node.ModTime != fi.ModTime() || + node.Size != size { + debug.Log("node.isNewer", "node %v is newer: timestamp, size or inode changed", path) + return true + } + return false + } + + inode := extendedStat.ino() if node.ModTime != fi.ModTime() || node.ChangeTime != changeTime(extendedStat) || @@ -397,11 +412,11 @@ func (node *Node) isNewer(path string, fi os.FileInfo) bool { return false } -func (node *Node) fillUser(stat *syscall.Stat_t) error { - node.UID = stat.Uid - node.GID = stat.Gid +func (node *Node) fillUser(stat statT) error { + node.UID = stat.uid() + node.GID = stat.gid() - username, err := lookupUsername(strconv.Itoa(int(stat.Uid))) + username, err := lookupUsername(strconv.Itoa(int(stat.uid()))) if err != nil { return errors.Annotate(err, "fillUser") } @@ -439,12 +454,12 @@ func lookupUsername(uid string) (string, error) { } func (node *Node) fillExtra(path string, fi os.FileInfo) error { - stat, ok := fi.Sys().(*syscall.Stat_t) + stat, ok := toStatT(fi.Sys()) if !ok { return nil } - node.Inode = uint64(stat.Ino) + node.Inode = uint64(stat.ino()) node.fillTimes(stat) @@ -456,15 +471,15 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { switch node.Type { case "file": - node.Size = uint64(stat.Size) - node.Links = uint64(stat.Nlink) + node.Size = uint64(stat.size()) + node.Links = uint64(stat.nlink()) case "dir": case "symlink": node.LinkTarget, err = os.Readlink(path) case "dev": - node.Device = uint64(stat.Rdev) + node.Device = uint64(stat.rdev()) case "chardev": - node.Device = uint64(stat.Rdev) + node.Device = uint64(stat.rdev()) case "fifo": case "socket": default: @@ -473,3 +488,32 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { return err } + +type statT interface { + dev() uint64 + ino() uint64 + nlink() uint64 + uid() uint32 + gid() uint32 + rdev() uint64 + size() int64 + atim() syscall.Timespec + mtim() syscall.Timespec + ctim() syscall.Timespec +} + +func mkfifo(path string, mode uint32) (err error) { + return mknod(path, mode|syscall.S_IFIFO, 0) +} + +func (node *Node) fillTimes(stat statT) { + ctim := stat.ctim() + atim := stat.atim() + node.ChangeTime = time.Unix(ctim.Unix()) + node.AccessTime = time.Unix(atim.Unix()) +} + +func changeTime(stat statT) time.Time { + ctim := stat.ctim() + return time.Unix(ctim.Unix()) +} diff --git a/node_darwin.go b/node_darwin.go index f63b41f13..adc980295 100644 --- a/node_darwin.go +++ b/node_darwin.go @@ -3,22 +3,16 @@ package restic import ( "os" "syscall" - "time" ) func (node *Node) OpenForReading() (*os.File, error) { return os.Open(node.path) } -func changeTime(stat *syscall.Stat_t) time.Time { - return time.Unix(stat.Ctimespec.Unix()) -} - -func (node *Node) fillTimes(stat *syscall.Stat_t) { - node.ChangeTime = time.Unix(stat.Ctimespec.Unix()) - node.AccessTime = time.Unix(stat.Atimespec.Unix()) -} - func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } + +func (s statUnix) atim() syscall.Timespec { return s.Atimespec } +func (s statUnix) mtim() syscall.Timespec { return s.Mtimespec } +func (s statUnix) ctim() syscall.Timespec { return s.Ctimespec } diff --git a/node_freebsd.go b/node_freebsd.go index 231cf8db7..67bcdf3e9 100644 --- a/node_freebsd.go +++ b/node_freebsd.go @@ -3,22 +3,16 @@ package restic import ( "os" "syscall" - "time" ) func (node *Node) OpenForReading() (*os.File, error) { return os.OpenFile(node.path, os.O_RDONLY, 0) } -func (node *Node) fillTimes(stat *syscall.Stat_t) { - node.ChangeTime = time.Unix(stat.Ctimespec.Unix()) - node.AccessTime = time.Unix(stat.Atimespec.Unix()) -} - -func changeTime(stat *syscall.Stat_t) time.Time { - return time.Unix(stat.Ctimespec.Unix()) -} - func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } + +func (s statUnix) atim() syscall.Timespec { return s.Atimespec } +func (s statUnix) mtim() syscall.Timespec { return s.Mtimespec } +func (s statUnix) ctim() syscall.Timespec { return s.Ctimespec } diff --git a/node_linux.go b/node_linux.go index 1043397a8..2304c13d5 100644 --- a/node_linux.go +++ b/node_linux.go @@ -4,7 +4,6 @@ import ( "os" "path/filepath" "syscall" - "time" "unsafe" "github.com/juju/errors" @@ -18,15 +17,6 @@ func (node *Node) OpenForReading() (*os.File, error) { return file, err } -func (node *Node) fillTimes(stat *syscall.Stat_t) { - node.ChangeTime = time.Unix(stat.Ctim.Unix()) - node.AccessTime = time.Unix(stat.Atim.Unix()) -} - -func changeTime(stat *syscall.Stat_t) time.Time { - return time.Unix(stat.Ctim.Unix()) -} - func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { dir, err := os.Open(filepath.Dir(path)) defer dir.Close() @@ -65,3 +55,7 @@ func utimensat(dirfd int, path string, times *[2]syscall.Timespec, flags int) (e func utimesNanoAt(dirfd int, path string, ts [2]syscall.Timespec, flags int) (err error) { return utimensat(dirfd, path, (*[2]syscall.Timespec)(unsafe.Pointer(&ts[0])), flags) } + +func (s statUnix) atim() syscall.Timespec { return s.Atim } +func (s statUnix) mtim() syscall.Timespec { return s.Mtim } +func (s statUnix) ctim() syscall.Timespec { return s.Ctim } diff --git a/node_openbsd.go b/node_openbsd.go index 38f1d8441..feafe307e 100644 --- a/node_openbsd.go +++ b/node_openbsd.go @@ -3,7 +3,6 @@ package restic import ( "os" "syscall" - "time" ) func (node *Node) OpenForReading() (*os.File, error) { @@ -14,15 +13,10 @@ func (node *Node) OpenForReading() (*os.File, error) { return file, err } -func (node *Node) fillTimes(stat *syscall.Stat_t) { - node.ChangeTime = time.Unix(stat.Ctim.Unix()) - node.AccessTime = time.Unix(stat.Atim.Unix()) -} - -func changeTime(stat *syscall.Stat_t) time.Time { - return time.Unix(stat.Ctim.Unix()) -} - func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } + +func (s statUnix) atim() syscall.Timespec { return s.Atim } +func (s statUnix) mtim() syscall.Timespec { return s.Mtim } +func (s statUnix) ctim() syscall.Timespec { return s.Ctim } diff --git a/node_test.go b/node_test.go index f6104b49b..5bdf284bb 100644 --- a/node_test.go +++ b/node_test.go @@ -119,6 +119,9 @@ func TestNodeRestoreAt(t *testing.T) { nodePath := filepath.Join(tempdir, test.Name) OK(t, test.CreateAt(nodePath, nil)) + if test.Type == "symlink" && runtime.GOOS == "windows" { + continue + } if test.Type == "dir" { OK(t, test.RestoreTimestamps(nodePath)) } @@ -135,14 +138,16 @@ func TestNodeRestoreAt(t *testing.T) { "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) Assert(t, test.Size == n2.Size, "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) - Assert(t, test.UID == n2.UID, - "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) - Assert(t, test.GID == n2.GID, - "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) - if test.Type != "symlink" { - Assert(t, test.Mode == n2.Mode, - "%v: mode doesn't match (%v != %v)", test.Type, test.Mode, n2.Mode) + if runtime.GOOS != "windows" { + Assert(t, test.UID == n2.UID, + "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) + Assert(t, test.GID == n2.GID, + "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) + if test.Type != "symlink" { + Assert(t, test.Mode == n2.Mode, + "%v: mode doesn't match (%v != %v)", test.Type, test.Mode, n2.Mode) + } } AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) diff --git a/node_unix.go b/node_unix.go new file mode 100644 index 000000000..5c0c4d63c --- /dev/null +++ b/node_unix.go @@ -0,0 +1,34 @@ +// + +// +build dragonfly linux netbsd openbsd freebsd solaris darwin + +package restic + +import ( + "os" + "syscall" +) + +var mknod = syscall.Mknod +var lchown = os.Lchown + +type statUnix syscall.Stat_t + +func toStatT(i interface{}) (statT, bool) { + if i == nil { + return nil, false + } + s, ok := i.(*syscall.Stat_t) + if ok && s != nil { + return statUnix(*s), true + } + return nil, false +} + +func (s statUnix) dev() uint64 { return uint64(s.Dev) } +func (s statUnix) ino() uint64 { return uint64(s.Ino) } +func (s statUnix) nlink() uint64 { return uint64(s.Nlink) } +func (s statUnix) uid() uint32 { return uint32(s.Uid) } +func (s statUnix) gid() uint32 { return uint32(s.Gid) } +func (s statUnix) rdev() uint64 { return uint64(s.Rdev) } +func (s statUnix) size() int64 { return int64(s.Size) } diff --git a/node_windows.go b/node_windows.go new file mode 100644 index 000000000..7d5854b80 --- /dev/null +++ b/node_windows.go @@ -0,0 +1,66 @@ +package restic + +import ( + "os" + "syscall" +) + +func (node *Node) OpenForReading() (*os.File, error) { + file, err := os.OpenFile(node.path, os.O_RDONLY, 0) + if os.IsPermission(err) { + return os.OpenFile(node.path, os.O_RDONLY, 0) + } + return file, err +} + +// mknod() creates a filesystem node (file, device +// special file, or named pipe) named pathname, with attributes +// specified by mode and dev. +var mknod = func(path string, mode uint32, dev int) (err error) { + panic("mknod not implemented") +} + +// Windows doesn't need lchown +var lchown = func(path string, uid int, gid int) (err error) { + return nil +} + +func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { + return nil +} + +type statWin syscall.Win32FileAttributeData + +func toStatT(i interface{}) (statT, bool) { + if i == nil { + return nil, false + } + s, ok := i.(*syscall.Win32FileAttributeData) + if ok && s != nil { + return statWin(*s), true + } + return nil, false +} + +func (s statWin) dev() uint64 { return 0 } +func (s statWin) ino() uint64 { return 0 } +func (s statWin) nlink() uint64 { return 0 } +func (s statWin) uid() uint32 { return 0 } +func (s statWin) gid() uint32 { return 0 } +func (s statWin) rdev() uint64 { return 0 } + +func (s statWin) size() int64 { + return int64(s.FileSizeLow) | (int64(s.FileSizeHigh) << 32) +} + +func (s statWin) atim() syscall.Timespec { + return syscall.NsecToTimespec(s.LastAccessTime.Nanoseconds()) +} + +func (s statWin) mtim() syscall.Timespec { + return syscall.NsecToTimespec(s.LastWriteTime.Nanoseconds()) +} + +func (s statWin) ctim() syscall.Timespec { + return syscall.NsecToTimespec(s.CreationTime.Nanoseconds()) +} From 520b1b65b0759b4823999512c1a8ee632277cf76 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 12:39:38 +0200 Subject: [PATCH 10/24] Create setNewFileMode function. Create separate files with setNewFileMode to avoid runtime checks. --- backend/local/local.go | 12 +----------- backend/local/local_unix.go | 12 ++++++++++++ backend/local/local_windows.go | 12 ++++++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 backend/local/local_unix.go create mode 100644 backend/local/local_windows.go diff --git a/backend/local/local.go b/backend/local/local.go index f63a71743..6f5d9d976 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path/filepath" - "runtime" "sort" "sync" @@ -147,16 +146,7 @@ func (lb *localBlob) Finalize(t backend.Type, name string) error { return err } - // set file to readonly, except on Windows, - // otherwise deletion will fail. - if runtime.GOOS != "windows" { - err = os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222))) - if err != nil { - return err - } - } - - return nil + return setNewFileMode(f, fi) } // Create creates a new Blob. The data is available only after Finalize() diff --git a/backend/local/local_unix.go b/backend/local/local_unix.go new file mode 100644 index 000000000..8b8ecec69 --- /dev/null +++ b/backend/local/local_unix.go @@ -0,0 +1,12 @@ +// +build !windows + +package local + +import ( + "os" +) + +// set file to readonly +func setNewFileMode(f string, fi os.FileInfo) error { + return os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222))) +} diff --git a/backend/local/local_windows.go b/backend/local/local_windows.go new file mode 100644 index 000000000..73633fa3e --- /dev/null +++ b/backend/local/local_windows.go @@ -0,0 +1,12 @@ +package local + +import ( + "os" +) + +// We don't modify read-only on windows, +// since it will make us unable to delete the file, +// and this isn't common practice on this platform. +func setNewFileMode(f string, fi os.FileInfo) error { + return nil +} From 0e7d0d8dbae9b39d2783957377881520295fc3c4 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 12:51:01 +0200 Subject: [PATCH 11/24] Create ignoreSigIntProcAttr() Retrieve the SysProcAttr from a separate function. Completely eliminates syscall from main file. --- backend/sftp/sftp.go | 5 +---- backend/sftp/sftp_unix.go | 7 ++++--- backend/sftp/sftp_windows.go | 11 +++++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 backend/sftp/sftp_windows.go diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 08641fd5e..90cb687a9 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -10,7 +10,6 @@ import ( "os/exec" "path/filepath" "sort" - "syscall" "github.com/juju/errors" "github.com/pkg/sftp" @@ -28,8 +27,6 @@ type SFTP struct { cmd *exec.Cmd } -var sysProcAttr syscall.SysProcAttr - func startClient(program string, args ...string) (*SFTP, error) { // Connect to a remote host and request the sftp subsystem via the 'ssh' // command. This assumes that passwordless login is correctly configured. @@ -39,7 +36,7 @@ func startClient(program string, args ...string) (*SFTP, error) { cmd.Stderr = os.Stderr // ignore signals sent to the parent (e.g. SIGINT) - cmd.SysProcAttr = &sysProcAttr + cmd.SysProcAttr = ignoreSigIntProcAttr() // get stdin and stdout wr, err := cmd.StdinPipe() diff --git a/backend/sftp/sftp_unix.go b/backend/sftp/sftp_unix.go index 577649426..f924f0d05 100644 --- a/backend/sftp/sftp_unix.go +++ b/backend/sftp/sftp_unix.go @@ -6,7 +6,8 @@ import ( "syscall" ) -func init() { - // ignore signals sent to the parent (e.g. SIGINT) - sysProcAttr = syscall.SysProcAttr{Setsid: true} +// ignoreSigIntProcAttr returns a syscall.SysProcAttr that +// disables SIGINT on parent. +func ignoreSigIntProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setsid: true} } diff --git a/backend/sftp/sftp_windows.go b/backend/sftp/sftp_windows.go new file mode 100644 index 000000000..62f748c6d --- /dev/null +++ b/backend/sftp/sftp_windows.go @@ -0,0 +1,11 @@ +package sftp + +import ( + "syscall" +) + +// ignoreSigIntProcAttr returns a default syscall.SysProcAttr +// on Windows. +func ignoreSigIntProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{} +} From e1f6bbac9f7c2a1d1ce0cacebb6efcf6714776f5 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 13:16:02 +0200 Subject: [PATCH 12/24] Create uid/gid convertion function. Create separate function to convert user ID / group ID of a user to integer. Windows is a stub, since this doesn't work on Windows (uid/gid is not a number). --- lock.go | 12 ++---------- lock_unix.go | 24 ++++++++++++++++++++++++ lock_windows.go | 10 ++++++++++ snapshot.go | 13 +++---------- 4 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 lock_unix.go create mode 100644 lock_windows.go diff --git a/lock.go b/lock.go index 767f21511..dd24e8b15 100644 --- a/lock.go +++ b/lock.go @@ -5,7 +5,6 @@ import ( "os" "os/signal" "os/user" - "strconv" "sync" "syscall" "time" @@ -117,15 +116,8 @@ func (l *Lock) fillUserInfo() error { } l.Username = usr.Username - // We ignore the error. On Windows Uid is not a number - uid, _ := strconv.ParseInt(usr.Uid, 10, 32) - l.UID = uint32(uid) - - // We ignore the error. On Windows Gid is not a number - gid, _ := strconv.ParseInt(usr.Gid, 10, 32) - l.GID = uint32(gid) - - return nil + l.UID, l.GID, err = uidGidInt(*usr) + return err } // checkForOtherLocks looks for other locks that currently exist in the repository. diff --git a/lock_unix.go b/lock_unix.go new file mode 100644 index 000000000..851071ab2 --- /dev/null +++ b/lock_unix.go @@ -0,0 +1,24 @@ +// +build !windows + +package restic + +import ( + "os/user" + "strconv" +) + +// uidGidInt returns uid, gid of the user as a number. +func uidGidInt(u user.User) (uid, gid uint32, err error) { + var ui, gi int + ui, err = strconv.ParseInt(u.Uid, 10, 32) + if err != nil { + return + } + gi, err = strconv.ParseInt(u.Gid, 10, 32) + if err != nil { + return + } + uid = uint32(ui) + gid = uint32(gi) + return +} diff --git a/lock_windows.go b/lock_windows.go new file mode 100644 index 000000000..2e0c68b48 --- /dev/null +++ b/lock_windows.go @@ -0,0 +1,10 @@ +package restic + +import ( + "os/user" +) + +// uidGidInt always returns 0 on Windows, since uid isn't numbers +func uidGidInt(u user.User) (uid, gid uint32, err error) { + return 0, 0, nil +} diff --git a/snapshot.go b/snapshot.go index 1693d2b75..91c40a62d 100644 --- a/snapshot.go +++ b/snapshot.go @@ -5,7 +5,6 @@ import ( "os" "os/user" "path/filepath" - "strconv" "time" "github.com/restic/restic/backend" @@ -76,15 +75,9 @@ func (sn *Snapshot) fillUserInfo() error { } sn.Username = usr.Username - // We ignore the error. On Windows Uid is not a number - uid, _ := strconv.ParseInt(usr.Uid, 10, 32) - sn.UID = uint32(uid) - - // We ignore the error. On Windows Gid is not a number - gid, _ := strconv.ParseInt(usr.Gid, 10, 32) - sn.GID = uint32(gid) - - return nil + // set userid and groupid + sn.UID, sn.GID, err = uidGidInt(*usr) + return err } // FindSnapshot takes a string and tries to find a snapshot whose ID matches From 458e4bb3f7e2b3c64c0eb3bcaf7b9bc80c5116b6 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 13:21:00 +0200 Subject: [PATCH 13/24] Remove empty lines. --- node_unix.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/node_unix.go b/node_unix.go index 5c0c4d63c..eec07fc5a 100644 --- a/node_unix.go +++ b/node_unix.go @@ -1,5 +1,3 @@ -// - // +build dragonfly linux netbsd openbsd freebsd solaris darwin package restic From 3f992c3d6e5b451277971c78fd7b58c7ef02820f Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 13:22:27 +0200 Subject: [PATCH 14/24] Remove redundant code. --- node_windows.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/node_windows.go b/node_windows.go index 7d5854b80..8a210aa01 100644 --- a/node_windows.go +++ b/node_windows.go @@ -6,11 +6,7 @@ import ( ) func (node *Node) OpenForReading() (*os.File, error) { - file, err := os.OpenFile(node.path, os.O_RDONLY, 0) - if os.IsPermission(err) { - return os.OpenFile(node.path, os.O_RDONLY, 0) - } - return file, err + return os.OpenFile(node.path, os.O_RDONLY, 0) } // mknod() creates a filesystem node (file, device From 346c0c0c587f829157b1407c692426d1f0277ed4 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 13:24:21 +0200 Subject: [PATCH 15/24] Don't panic on mknod on Windows Instead return an error. --- node_windows.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node_windows.go b/node_windows.go index 8a210aa01..5a5a9dba3 100644 --- a/node_windows.go +++ b/node_windows.go @@ -1,6 +1,7 @@ package restic import ( + "errors" "os" "syscall" ) @@ -13,7 +14,7 @@ func (node *Node) OpenForReading() (*os.File, error) { // special file, or named pipe) named pathname, with attributes // specified by mode and dev. var mknod = func(path string, mode uint32, dev int) (err error) { - panic("mknod not implemented") + return errors.New("device nodes cannot be created on windows") } // Windows doesn't need lchown From 2291e703226741c3c392c944796bebf3626f0a95 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 13:36:21 +0200 Subject: [PATCH 16/24] Match ParseInt return type. --- lock_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lock_unix.go b/lock_unix.go index 851071ab2..04d5005ba 100644 --- a/lock_unix.go +++ b/lock_unix.go @@ -9,7 +9,7 @@ import ( // uidGidInt returns uid, gid of the user as a number. func uidGidInt(u user.User) (uid, gid uint32, err error) { - var ui, gi int + var ui, gi int64 ui, err = strconv.ParseInt(u.Uid, 10, 32) if err != nil { return From d6935d662550a0d28d07903a969de932fb575305 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 14:16:40 +0200 Subject: [PATCH 17/24] Fix log message 'inode' isn't tested, so we don't want to imply that in the debug message. --- node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node.go b/node.go index 974f83834..e79306d5d 100644 --- a/node.go +++ b/node.go @@ -392,7 +392,7 @@ func (node *Node) isNewer(path string, fi os.FileInfo) bool { if !ok { if node.ModTime != fi.ModTime() || node.Size != size { - debug.Log("node.isNewer", "node %v is newer: timestamp, size or inode changed", path) + debug.Log("node.isNewer", "node %v is newer: timestamp or size changed", path) return true } return false From 347e800b4e4e29a1dd1c7b6ab8f486c886c6ea1a Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 14:20:37 +0200 Subject: [PATCH 18/24] Set output binary name in a variable. --- build.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.go b/build.go index 94d245c4c..c62cd8574 100644 --- a/build.go +++ b/build.go @@ -259,15 +259,15 @@ func main() { version := getVersion() compileTime := time.Now().Format(timeFormat) + output := "restic" + if runtime.GOOS == "windows" { + output = "restic.exe" + } + args := []string{ "-tags", strings.Join(buildTags, " "), "-ldflags", fmt.Sprintf(`-s -X main.version %q -X main.compiledAt %q`, version, compileTime), - } - if runtime.GOOS != "windows" { - args = append(args, "-o", "restic", "github.com/restic/restic/cmd/restic") - - } else { - args = append(args, "-o", "restic.exe", "github.com/restic/restic/cmd/restic") + "-o", output, "github.com/restic/restic/cmd/restic", } err = build(gopath, args...) From 35bd8f80c0f4be5d8fd49f65a5cb92b03c0005f3 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 16 Aug 2015 15:30:36 +0200 Subject: [PATCH 19/24] Split out process check as separate function. This will allow the checks to be changed for different operating systems. Issue #260 is related to this, but this does not change any current behaviour. --- lock.go | 19 ++++--------------- lock_unix.go | 24 ++++++++++++++++++++++++ lock_windows.go | 15 +++++++++++++++ 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/lock.go b/lock.go index dd24e8b15..0d4000fec 100644 --- a/lock.go +++ b/lock.go @@ -12,7 +12,6 @@ import ( "github.com/restic/restic/backend" "github.com/restic/restic/debug" "github.com/restic/restic/repository" - "runtime" ) // Lock represents a process locking the repository for an operation. @@ -195,22 +194,12 @@ func (l *Lock) Stale() bool { return true } - proc, err := os.FindProcess(l.PID) - if err != nil { - debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) + // check if we can reach the process retaining the lock + exists := l.processExists() + if !exists { + debug.Log("Lock.Stale", "could not reach process, %d, lock is probably stale\n", l.PID) return true } - defer proc.Release() - - // Windows does not have SIGHUP - if runtime.GOOS != "windows" { - debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) - err = proc.Signal(syscall.SIGHUP) - if err != nil { - debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) - return true - } - } debug.Log("Lock.Stale", "lock not stale\n") return false diff --git a/lock_unix.go b/lock_unix.go index 04d5005ba..aaf0cfdd5 100644 --- a/lock_unix.go +++ b/lock_unix.go @@ -3,8 +3,12 @@ package restic import ( + "os" "os/user" "strconv" + "syscall" + + "github.com/restic/restic/debug" ) // uidGidInt returns uid, gid of the user as a number. @@ -22,3 +26,23 @@ func uidGidInt(u user.User) (uid, gid uint32, err error) { gid = uint32(gi) return } + +// checkProcess will check if the process retaining the lock +// exists and responds to SIGHUP signal. +// Returns true if the process exists and responds. +func (l Lock) processExists() bool { + proc, err := os.FindProcess(l.PID) + if err != nil { + debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) + return false + } + defer proc.Release() + + debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) + err = proc.Signal(syscall.SIGHUP) + if err != nil { + debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) + return false + } + return true +} diff --git a/lock_windows.go b/lock_windows.go index 2e0c68b48..fc700a9b5 100644 --- a/lock_windows.go +++ b/lock_windows.go @@ -1,10 +1,25 @@ package restic import ( + "os" "os/user" + + "github.com/restic/restic/debug" ) // uidGidInt always returns 0 on Windows, since uid isn't numbers func uidGidInt(u user.User) (uid, gid uint32, err error) { return 0, 0, nil } + +// checkProcess will check if the process retaining the lock exists. +// Returns true if the process exists. +func (l Lock) processExists() bool { + proc, err := os.FindProcess(l.PID) + if err != nil { + debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) + return false + } + proc.Release() + return true +} From 73de59a615256579502f27f360ee358c7e3d9119 Mon Sep 17 00:00:00 2001 From: klauspost Date: Mon, 17 Aug 2015 11:01:24 +0200 Subject: [PATCH 20/24] Make check for non-existing paths OS independent. --- restorer.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/restorer.go b/restorer.go index 524a9c734..d9c6ba0e3 100644 --- a/restorer.go +++ b/restorer.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "syscall" "github.com/restic/restic/backend" "github.com/restic/restic/debug" @@ -96,16 +95,13 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error { } // Did it fail because of ENOENT? - if pe, ok := errors.Cause(err).(*os.PathError); ok { - errn, ok := pe.Err.(syscall.Errno) - if ok && errn == syscall.ENOENT { - debug.Log("Restorer.restoreNodeTo", "create intermediate paths") + if err != nil && os.IsNotExist(errors.Cause(err)) { + debug.Log("Restorer.restoreNodeTo", "create intermediate paths") - // Create parent directories and retry - err = os.MkdirAll(filepath.Dir(dstPath), 0700) - if err == nil || err == os.ErrExist { - err = node.CreateAt(dstPath, res.repo) - } + // Create parent directories and retry + err = os.MkdirAll(filepath.Dir(dstPath), 0700) + if err == nil || err == os.ErrExist { + err = node.CreateAt(dstPath, res.repo) } } From d5dab39a4ac50bc8964211f3c9426a4da5ce55c5 Mon Sep 17 00:00:00 2001 From: klauspost Date: Mon, 17 Aug 2015 11:02:04 +0200 Subject: [PATCH 21/24] Disable FUSE test on Windows. --- cmd/restic/integration_fuse_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index 897e1fc75..5a6d26ac9 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -1,4 +1,5 @@ // +build !openbsd +// +build !windows package main From a3570af5001e083b58d4d7f11b0b75f3b8d24b1b Mon Sep 17 00:00:00 2001 From: klauspost Date: Mon, 17 Aug 2015 11:10:12 +0200 Subject: [PATCH 22/24] Create separate disEntry.equals for Windows. Windows does not have UID/GID the same way as unix, so we don't attempt to compare them. --- cmd/restic/integration_helpers_test.go | 35 +--------------- cmd/restic/integration_helpers_unix_test.go | 41 +++++++++++++++++++ .../integration_helpers_windows_test.go | 27 ++++++++++++ 3 files changed, 70 insertions(+), 33 deletions(-) create mode 100644 cmd/restic/integration_helpers_unix_test.go create mode 100644 cmd/restic/integration_helpers_windows_test.go diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 6fd90ac3f..a31198a53 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "runtime" - "syscall" "testing" . "github.com/restic/restic/test" @@ -70,38 +69,6 @@ func sameModTime(fi1, fi2 os.FileInfo) bool { return fi1.ModTime() == fi2.ModTime() } -func (e *dirEntry) equals(other *dirEntry) bool { - if e.path != other.path { - fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) - return false - } - - if e.fi.Mode() != other.fi.Mode() { - fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) - return false - } - - if !sameModTime(e.fi, other.fi) { - fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) - return false - } - - stat, _ := e.fi.Sys().(*syscall.Stat_t) - stat2, _ := other.fi.Sys().(*syscall.Stat_t) - - if stat.Uid != stat2.Uid { - fmt.Fprintf(os.Stderr, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid) - return false - } - - if stat.Gid != stat2.Gid { - fmt.Fprintf(os.Stderr, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid) - return false - } - - return true -} - // directoriesEqualContents checks if both directories contain exactly the same // contents. func directoriesEqualContents(dir1, dir2 string) bool { @@ -237,6 +204,8 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) } OK(t, os.MkdirAll(env.testdata, 0700)) + OK(t, os.MkdirAll(env.cache, 0700)) + OK(t, os.MkdirAll(env.repo, 0700)) f(&env, configureRestic(t, env.cache, env.repo)) diff --git a/cmd/restic/integration_helpers_unix_test.go b/cmd/restic/integration_helpers_unix_test.go new file mode 100644 index 000000000..a182898e8 --- /dev/null +++ b/cmd/restic/integration_helpers_unix_test.go @@ -0,0 +1,41 @@ +//+build !windows + +package main + +import ( + "fmt" + "os" + "syscall" +) + +func (e *dirEntry) equals(other *dirEntry) bool { + if e.path != other.path { + fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) + return false + } + + if e.fi.Mode() != other.fi.Mode() { + fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) + return false + } + + if !sameModTime(e.fi, other.fi) { + fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) + return false + } + + stat, _ := e.fi.Sys().(*syscall.Stat_t) + stat2, _ := other.fi.Sys().(*syscall.Stat_t) + + if stat.Uid != stat2.Uid { + fmt.Fprintf(os.Stderr, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid) + return false + } + + if stat.Gid != stat2.Gid { + fmt.Fprintf(os.Stderr, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid) + return false + } + + return true +} diff --git a/cmd/restic/integration_helpers_windows_test.go b/cmd/restic/integration_helpers_windows_test.go new file mode 100644 index 000000000..d67e9ca11 --- /dev/null +++ b/cmd/restic/integration_helpers_windows_test.go @@ -0,0 +1,27 @@ +//+build windows + +package main + +import ( + "fmt" + "os" +) + +func (e *dirEntry) equals(other *dirEntry) bool { + if e.path != other.path { + fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) + return false + } + + if e.fi.Mode() != other.fi.Mode() { + fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) + return false + } + + if !sameModTime(e.fi, other.fi) { + fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) + return false + } + + return true +} From 21ab5a488db5d4ed162d4d821d65efc9e05b7852 Mon Sep 17 00:00:00 2001 From: klauspost Date: Mon, 17 Aug 2015 11:48:24 +0200 Subject: [PATCH 23/24] Allow filter patterns/paths to be both '/' and os.PathSeparator This converts filepath.Separator to '/', if it isn't already. This allows both native paths and patterns, as well as a mix of them. Added tests. --- filter/filter.go | 10 ++++++++-- filter/filter_test.go | 42 ++++++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/filter/filter.go b/filter/filter.go index f8c335e34..4092b4025 100644 --- a/filter/filter.go +++ b/filter/filter.go @@ -25,8 +25,14 @@ func Match(pattern, str string) (matched bool, err error) { return false, ErrBadString } - patterns := strings.Split(pattern, string(filepath.Separator)) - strs := strings.Split(str, string(filepath.Separator)) + // convert file path separator to '/' + if filepath.Separator != '/' { + pattern = strings.Replace(pattern, string(filepath.Separator), "/", -1) + str = strings.Replace(str, string(filepath.Separator), "/", -1) + } + + patterns := strings.Split(pattern, "/") + strs := strings.Split(str, "/") return match(patterns, strs) } diff --git a/filter/filter_test.go b/filter/filter_test.go index 78e731b68..15892e910 100644 --- a/filter/filter_test.go +++ b/filter/filter_test.go @@ -5,6 +5,8 @@ import ( "compress/bzip2" "fmt" "os" + "path/filepath" + "strings" "testing" "github.com/restic/restic/filter" @@ -71,20 +73,40 @@ var matchTests = []struct { {"foo/**/bar", "/home/user/foo/x/y/bar/main.go", true}, {"user/**/important*", "/home/user/work/x/y/hidden/x", false}, {"user/**/hidden*/**/c", "/home/user/work/x/y/hidden/z/a/b/c", true}, + {"c:/foo/*test.*", "c:/foo/bar/test.go", false}, + {"c:/foo/*/test.*", "c:/foo/bar/test.go", true}, + {"c:/foo/*/bar/test.*", "c:/foo/bar/test.go", false}, +} + +func testpattern(t *testing.T, pattern, path string, shouldMatch bool) { + match, err := filter.Match(pattern, path) + if err != nil { + t.Errorf("test pattern %q failed: expected no error for path %q, but error returned: %v", + pattern, path, err) + } + + if match != shouldMatch { + t.Errorf("test: filter.Match(%q, %q): expected %v, got %v", + pattern, path, shouldMatch, match) + } } func TestMatch(t *testing.T) { - for i, test := range matchTests { - match, err := filter.Match(test.pattern, test.path) - if err != nil { - t.Errorf("test %d failed: expected no error for pattern %q, but error returned: %v", - i, test.pattern, err) - continue - } + for _, test := range matchTests { + testpattern(t, test.pattern, test.path, test.match) - if match != test.match { - t.Errorf("test %d: filter.Match(%q, %q): expected %v, got %v", - i, test.pattern, test.path, test.match, match) + // Test with native path separator + if filepath.Separator != '/' { + // Test with pattern as native + pattern := strings.Replace(test.pattern, "/", string(filepath.Separator), -1) + testpattern(t, pattern, test.path, test.match) + + // Test with path as native + path := strings.Replace(test.path, "/", string(filepath.Separator), -1) + testpattern(t, test.pattern, path, test.match) + + // Test with both pattern and path as native + testpattern(t, pattern, path, test.match) } } } From 8a2d7ff2bcf062b4063030e31de7fd9e6d2e029b Mon Sep 17 00:00:00 2001 From: klauspost Date: Mon, 17 Aug 2015 14:17:54 +0200 Subject: [PATCH 24/24] Add default cache location for Windows. Primary place for Windows cache is %APPDATA%\restic. If that environment variable isn't set, we create a 'restic' folder in the 'temp' directory. --- cache.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cache.go b/cache.go index 4cdde46f0..8dfd9ff84 100644 --- a/cache.go +++ b/cache.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "github.com/restic/restic/backend" @@ -212,10 +213,40 @@ func getCacheDir() (string, error) { if dir := os.Getenv("RESTIC_CACHE"); dir != "" { return dir, nil } + if runtime.GOOS == "windows" { + return getWindowsCacheDir() + } return getXDGCacheDir() } +// getWindowsCacheDir will return %APPDATA%\restic or create +// a folder in the temporary folder called "restic". +func getWindowsCacheDir() (string, error) { + cachedir := os.Getenv("APPDATA") + if cachedir == "" { + cachedir = os.TempDir() + } + cachedir = filepath.Join(cachedir, "restic") + fi, err := os.Stat(cachedir) + + if os.IsNotExist(err) { + err = os.MkdirAll(cachedir, 0700) + if err != nil { + return "", err + } + } + + if err != nil { + return "", err + } + + if !fi.IsDir() { + return "", fmt.Errorf("cache dir %v is not a directory", cachedir) + } + return cachedir, nil +} + // getXDGCacheDir returns the cache directory according to XDG basedir spec, see // http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html func getXDGCacheDir() (string, error) {