Update Gopkg.lock

Gopkg.lock

@@ -178,7 +178,7 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/net"
-  packages = ["context","context/ctxhttp"]
+  packages = ["context","context/ctxhttp","webdav","webdav/internal/xml"]
   revision = "a8b9294777976932365dabb6640cf1468d95c70f"
 
 [[projects]]
@@ -214,6 +214,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "f0a207197cb502238ac87ca8e07b2640c02ec380a50b036e09ef87e40e31ca2d"
+  inputs-digest = "b01eeeb2be041c7cd11f9ee50324ef456ac1e1cd0720408c6d72f88f92f09320"
   solver-name = "gps-cdcl"
   solver-version = 1

changelog/0.8.2/pull-1554

@@ -0,0 +1,7 @@
+Enhancement: fuse/mount: Correctly handle EOF, add template option
+
+We've added the `--snapshot-template` string, which can be used to specify a
+template for a snapshot directory. In addition, accessing data after the end of
+a file via the fuse mount is now handled correctly.
+
+https://github.com/restic/restic/pull/1554

cmd/restic/cmd_mount.go

@@ -5,6 +5,8 @@ package main
 
 import (
 	"os"
+	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -25,6 +27,21 @@ var cmdMount = &cobra.Command{
 	Long: `
 The "mount" command mounts the repository via fuse to a directory. This is a
 read-only mount.
+
+Snapshot Directories
+====================
+
+If you need a different template for all directories that contain snapshots,
+you can pass a template via --snapshot-template. Example without colons:
+
+    --snapshot-template "2006-01-02_15-04-05"
+
+You need to specify a sample format for exactly the following timestamp:
+
+    Mon Jan 2 15:04:05 -0700 MST 2006
+
+For details please see the documentation for time.Format() at:
+  https://godoc.org/time#Time.Format
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -34,12 +51,13 @@ read-only mount.
 
 // MountOptions collects all options for the mount command.
 type MountOptions struct {
-	OwnerRoot  bool
-	AllowRoot  bool
-	AllowOther bool
-	Host       string
-	Tags       restic.TagLists
-	Paths      []string
+	OwnerRoot        bool
+	AllowRoot        bool
+	AllowOther       bool
+	Host             string
+	Tags             restic.TagLists
+	Paths            []string
+	SnapshotTemplate string
 }
 
 var mountOptions MountOptions
@@ -55,6 +73,8 @@ func init() {
 	mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
 	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
 	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
+
+	mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
 }
 
 func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@@ -108,10 +128,11 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 	}
 
 	cfg := fuse.Config{
-		OwnerIsRoot: opts.OwnerRoot,
-		Host:        opts.Host,
-		Tags:        opts.Tags,
-		Paths:       opts.Paths,
+		OwnerIsRoot:      opts.OwnerRoot,
+		Host:             opts.Host,
+		Tags:             opts.Tags,
+		Paths:            opts.Paths,
+		SnapshotTemplate: opts.SnapshotTemplate,
 	}
 	root, err := fuse.NewRoot(gopts.ctx, repo, cfg)
 	if err != nil {
@@ -136,6 +157,10 @@ func umount(mountpoint string) error {
 }
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
+	if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
+		return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
+	}
+
 	if len(args) == 0 {
 		return errors.Fatal("wrong number of parameters")
 	}

cmd/restic/cmd_webdav.go

@@ -0,0 +1,96 @@
+// +build !openbsd
+// +build !windows
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/serve"
+)
+
+var cmdWebDAV = &cobra.Command{
+	Use:   "webdav [flags]",
+	Short: "runs a WebDAV server for the repository",
+	Long: `
+The webdav command runs a WebDAV server for the reposiotry that you can then access via a WebDAV client.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runWebDAV(webdavOptions, globalOptions, args)
+	},
+}
+
+// WebDAVOptions collects all options for the webdav command.
+type WebDAVOptions struct {
+	Listen string
+
+	Host  string
+	Tags  restic.TagLists
+	Paths []string
+}
+
+var webdavOptions WebDAVOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdWebDAV)
+
+	webdavFlags := cmdWebDAV.Flags()
+	webdavFlags.StringVarP(&webdavOptions.Listen, "listen", "l", "localhost:3080", "set the listen host name and `address`")
+
+	webdavFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
+	webdavFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
+	webdavFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
+}
+
+func runWebDAV(opts WebDAVOptions, gopts GlobalOptions, args []string) error {
+	if len(args) > 0 {
+		return errors.Fatal("this command does not accept additional arguments")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
+	errorLogger := log.New(os.Stderr, "error log: ", log.Flags())
+
+	cfg := serve.Config{
+		Host:  opts.Host,
+		Tags:  opts.Tags,
+		Paths: opts.Paths,
+	}
+
+	h, err := serve.NewWebDAV(gopts.ctx, repo, cfg)
+	if err != nil {
+		return err
+	}
+
+	srv := &http.Server{
+		ReadTimeout:  60 * time.Second,
+		WriteTimeout: 60 * time.Second,
+		Addr:         opts.Listen,
+		Handler:      h,
+		ErrorLog:     errorLogger,
+	}
+
+	return srv.ListenAndServe()
+}

internal/fuse/root.go

@@ -16,10 +16,11 @@ import (
 
 // Config holds settings for the fuse mount.
 type Config struct {
-	OwnerIsRoot bool
-	Host        string
-	Tags        []restic.TagList
-	Paths       []string
+	OwnerIsRoot      bool
+	Host             string
+	Tags             []restic.TagList
+	Paths            []string
+	SnapshotTemplate string
 }
 
 // Root is the root node of the fuse mount of a repository.

internal/fuse/snapshots_dir.go

@@ -26,6 +26,8 @@ type SnapshotsDir struct {
 	tag     string
 	host    string
 	snCount int
+
+	template string
 }
 
 // SnapshotsIDSDir is a fuse directory which contains snapshots named by ids.
@@ -112,12 +114,13 @@ func updateSnapshotIDSNames(d *SnapshotsIDSDir) {
 func NewSnapshotsDir(root *Root, inode uint64, tag string, host string) *SnapshotsDir {
 	debug.Log("create snapshots dir, inode %d", inode)
 	d := &SnapshotsDir{
-		root:   root,
-		inode:  inode,
-		names:  make(map[string]*restic.Snapshot),
-		latest: "",
-		tag:    tag,
-		host:   host,
+		root:     root,
+		inode:    inode,
+		names:    make(map[string]*restic.Snapshot),
+		latest:   "",
+		tag:      tag,
+		host:     host,
+		template: root.cfg.SnapshotTemplate,
 	}
 
 	return d
@@ -239,7 +242,7 @@ func updateSnapshots(ctx context.Context, root *Root) {
 }
 
 // read snapshot timestamps from the current repository-state.
-func updateSnapshotNames(d *SnapshotsDir) {
+func updateSnapshotNames(d *SnapshotsDir, template string) {
 	if d.snCount != d.root.snCount {
 		d.snCount = d.root.snCount
 		var latestTime time.Time
@@ -248,7 +251,7 @@ func updateSnapshotNames(d *SnapshotsDir) {
 		for _, sn := range d.root.snapshots {
 			if d.tag == "" || isElem(d.tag, sn.Tags) {
 				if d.host == "" || d.host == sn.Hostname {
-					name := sn.Time.Format(time.RFC3339)
+					name := sn.Time.Format(template)
 					if d.latest == "" || !sn.Time.Before(latestTime) {
 						latestTime = sn.Time
 						d.latest = name
@@ -258,7 +261,7 @@ func updateSnapshotNames(d *SnapshotsDir) {
 							break
 						}
 
-						name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i)
+						name = fmt.Sprintf("%s-%d", sn.Time.Format(template), i)
 					}
 
 					d.names[name] = sn
@@ -276,7 +279,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	updateSnapshots(ctx, d.root)
 
 	// update snapshot names
-	updateSnapshotNames(d)
+	updateSnapshotNames(d, d.root.cfg.SnapshotTemplate)
 
 	items := []fuse.Dirent{
 		{
@@ -450,7 +453,7 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
 		updateSnapshots(ctx, d.root)
 
 		// update snapshot names
-		updateSnapshotNames(d)
+		updateSnapshotNames(d, d.root.cfg.SnapshotTemplate)
 
 		sn, ok := d.names[name]
 		if ok {

internal/serve/dir.go

@@ -0,0 +1,74 @@
+package serve
+
+import (
+	"io"
+	"os"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+	"golang.org/x/net/webdav"
+)
+
+// RepoDir implements a read-only directory from a repository.
+type RepoDir struct {
+	fi    os.FileInfo
+	nodes []*restic.Node
+}
+
+// statically ensure that RepoDir implements webdav.File
+var _ webdav.File = &RepoDir{}
+
+func (f *RepoDir) Write(p []byte) (int, error) {
+	return 0, webdav.ErrForbidden
+}
+
+// Close closes the repo file.
+func (f *RepoDir) Close() error {
+	debug.Log("")
+	return nil
+}
+
+// Read reads up to len(p) byte from the file.
+func (f *RepoDir) Read(p []byte) (int, error) {
+	debug.Log("")
+	return 0, io.EOF
+}
+
+// Seek sets the offset for the next Read or Write to offset, interpreted
+// according to whence: SeekStart means relative to the start of the file,
+// SeekCurrent means relative to the current offset, and SeekEnd means relative
+// to the end. Seek returns the new offset relative to the start of the file
+// and an error, if any.
+func (f *RepoDir) Seek(offset int64, whence int) (int64, error) {
+	debug.Log("")
+	return 0, webdav.ErrForbidden
+}
+
+// Readdir reads the contents of the directory associated with file and returns
+// a slice of up to n FileInfo values, as would be returned by Lstat, in
+// directory order. Subsequent calls on the same file will yield further
+// FileInfos.
+//
+// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
+// Readdir returns an empty slice, it will return a non-nil error explaining
+// why. At the end of a directory, the error is io.EOF.
+//
+// If n <= 0, Readdir returns all the FileInfo from the directory in a single
+// slice. In this case, if Readdir succeeds (reads all the way to the end of
+// the directory), it returns the slice and a nil error. If it encounters an
+// error before the end of the directory, Readdir returns the FileInfo read
+// until that point and a non-nil error.
+func (f *RepoDir) Readdir(count int) (entries []os.FileInfo, err error) {
+	debug.Log("count %d, %d nodes", count, len(f.nodes))
+
+	entries = make([]os.FileInfo, 0, len(f.nodes))
+	for _, node := range f.nodes {
+		entries = append(entries, fileInfoFromNode(node))
+	}
+	return entries, nil
+}
+
+// Stat returns a FileInfo describing the named file.
+func (f *RepoDir) Stat() (os.FileInfo, error) {
+	return f.fi, nil
+}

internal/serve/file.go

@@ -0,0 +1,67 @@
+package serve
+
+import (
+	"io"
+	"os"
+
+	"github.com/restic/restic/internal/restic"
+	"golang.org/x/net/webdav"
+)
+
+// RepoFile implements a read-only directory from a repository.
+type RepoFile struct {
+	fi   os.FileInfo
+	node *restic.Node
+}
+
+// statically ensure that RepoFile implements webdav.File
+var _ webdav.File = &RepoFile{}
+
+func (f *RepoFile) Write(p []byte) (int, error) {
+	return 0, webdav.ErrForbidden
+}
+
+// Close closes the repo file.
+func (f *RepoFile) Close() error {
+	return nil
+}
+
+// Read reads up to len(p) byte from the file.
+func (f *RepoFile) Read(p []byte) (int, error) {
+	// TODO
+	return 0, io.EOF
+}
+
+// Seek sets the offset for the next Read or Write to offset, interpreted
+// according to whence: SeekStart means relative to the start of the file,
+// SeekCurrent means relative to the current offset, and SeekEnd means relative
+// to the end. Seek returns the new offset relative to the start of the file
+// and an error, if any.
+func (f *RepoFile) Seek(offset int64, whence int) (int64, error) {
+	// TODO
+	return 0, io.EOF
+}
+
+// Readdir reads the contents of the directory associated with file and returns
+// a slice of up to n FileInfo values, as would be returned by Lstat, in
+// directory order. Subsequent calls on the same file will yield further
+// FileInfos.
+//
+// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
+// Readdir returns an empty slice, it will return a non-nil error explaining
+// why. At the end of a directory, the error is io.EOF.
+//
+// If n <= 0, Readdir returns all the FileInfo from the directory in a single
+// slice. In this case, if Readdir succeeds (reads all the way to the end of
+// the directory), it returns the slice and a nil error. If it encounters an
+// error before the end of the directory, Readdir returns the FileInfo read
+// until that point and a non-nil error.
+func (f *RepoFile) Readdir(count int) ([]os.FileInfo, error) {
+	// TODO
+	return nil, io.EOF
+}
+
+// Stat returns a FileInfo describing the named file.
+func (f *RepoFile) Stat() (os.FileInfo, error) {
+	return f.fi, nil
+}

internal/serve/fileinfo.go

@@ -0,0 +1,43 @@
+package serve
+
+import (
+	"os"
+	"time"
+
+	"github.com/restic/restic/internal/restic"
+)
+
+// virtFileInfo is used to construct an os.FileInfo for a server.
+type virtFileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modtime time.Time
+	isdir   bool
+}
+
+// statically ensure that virtFileInfo implements os.FileInfo.
+var _ os.FileInfo = virtFileInfo{}
+
+func (fi virtFileInfo) Name() string       { return fi.name }
+func (fi virtFileInfo) Size() int64        { return fi.size }
+func (fi virtFileInfo) Mode() os.FileMode  { return fi.mode }
+func (fi virtFileInfo) ModTime() time.Time { return fi.modtime }
+func (fi virtFileInfo) IsDir() bool        { return fi.isdir }
+func (fi virtFileInfo) Sys() interface{}   { return nil }
+
+func fileInfoFromNode(node *restic.Node) os.FileInfo {
+	fi := virtFileInfo{
+		name:    node.Name,
+		size:    int64(node.Size),
+		mode:    node.Mode,
+		modtime: node.ModTime,
+	}
+
+	if node.Type == "dir" {
+		fi.isdir = true
+		fi.mode |= os.ModeDir
+	}
+
+	return fi
+}

internal/serve/fs.go

@@ -0,0 +1,231 @@
+package serve
+
+import (
+	"context"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/restic/restic/internal/errors"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+	"golang.org/x/net/webdav"
+)
+
+// Config holds settings for the file system served.
+type Config struct {
+	Host  string
+	Tags  []restic.TagList
+	Paths []string
+}
+
+const snapshotFormat = "2006-01-02_150405"
+
+// RepoFileSystem implements a read-only file system on top of a repositoy.
+type RepoFileSystem struct {
+	repo      restic.Repository
+	lastCheck time.Time
+
+	entries map[string]webdav.File
+	m       sync.Mutex
+}
+
+// NewRepoFileSystem returns a new file system for the repo.
+func NewRepoFileSystem(ctx context.Context, repo restic.Repository, cfg Config) (*RepoFileSystem, error) {
+	snapshots := restic.FindFilteredSnapshots(ctx, repo, cfg.Host, cfg.Tags, cfg.Paths)
+
+	lastcheck := time.Now()
+
+	nodes := make([]*restic.Node, 0, len(snapshots))
+	entries := make(map[string]webdav.File)
+
+	for _, sn := range snapshots {
+		name := sn.Time.Format(snapshotFormat)
+		snFileInfo := virtFileInfo{
+			name:    name,
+			size:    0,
+			mode:    0755 | os.ModeDir,
+			modtime: sn.Time,
+			isdir:   true,
+		}
+
+		if sn.Tree == nil {
+			return nil, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
+		}
+
+		tree, err := repo.LoadTree(ctx, *sn.Tree)
+		if err != nil {
+			return nil, err
+		}
+
+		p := path.Join("/", name)
+		entries[p] = &RepoDir{
+			fi:    snFileInfo,
+			nodes: tree.Nodes,
+		}
+
+		nodes = append(nodes, &restic.Node{
+			Name: name,
+			Type: "dir",
+		})
+	}
+
+	entries["/"] = &RepoDir{
+		nodes: nodes,
+		fi: virtFileInfo{
+			name:    "root",
+			size:    0,
+			mode:    0755 | os.ModeDir,
+			modtime: lastcheck,
+			isdir:   true,
+		},
+	}
+
+	fs := &RepoFileSystem{
+		repo:      repo,
+		lastCheck: lastcheck,
+		entries:   entries,
+	}
+
+	return fs, nil
+}
+
+// statically ensure that RepoFileSystem implements webdav.FileSystem
+var _ webdav.FileSystem = &RepoFileSystem{}
+
+// Mkdir creates a new directory, it is not available for RepoFileSystem.
+func (fs *RepoFileSystem) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
+	return webdav.ErrForbidden
+}
+
+func (fs *RepoFileSystem) loadPath(ctx context.Context, name string) error {
+	debug.Log("%v", name)
+
+	fs.m.Lock()
+	_, ok := fs.entries[name]
+	fs.m.Unlock()
+	if ok {
+		return nil
+	}
+
+	dirname := path.Dir(name)
+	if dirname == "/" {
+		return nil
+	}
+
+	err := fs.loadPath(ctx, dirname)
+	if err != nil {
+		return err
+	}
+
+	entry, ok := fs.entries[dirname]
+	if !ok {
+		// loadPath did not succeed
+		return nil
+	}
+
+	repodir, ok := entry.(*RepoDir)
+	if !ok {
+		return nil
+	}
+
+	filename := path.Base(name)
+	for _, node := range repodir.nodes {
+		if node.Name != filename {
+			continue
+		}
+
+		debug.Log("found item %v :%v", filename, node)
+
+		switch node.Type {
+		case "dir":
+			if node.Subtree == nil {
+				return errors.Errorf("tree %v has nil tree", dirname)
+			}
+
+			tree, err := fs.repo.LoadTree(ctx, *node.Subtree)
+			if err != nil {
+				return err
+			}
+
+			newEntry := &RepoDir{
+				fi:    fileInfoFromNode(node),
+				nodes: tree.Nodes,
+			}
+
+			fs.m.Lock()
+			fs.entries[name] = newEntry
+			fs.m.Unlock()
+		case "file":
+			newEntry := &RepoFile{
+				fi:   fileInfoFromNode(node),
+				node: node,
+			}
+			fs.m.Lock()
+			fs.entries[name] = newEntry
+			fs.m.Unlock()
+		}
+
+		return nil
+	}
+
+	return nil
+}
+
+// OpenFile opens a file for reading.
+func (fs *RepoFileSystem) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
+	name = path.Clean(name)
+	debug.Log("%v", name)
+	if flag != os.O_RDONLY {
+		return nil, webdav.ErrForbidden
+	}
+
+	err := fs.loadPath(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fs.m.Lock()
+	entry, ok := fs.entries[name]
+	fs.m.Unlock()
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+
+	return entry, nil
+}
+
+// RemoveAll recursively removes files and directories, it is not available for RepoFileSystem.
+func (fs *RepoFileSystem) RemoveAll(ctx context.Context, name string) error {
+	debug.Log("%v", name)
+	return webdav.ErrForbidden
+}
+
+// Rename renames files or directories, it is not available for RepoFileSystem.
+func (fs *RepoFileSystem) Rename(ctx context.Context, oldName, newName string) error {
+	debug.Log("%v -> %v", oldName, newName)
+	return webdav.ErrForbidden
+}
+
+// Stat returns information on a file or directory.
+func (fs *RepoFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) {
+	name = path.Clean(name)
+	err := fs.loadPath(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fs.m.Lock()
+	entry, ok := fs.entries[name]
+	fs.m.Unlock()
+	if !ok {
+		debug.Log("%v not found", name)
+		return nil, os.ErrNotExist
+	}
+
+	fi, err := entry.Stat()
+	debug.Log("%v %v", name, fi)
+	return fi, err
+}

internal/serve/webdav.go

@@ -0,0 +1,46 @@
+package serve
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/restic/restic/internal/restic"
+	"golang.org/x/net/webdav"
+)
+
+// WebDAV implements a WebDAV handler on the repo.
+type WebDAV struct {
+	restic.Repository
+	webdav.Handler
+}
+
+var logger = log.New(os.Stderr, "webdav log: ", log.Flags())
+
+func logRequest(req *http.Request, err error) {
+	logger.Printf("req %v %v -> %v\n", req.Method, req.URL.Path, err)
+}
+
+// NewWebDAV returns a new *WebDAV which allows serving the repo via WebDAV.
+func NewWebDAV(ctx context.Context, repo restic.Repository, cfg Config) (*WebDAV, error) {
+	fs, err := NewRepoFileSystem(ctx, repo, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	wd := &WebDAV{
+		Repository: repo,
+		Handler: webdav.Handler{
+			FileSystem: fs,
+			LockSystem: webdav.NewMemLS(),
+			Logger:     logRequest,
+		},
+	}
+	return wd, nil
+}
+
+func (srv *WebDAV) ServeHTTP(res http.ResponseWriter, req *http.Request) {
+	logger.Printf("handle %v %v\n", req.Method, req.URL.Path)
+	srv.Handler.ServeHTTP(res, req)
+}