Merge pull request #509 from restic/read-from-stdin

Allow reading data from stdin

doc/Manual.md

@@ -182,6 +182,22 @@ see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
 Additionally `**` exludes arbitrary subdirectories.  
 Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
 
+## Reading data from stdin
+
+Sometimes it can be nice to directly save the output of a program, e.g.
+`mysqldump` so that the SQL can later be restored. Restic supports this mode of
+operation, just supply the option `--stdin` to the `backup` command like this:
+
+    $ mysqldump [...] | restic -r /tmp/backup backup --stdin
+
+This creates a new snapshot of the output of `mqsqldump`. You can then use e.g.
+the fuse mounting option (see below) to mount the repository and read the file.
+
+By default, the file name `stdin` is used, a different name can be specified
+with `--stdin-filename`, e.g. like this:
+
+    $ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filenam production.sql
+
 # List all snapshots
 
 Now, you can list all the snapshots stored in the repository:

src/cmds/restic/cmd_backup.go

@@ -18,10 +18,12 @@ import (
 )
 
 type CmdBackup struct {
-	Parent      string   `short:"p" long:"parent"  description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
-	Force       bool     `short:"f" long:"force"   description:"Force re-reading the target. Overrides the \"parent\" flag"`
-	Excludes    []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
-	ExcludeFile string   `long:"exclude-file" description:"Read exclude-patterns from file"`
+	Parent        string   `short:"p" long:"parent"                  description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
+	Force         bool     `short:"f" long:"force"                   description:"Force re-reading the target. Overrides the \"parent\" flag"`
+	Excludes      []string `short:"e" long:"exclude"                 description:"Exclude a pattern (can be specified multiple times)"`
+	ExcludeFile   string   `long:"exclude-file"                      description:"Read exclude-patterns from file"`
+	Stdin         bool     `long:"stdin"                             description:"read backup data from stdin"`
+	StdinFilename string   `long:"stdin-filename"    default:"stdin" description:"file name to use when reading from stdin"`
 
 	global *GlobalOptions
 }
@@ -175,6 +177,47 @@ func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress {
 	return archiveProgress
 }
 
+func (cmd CmdBackup) newArchiveStdinProgress() *restic.Progress {
+	if !cmd.global.ShowProgress() {
+		return nil
+	}
+
+	archiveProgress := restic.NewProgress(time.Second)
+
+	var bps uint64
+
+	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		sec := uint64(d / time.Second)
+		if s.Bytes > 0 && sec > 0 && ticker {
+			bps = s.Bytes / sec
+		}
+
+		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
+			formatBytes(s.Bytes),
+			formatBytes(bps))
+
+		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+		if err == nil {
+			maxlen := w - len(status1)
+
+			if maxlen < 4 {
+				status1 = ""
+			} else if len(status1) > maxlen {
+				status1 = status1[:maxlen-4]
+				status1 += "... "
+			}
+		}
+
+		fmt.Printf("\x1b[2K%s\r", status1)
+	}
+
+	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
+		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
+	}
+
+	return archiveProgress
+}
+
 func samePaths(expected, actual []string) bool {
 	if expected == nil || actual == nil {
 		return true
@@ -243,7 +286,41 @@ func filterExisting(items []string) (result []string, err error) {
 	return
 }
 
+func (cmd CmdBackup) readFromStdin(args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("when reading from stdin, no additional files can be specified")
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	_, id, err := restic.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("archived as %v\n", id.Str())
+	return nil
+}
+
 func (cmd CmdBackup) Execute(args []string) error {
+	if cmd.Stdin {
+		return cmd.readFromStdin(args)
+	}
+
 	if len(args) == 0 {
 		return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
 	}

src/restic/archive_reader.go

@@ -0,0 +1,122 @@
+package restic
+
+import (
+	"encoding/json"
+	"io"
+	"restic/backend"
+	"restic/debug"
+	"restic/pack"
+	"restic/repository"
+	"time"
+
+	"github.com/restic/chunker"
+)
+
+// saveTreeJSON stores a tree in the repository.
+func saveTreeJSON(repo *repository.Repository, item interface{}) (backend.ID, error) {
+	data, err := json.Marshal(item)
+	if err != nil {
+		return backend.ID{}, err
+	}
+	data = append(data, '\n')
+
+	// check if tree has been saved before
+	id := backend.Hash(data)
+	if repo.Index().Has(id) {
+		return id, nil
+	}
+
+	return repo.SaveJSON(pack.Tree, item)
+}
+
+// ArchiveReader reads from the reader and archives the data. Returned is the
+// resulting snapshot and its ID.
+func ArchiveReader(repo *repository.Repository, p *Progress, rd io.Reader, name string) (*Snapshot, backend.ID, error) {
+	debug.Log("ArchiveReader", "start archiving %s", name)
+	sn, err := NewSnapshot([]string{name})
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	p.Start()
+	defer p.Done()
+
+	chnker := chunker.New(rd, repo.Config.ChunkerPolynomial)
+
+	var ids backend.IDs
+	var fileSize uint64
+
+	for {
+		chunk, err := chnker.Next(getBuf())
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return nil, backend.ID{}, err
+		}
+
+		id := backend.Hash(chunk.Data)
+
+		if !repo.Index().Has(id) {
+			_, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil)
+			if err != nil {
+				return nil, backend.ID{}, err
+			}
+			debug.Log("ArchiveReader", "saved blob %v (%d bytes)\n", id.Str(), chunk.Length)
+		} else {
+			debug.Log("ArchiveReader", "blob %v already saved in the repo\n", id.Str())
+		}
+
+		freeBuf(chunk.Data)
+
+		ids = append(ids, id)
+
+		p.Report(Stat{Bytes: uint64(chunk.Length)})
+		fileSize += uint64(chunk.Length)
+	}
+
+	tree := &Tree{
+		Nodes: []*Node{
+			&Node{
+				Name:       name,
+				AccessTime: time.Now(),
+				ModTime:    time.Now(),
+				Type:       "file",
+				Mode:       0644,
+				Size:       fileSize,
+				UID:        sn.UID,
+				GID:        sn.GID,
+				User:       sn.Username,
+				Content:    ids,
+			},
+		},
+	}
+
+	treeID, err := saveTreeJSON(repo, tree)
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+	sn.Tree = &treeID
+	debug.Log("ArchiveReader", "tree saved as %v", treeID.Str())
+
+	id, err := repo.SaveJSONUnpacked(backend.Snapshot, sn)
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	sn.id = &id
+	debug.Log("ArchiveReader", "snapshot saved as %v", id.Str())
+
+	err = repo.Flush()
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	err = repo.SaveIndex()
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	return sn, id, nil
+}

src/restic/archive_reader_test.go

@@ -0,0 +1,103 @@
+package restic
+
+import (
+	"bytes"
+	"io"
+	"math/rand"
+	"restic/backend"
+	"restic/pack"
+	"restic/repository"
+	"testing"
+
+	"github.com/restic/chunker"
+)
+
+func loadBlob(t *testing.T, repo *repository.Repository, id backend.ID, buf []byte) []byte {
+	buf, err := repo.LoadBlob(pack.Data, id, buf)
+	if err != nil {
+		t.Fatalf("LoadBlob(%v) returned error %v", id, err)
+	}
+
+	return buf
+}
+
+func checkSavedFile(t *testing.T, repo *repository.Repository, treeID backend.ID, name string, rd io.Reader) {
+	tree, err := LoadTree(repo, treeID)
+	if err != nil {
+		t.Fatalf("LoadTree() returned error %v", err)
+	}
+
+	if len(tree.Nodes) != 1 {
+		t.Fatalf("wrong number of nodes for tree, want %v, got %v", 1, len(tree.Nodes))
+	}
+
+	node := tree.Nodes[0]
+	if node.Name != "fakefile" {
+		t.Fatalf("wrong filename, want %v, got %v", "fakefile", node.Name)
+	}
+
+	if len(node.Content) == 0 {
+		t.Fatalf("node.Content has length 0")
+	}
+
+	// check blobs
+	buf := make([]byte, chunker.MaxSize)
+	buf2 := make([]byte, chunker.MaxSize)
+	for i, id := range node.Content {
+		buf = loadBlob(t, repo, id, buf)
+
+		buf2 = buf2[:len(buf)]
+		_, err = io.ReadFull(rd, buf2)
+
+		if !bytes.Equal(buf, buf2) {
+			t.Fatalf("blob %d (%v) is wrong", i, id.Str())
+		}
+	}
+}
+
+func TestArchiveReader(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	seed := rand.Int63()
+	size := int64(rand.Intn(50*1024*1024) + 50*1024*1024)
+	t.Logf("seed is 0x%016x, size is %v", seed, size)
+
+	f := fakeFile(t, seed, size)
+
+	sn, id, err := ArchiveReader(repo, nil, f, "fakefile")
+	if err != nil {
+		t.Fatalf("ArchiveReader() returned error %v", err)
+	}
+
+	if id.IsNull() {
+		t.Fatalf("ArchiveReader() returned null ID")
+	}
+
+	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
+
+	checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size))
+}
+
+func BenchmarkArchiveReader(t *testing.B) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	const size = 50 * 1024 * 1024
+
+	buf := make([]byte, size)
+	_, err := io.ReadFull(fakeFile(t, 23, size), buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.SetBytes(size)
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}

src/restic/fuse/dir.go

@@ -11,6 +11,7 @@ import (
 	"golang.org/x/net/context"
 
 	"restic"
+	"restic/debug"
 	"restic/repository"
 )
 
@@ -27,8 +28,10 @@ type dir struct {
 }
 
 func newDir(repo *repository.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) {
+	debug.Log("newDir", "new dir for %v (%v)", node.Name, node.Subtree.Str())
 	tree, err := restic.LoadTree(repo, *node.Subtree)
 	if err != nil {
+		debug.Log("newDir", "  error loading tree %v: %v", node.Subtree.Str(), err)
 		return nil, err
 	}
 	items := make(map[string]*restic.Node)
@@ -65,14 +68,17 @@ func replaceSpecialNodes(repo *repository.Repository, node *restic.Node) ([]*res
 }
 
 func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) {
+	debug.Log("newDirFromSnapshot", "new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str())
 	tree, err := restic.LoadTree(repo, *snapshot.Tree)
 	if err != nil {
+		debug.Log("newDirFromSnapshot", "  loadTree(%v) failed: %v", snapshot.ID.Str(), err)
 		return nil, err
 	}
 	items := make(map[string]*restic.Node)
 	for _, n := range tree.Nodes {
 		nodes, err := replaceSpecialNodes(repo, n)
 		if err != nil {
+			debug.Log("newDirFromSnapshot", "  replaceSpecialNodes(%v) failed: %v", n, err)
 			return nil, err
 		}
 
@@ -98,6 +104,7 @@ func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ow
 }
 
 func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
+	debug.Log("dir.Attr", "called")
 	a.Inode = d.inode
 	a.Mode = os.ModeDir | d.node.Mode
 
@@ -112,6 +119,7 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
 }
 
 func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
+	debug.Log("dir.ReadDirAll", "called")
 	ret := make([]fuse.Dirent, 0, len(d.items))
 
 	for _, node := range d.items {
@@ -136,8 +144,10 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 }
 
 func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
+	debug.Log("dir.Lookup", "Lookup(%v)", name)
 	node, ok := d.items[name]
 	if !ok {
+		debug.Log("dir.Lookup", "  Lookup(%v) -> not found", name)
 		return nil, fuse.ENOENT
 	}
 	switch node.Type {
@@ -148,6 +158,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 	case "symlink":
 		return newLink(d.repo, node, d.ownerIsRoot)
 	default:
+		debug.Log("dir.Lookup", "  node %v has unknown type %v", name, node.Type)
 		return nil, fuse.ENOENT
 	}
 }

src/restic/fuse/file.go

@@ -8,6 +8,7 @@ import (
 
 	"restic"
 	"restic/backend"
+	"restic/debug"
 	"restic/pack"
 
 	"bazil.org/fuse"
@@ -47,6 +48,8 @@ var blobPool = sync.Pool{
 }
 
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
+	debug.Log("newFile", "create new file for %v with %d blobs", node.Name, len(node.Content))
+	var bytes uint64
 	sizes := make([]uint, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id)
@@ -55,6 +58,12 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error
 		}
 
 		sizes[i] = size
+		bytes += uint64(size)
+	}
+
+	if bytes != node.Size {
+		debug.Log("newFile", "sizes do not match: node.Size %v != size %v, using real size", node.Size, bytes)
+		node.Size = bytes
 	}
 
 	return &file{
@@ -67,6 +76,7 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error
 }
 
 func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
+	debug.Log("file.Attr", "Attr(%v)", f.node.Name)
 	a.Inode = f.node.Inode
 	a.Mode = f.node.Mode
 	a.Size = f.node.Size
@@ -84,6 +94,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 }
 
 func (f *file) getBlobAt(i int) (blob []byte, err error) {
+	debug.Log("file.getBlobAt", "getBlobAt(%v, %v)", f.node.Name, i)
 	if f.blobs[i] != nil {
 		return f.blobs[i], nil
 	}
@@ -100,6 +111,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 
 	blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf)
 	if err != nil {
+		debug.Log("file.getBlobAt", "LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
 		return nil, err
 	}
 	f.blobs[i] = blob
@@ -108,6 +120,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 }
 
 func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
+	debug.Log("file.Read", "Read(%v), file size %v", req.Size, f.node.Size)
 	offset := req.Offset
 
 	// Skip blobs before the offset

src/restic/fuse/snapshot.go

@@ -13,6 +13,7 @@ import (
 
 	"restic"
 	"restic/backend"
+	"restic/debug"
 	"restic/repository"
 
 	"golang.org/x/net/context"
@@ -39,6 +40,7 @@ type SnapshotsDir struct {
 }
 
 func NewSnapshotsDir(repo *repository.Repository, ownerIsRoot bool) *SnapshotsDir {
+	debug.Log("NewSnapshotsDir", "fuse mount initiated")
 	return &SnapshotsDir{
 		repo:           repo,
 		knownSnapshots: make(map[string]SnapshotWithId),
@@ -54,10 +56,12 @@ func (sn *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
 		attr.Uid = uint32(os.Getuid())
 		attr.Gid = uint32(os.Getgid())
 	}
+	debug.Log("SnapshotsDir.Attr", "attr is %v", attr)
 	return nil
 }
 
 func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
+	debug.Log("SnapshotsDir.updateCache", "called")
 	sn.Lock()
 	defer sn.Unlock()
 
@@ -75,10 +79,12 @@ func (sn *SnapshotsDir) get(name string) (snapshot SnapshotWithId, ok bool) {
 	sn.RLock()
 	snapshot, ok = sn.knownSnapshots[name]
 	sn.RUnlock()
+	debug.Log("SnapshotsDir.get", "get(%s) -> %v %v", name, snapshot, ok)
 	return snapshot, ok
 }
 
 func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
+	debug.Log("SnapshotsDir.ReadDirAll", "called")
 	err := sn.updateCache(ctx)
 	if err != nil {
 		return nil, err
@@ -96,21 +102,25 @@ func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 		})
 	}
 
+	debug.Log("SnapshotsDir.ReadDirAll", "  -> %d entries", len(ret))
 	return ret, nil
 }
 
 func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
+	debug.Log("SnapshotsDir.updateCache", "Lookup(%s)", name)
 	snapshot, ok := sn.get(name)
 
 	if !ok {
 		// We don't know about it, update the cache
 		err := sn.updateCache(ctx)
 		if err != nil {
+			debug.Log("SnapshotsDir.updateCache", "  Lookup(%s) -> err %v", name, err)
 			return nil, err
 		}
 		snapshot, ok = sn.get(name)
 		if !ok {
 			// We still don't know about it, this time it really doesn't exist
+			debug.Log("SnapshotsDir.updateCache", "  Lookup(%s) -> not found", name)
 			return nil, fuse.ENOENT
 		}
 	}