KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #2124 from Kidswiss/tar

Browse source 
Ability to dump folders to tar via stdout
This commit is contained in:
Alexander Neumann 2019-04-22 20:58:59 +02:00
commit 5979414bcd
6 changed files with 401 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
build.go
View file

@ -60,12 +60,12 @@ import (
// config contains the configuration for the program to build.
var config = Config{
Name: "restic", // name of the program executable and directory
Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar"
Main: "./cmd/restic", // package name for the main package
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported
Name: "restic", // name of the program executable and directory
Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar"
Main: "./cmd/restic", // package name for the main package
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 10, Patch: 0}, // minimum Go version supported
}
// Config configures the build.

8
changelog/unreleased/pull-2124 Normal file
View file

@ -0,0 +1,8 @@
Enhancement: Ability to dump folders to tar via stdout
We've added the ability to dump whole folders to stdout via the `dump` command.
Restic now requires at least Go 1.10 due to a limitation of the standard
library for Go <= 1.9.
https://github.com/restic/restic/pull/2124
https://github.com/restic/restic/issues/2123

131
cmd/restic/acl.go Normal file
View file

@ -0,0 +1,131 @@
package main
// Adapted from https://github.com/maxymania/go-system/blob/master/posix_acl/posix_acl.go
import (
"bytes"
"encoding/binary"
"fmt"
)
const (
aclUserOwner = 0x0001
aclUser = 0x0002
aclGroupOwner = 0x0004
aclGroup = 0x0008
aclMask = 0x0010
aclOthers = 0x0020
)
type aclSID uint64
type aclElem struct {
Tag uint16
Perm uint16
ID uint32
}
type acl struct {
Version uint32
List []aclElement
}
type aclElement struct {
aclSID
Perm uint16
}
func (a *aclSID) setUID(uid uint32) {
*a = aclSID(uid) | (aclUser << 32)
}
func (a *aclSID) setGID(gid uint32) {
*a = aclSID(gid) | (aclGroup << 32)
}
func (a *aclSID) setType(tp int) {
*a = aclSID(tp) << 32
}
func (a aclSID) getType() int {
return int(a >> 32)
}
func (a aclSID) getID() uint32 {
return uint32(a & 0xffffffff)
}
func (a aclSID) String() string {
switch a >> 32 {
case aclUserOwner:
return "user::"
case aclUser:
return fmt.Sprintf("user:%v:", a.getID())
case aclGroupOwner:
return "group::"
case aclGroup:
return fmt.Sprintf("group:%v:", a.getID())
case aclMask:
return "mask::"
case aclOthers:
return "other::"
}
return "?:"
}
func (a aclElement) String() string {
str := ""
if (a.Perm & 4) != 0 {
str += "r"
} else {
str += "-"
}
if (a.Perm & 2) != 0 {
str += "w"
} else {
str += "-"
}
if (a.Perm & 1) != 0 {
str += "x"
} else {
str += "-"
}
return fmt.Sprintf("%v%v", a.aclSID, str)
}
func (a *acl) decode(xattr []byte) {
var elem aclElement
ae := new(aclElem)
nr := bytes.NewReader(xattr)
e := binary.Read(nr, binary.LittleEndian, &a.Version)
if e != nil {
a.Version = 0
return
}
if len(a.List) > 0 {
a.List = a.List[:0]
}
for binary.Read(nr, binary.LittleEndian, ae) == nil {
elem.aclSID = (aclSID(ae.Tag) << 32) | aclSID(ae.ID)
elem.Perm = ae.Perm
a.List = append(a.List, elem)
}
}
func (a *acl) encode() []byte {
buf := new(bytes.Buffer)
ae := new(aclElem)
binary.Write(buf, binary.LittleEndian, &a.Version)
for _, elem := range a.List {
ae.Tag = uint16(elem.getType())
ae.Perm = elem.Perm
ae.ID = elem.getID()
binary.Write(buf, binary.LittleEndian, ae)
}
return buf.Bytes()
}
func (a *acl) String() string {
var finalacl string
for _, acl := range a.List {
finalacl += acl.String() + "\n"
}
return finalacl
}

96
cmd/restic/acl_test.go Normal file
View file

@ -0,0 +1,96 @@
package main
import (
"reflect"
"testing"
)
func Test_acl_decode(t *testing.T) {
type args struct {
xattr []byte
}
tests := []struct {
name string
args args
want string
}{
{
name: "decode string",
args: args{
xattr: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255},
},
want: "user::rw-\nuser:0:rwx\nuser:65534:rwx\ngroup::rwx\nmask::rwx\nother::r--\n",
},
{
name: "decode fail",
args: args{
xattr: []byte("abctest"),
},
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := &acl{}
a.decode(tt.args.xattr)
if tt.want != a.String() {
t.Errorf("acl.decode() = %v, want: %v", a.String(), tt.want)
}
})
}
}
func Test_acl_encode(t *testing.T) {
tests := []struct {
name string
want []byte
args []aclElement
}{
{
name: "encode values",
want: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255},
args: []aclElement{
{
aclSID: 8589934591,
Perm: 6,
},
{
aclSID: 8589934592,
Perm: 7,
},
{
aclSID: 8590000126,
Perm: 7,
},
{
aclSID: 21474836479,
Perm: 7,
},
{
aclSID: 73014444031,
Perm: 7,
},
{
aclSID: 141733920767,
Perm: 4,
},
},
},
{
name: "encode fail",
want: []byte{2, 0, 0, 0},
args: []aclElement{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := &acl{
Version: 2,
List: tt.args,
}
if got := a.encode(); !reflect.DeepEqual(got, tt.want) {
t.Errorf("acl.encode() = %v, want %v", got, tt.want)
}
})
}
}

183
cmd/restic/cmd_dump.go
View file

@ -1,15 +1,19 @@
package main
import (
"archive/tar"
"context"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
"github.com/spf13/cobra"
)
@ -50,41 +54,18 @@ func init() {
func splitPath(p string) []string {
d, f := path.Split(p)
if d == "" || d == "/" {
if d == "" {
return []string{f}
}
if d == "/" {
return []string{d}
}
s := splitPath(path.Clean(d))
return append(s, f)
}
func dumpNode(ctx context.Context, repo restic.Repository, node *restic.Node) error {
var buf []byte
for _, id := range node.Content {
size, found := repo.LookupBlobSize(id, restic.DataBlob)
if !found {
return errors.Errorf("id %v not found in repository", id)
}
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, pathToPrint string) error {
buf = buf[:cap(buf)]
if len(buf) < restic.CiphertextLength(int(size)) {
buf = restic.NewBlobBuffer(int(size))
}
n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf)
if err != nil {
return err
}
buf = buf[:n]
_, err = os.Stdout.Write(buf)
if err != nil {
return errors.Wrap(err, "Write")
}
}
return nil
}
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error {
if tree == nil {
return fmt.Errorf("called with a nil tree")
}
@ -97,16 +78,19 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
}
item := filepath.Join(prefix, pathComponents[0])
for _, node := range tree.Nodes {
if node.Name == pathComponents[0] {
if node.Name == pathComponents[0] || pathComponents[0] == "/" {
switch {
case l == 1 && node.Type == "file":
return dumpNode(ctx, repo, node)
return getNodeData(ctx, os.Stdout, repo, node)
case l > 1 && node.Type == "dir":
subtree, err := repo.LoadTree(ctx, *node.Subtree)
if err != nil {
return errors.Wrapf(err, "cannot load subtree for %q", item)
}
return printFromTree(ctx, subtree, repo, item, pathComponents[1:])
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], pathToPrint)
case node.Type == "dir":
node.Path = pathToPrint
return tarTree(ctx, repo, node, pathToPrint)
case l > 1:
return fmt.Errorf("%q should be a dir, but s a %q", item, node.Type)
case node.Type != "file":
@ -129,7 +113,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
debug.Log("dump file %q from %q", pathToPrint, snapshotIDString)
splittedPath := splitPath(pathToPrint)
splittedPath := splitPath(path.Clean(pathToPrint))
repo, err := OpenRepository(gopts)
if err != nil {
@ -173,10 +157,143 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
}
err = printFromTree(ctx, tree, repo, "", splittedPath)
err = printFromTree(ctx, tree, repo, "", splittedPath, pathToPrint)
if err != nil {
Exitf(2, "cannot dump file: %v", err)
}
return nil
}
func getNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error {
var buf []byte
for _, id := range node.Content {
size, found := repo.LookupBlobSize(id, restic.DataBlob)
if !found {
return errors.Errorf("id %v not found in repository", id)
}
buf = buf[:cap(buf)]
if len(buf) < restic.CiphertextLength(int(size)) {
buf = restic.NewBlobBuffer(int(size))
}
n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf)
if err != nil {
return err
}
buf = buf[:n]
_, err = output.Write(buf)
if err != nil {
return errors.Wrap(err, "Write")
}
}
return nil
}
func tarTree(ctx context.Context, repo restic.Repository, rootNode *restic.Node, rootPath string) error {
if stdoutIsTerminal() {
return fmt.Errorf("stdout is the terminal, please redirect output")
}
tw := tar.NewWriter(os.Stdout)
defer tw.Close()
// If we want to dump "/" we'll need to add the name of the first node, too
// as it would get lost otherwise.
if rootNode.Path == "/" {
rootNode.Path = path.Join(rootNode.Path, rootNode.Name)
rootPath = rootNode.Path
}
// we know that rootNode is a folder and walker.Walk will already process
// the next node, so we have to tar this one first, too
if err := tarNode(ctx, tw, rootNode, repo); err != nil {
return err
}
err := walker.Walk(ctx, repo, *rootNode.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}
if node == nil {
return false, nil
}
node.Path = path.Join(rootPath, nodepath)
if node.Type == "file" || node.Type == "symlink" || node.Type == "dir" {
err := tarNode(ctx, tw, node, repo)
if err != err {
return false, err
}
}
return false, nil
})
return err
}
func tarNode(ctx context.Context, tw *tar.Writer, node *restic.Node, repo restic.Repository) error {
header := &tar.Header{
Name: node.Path,
Size: int64(node.Size),
Mode: int64(node.Mode),
Uid: int(node.UID),
Gid: int(node.GID),
ModTime: node.ModTime,
AccessTime: node.AccessTime,
ChangeTime: node.ChangeTime,
PAXRecords: parseXattrs(node.ExtendedAttributes),
}
if node.Type == "symlink" {
header.Typeflag = tar.TypeSymlink
header.Linkname = node.LinkTarget
}
if node.Type == "dir" {
header.Typeflag = tar.TypeDir
}
err := tw.WriteHeader(header)
if err != nil {
return errors.Wrap(err, "TarHeader ")
}
return getNodeData(ctx, tw, repo, node)
}
func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string {
tmpMap := make(map[string]string)
for _, attr := range xattrs {
attrString := string(attr.Value)
if strings.HasPrefix(attr.Name, "system.posix_acl_") {
na := acl{}
na.decode(attr.Value)
if na.String() != "" {
if strings.Contains(attr.Name, "system.posix_acl_access") {
tmpMap["SCHILY.acl.access"] = na.String()
} else if strings.Contains(attr.Name, "system.posix_acl_default") {
tmpMap["SCHILY.acl.default"] = na.String()
}
}
} else {
tmpMap["SCHILY.xattr."+attr.Name] = attrString
}
}
return tmpMap
}

10
doc/050_restore.rst
View file

@ -124,3 +124,13 @@ e.g.:
.. code-block:: console
$ restic -r /srv/restic-repo dump --path /production.sql latest production.sql | mysql
It is also possible to ``dump`` the contents of a whole folder structure to
stdout. To retain the information about the files and folders Restic will
output the contents in the tar format:
.. code-block:: console
$ restic -r /srv/restic-repo dump /home/other/work latest > restore.tar