TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge branch 'improve-fsck'

Browse source
This commit is contained in:
Alexander Neumann 2014-12-22 14:47:21 +01:00
commit 0d882626d9
6 changed files with 160 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

134
cmd/restic/cmd_fsck.go
View file

@ -1,13 +1,24 @@
package main
import (
"errors"
"fmt"
"github.com/restic/restic"
"github.com/restic/restic/backend"
)
type CmdFsck struct{}
type CmdFsck struct {
CheckData bool ` long:"check-data" description:"Read data blobs" default:"false"`
Snapshot string `short:"s" long:"snapshot" description:"Only check this snapshot"`
Orphaned bool `short:"o" long:"orphaned" description:"Check for orphaned blobs"`
RemoveOrphaned bool `short:"x" long:"remove-orphaned" description:"Remove orphaned blobs (implies -o)"`
// lists checking for orphaned blobs
o_data *restic.BlobList
o_trees *restic.BlobList
o_maps *restic.BlobList
}
func init() {
_, err := parser.AddCommand("fsck",
@ -19,21 +30,45 @@ func init() {
}
}
func fsckFile(ch *restic.ContentHandler, IDs []backend.ID) error {
func fsckFile(opts CmdFsck, ch *restic.ContentHandler, IDs []backend.ID) error {
for _, id := range IDs {
debug("checking data blob %v\n", id)
if opts.CheckData {
// load content
_, err := ch.Load(backend.Data, id)
if err != nil {
return err
}
} else {
// test if data blob is there
ok, err := ch.Test(backend.Data, id)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("data blob %v not found", id)
}
}
// if orphan check is active, record storage id
if opts.o_data != nil {
// lookup storage ID
sid, err := ch.Lookup(id)
if err != nil {
return err
}
// add ID to list
opts.o_data.Insert(restic.Blob{ID: sid})
}
}
return nil
}
func fsckTree(ch *restic.ContentHandler, id backend.ID) error {
func fsckTree(opts CmdFsck, ch *restic.ContentHandler, id backend.ID) error {
debug("checking tree %v\n", id)
tree, err := restic.LoadTree(ch, id)
@ -41,6 +76,18 @@ func fsckTree(ch *restic.ContentHandler, id backend.ID) error {
return err
}
// if orphan check is active, record storage id
if opts.o_trees != nil {
// lookup storage ID
sid, err := ch.Lookup(id)
if err != nil {
return err
}
// add ID to list
opts.o_trees.Insert(restic.Blob{ID: sid})
}
for i, node := range tree {
if node.Name == "" {
return fmt.Errorf("node %v of tree %v has no name", i, id)
@ -56,7 +103,7 @@ func fsckTree(ch *restic.ContentHandler, id backend.ID) error {
return fmt.Errorf("file node %q of tree %v has no content", node.Name, id)
}
err := fsckFile(ch, node.Content)
err := fsckFile(opts, ch, node.Content)
if err != nil {
return err
}
@ -65,7 +112,7 @@ func fsckTree(ch *restic.ContentHandler, id backend.ID) error {
return fmt.Errorf("dir node %q of tree %v has no subtree", node.Name, id)
}
err := fsckTree(ch, node.Subtree)
err := fsckTree(opts, ch, node.Subtree)
if err != nil {
return err
}
@ -75,7 +122,7 @@ func fsckTree(ch *restic.ContentHandler, id backend.ID) error {
return nil
}
func fsck_snapshot(s restic.Server, id backend.ID) error {
func fsck_snapshot(opts CmdFsck, s restic.Server, id backend.ID) error {
debug("checking snapshot %v\n", id)
ch, err := restic.NewContentHandler(s)
@ -96,16 +143,25 @@ func fsck_snapshot(s restic.Server, id backend.ID) error {
return fmt.Errorf("snapshot %v has no map", sn.ID)
}
return fsckTree(ch, sn.Tree)
// if orphan check is active, record storage id for map
if opts.o_maps != nil {
opts.o_maps.Insert(restic.Blob{ID: sn.Map})
}
return fsckTree(opts, ch, sn.Tree)
}
func (cmd CmdFsck) Usage() string {
return "fsck [all|snapshot-ID]"
return "[fsck-options]"
}
func (cmd CmdFsck) Execute(args []string) error {
if len(args) == 0 {
return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage())
if len(args) != 0 {
return errors.New("fsck has no arguments")
}
if cmd.RemoveOrphaned && !cmd.Orphaned {
cmd.Orphaned = true
}
s, err := OpenRepo()
@ -113,27 +169,75 @@ func (cmd CmdFsck) Execute(args []string) error {
return err
}
if len(args) == 1 && args[0] != "all" {
snapshotID, err := s.FindSnapshot(args[0])
if cmd.Snapshot != "" {
snapshotID, err := s.FindSnapshot(cmd.Snapshot)
if err != nil {
return fmt.Errorf("invalid id %q: %v", args[0], err)
return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err)
}
return fsck_snapshot(s, snapshotID)
return fsck_snapshot(cmd, s, snapshotID)
}
if cmd.Orphaned {
cmd.o_data = restic.NewBlobList()
cmd.o_trees = restic.NewBlobList()
cmd.o_maps = restic.NewBlobList()
}
list, err := s.List(backend.Snapshot)
debug("checking %d snapshots\n", len(list))
if err != nil {
return err
}
for _, snapshotID := range list {
err := fsck_snapshot(s, snapshotID)
err := fsck_snapshot(cmd, s, snapshotID)
if err != nil {
return err
}
}
if !cmd.Orphaned {
return nil
}
debug("starting orphaned check\n")
l := []struct {
desc string
tpe backend.Type
list *restic.BlobList
}{
{"data blob", backend.Data, cmd.o_data},
{"tree", backend.Tree, cmd.o_trees},
{"maps", backend.Map, cmd.o_maps},
}
for _, d := range l {
debug("checking for orphaned %v\n", d.desc)
blobs, err := s.List(d.tpe)
if err != nil {
return err
}
for _, id := range blobs {
_, err := d.list.Find(restic.Blob{ID: id})
if err == restic.ErrBlobNotFound {
if !cmd.RemoveOrphaned {
fmt.Printf("orphaned %v %v\n", d.desc, id)
continue
}
fmt.Printf("removing orphaned %v %v\n", d.desc, id)
err := s.Remove(d.tpe, id)
if err != nil {
return err
}
}
}
}
return nil
}

2
cmd/restic/cmd_key.go
View file

@ -49,7 +49,7 @@ func list_keys(s restic.Server) error {
k.Username, k.Hostname, k.Created.Format(TimeFormat)})
})
tab.Print(os.Stdout)
tab.Write(os.Stdout)
return nil
}

2
cmd/restic/cmd_ls.go
View file

@ -60,7 +60,7 @@ func print_tree(prefix string, ch *restic.ContentHandler, id backend.ID) error {
}
func (cmd CmdLs) Usage() string {
return "ls snapshot-ID [DIR]"
return "snapshot-ID [DIR]"
}
func (cmd CmdLs) Execute(s restic.Server, key *restic.Key, args []string) error {

4
cmd/restic/cmd_snapshots.go
View file

@ -32,7 +32,7 @@ func NewTable() Table {
}
}
func (t Table) Print(w io.Writer) error {
func (t Table) Write(w io.Writer) error {
_, err := fmt.Fprintln(w, t.Header)
if err != nil {
return err
@ -136,7 +136,7 @@ func (cmd CmdSnapshots) Execute(args []string) error {
tab.Rows = append(tab.Rows, []interface{}{sn.ID()[:plen], sn.Time.Format(TimeFormat), sn.Hostname, sn.Dir})
}
tab.Print(os.Stdout)
tab.Write(os.Stdout)
return nil
}

4
cmd/restic/main.go
View file

@ -130,6 +130,10 @@ func create(u string) (backend.Backend, error) {
}
func OpenRepo() (restic.Server, error) {
if opts.Repo == "" {
return restic.Server{}, errors.New("Please specify repository location (-r)")
}
be, err := open(opts.Repo)
if err != nil {
return restic.Server{}, err

29
contenthandler.go
View file

@ -181,6 +181,17 @@ func (ch *ContentHandler) Load(t backend.Type, id backend.ID) ([]byte, error) {
return buf, nil
}
// Lookup returns the storage ID for the given blob
func (ch *ContentHandler) Lookup(id backend.ID) (backend.ID, error) {
// lookup storage hash
blob, err := ch.bl.Find(Blob{ID: id})
if err != nil {
return nil, err
}
return blob.Storage, nil
}
// LoadJSON calls Load() to get content from the backend and afterwards calls
// json.Unmarshal on the item.
func (ch *ContentHandler) LoadJSON(t backend.Type, id backend.ID, item interface{}) error {
@ -214,3 +225,21 @@ func (ch *ContentHandler) LoadJSONRaw(t backend.Type, id backend.ID, item interf
err = json.Unmarshal(backend.Uncompress(buf), item)
return err
}
// Test checks if a blob is in the repository. For Data and Tree blobs, the
// storage ID is looked up.
func (ch *ContentHandler) Test(t backend.Type, id backend.ID) (bool, error) {
if t == backend.Data || t == backend.Tree {
// lookup storage id
// lookup storage hash
blob, err := ch.bl.Find(Blob{ID: id})
if err != nil {
return false, fmt.Errorf("Storage ID %s not found", id)
}
id = blob.Storage
}
return ch.s.Test(t, id)
}