From 4a3a6861e20bbf3eed9913a4ab84eec73648b3dc Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 16 Nov 2014 21:29:11 +0100 Subject: [PATCH] Scan directory first, then backup. Add stats. --- archiver.go | 156 ++++++++++++++++++++++++--------------- cmd/khepri/cmd_backup.go | 71 +++++++++++++++++- tree.go | 2 + 3 files changed, 168 insertions(+), 61 deletions(-) diff --git a/archiver.go b/archiver.go index 15dbb3f43..acc2b9a1f 100644 --- a/archiver.go +++ b/archiver.go @@ -13,8 +13,20 @@ type Archiver struct { ch *ContentHandler smap *StorageMap // blobs used for the current snapshot + Stats Stats + Error func(dir string, fi os.FileInfo, err error) error Filter func(item string, fi os.FileInfo) bool + + ScannerUpdate func(stats Stats) + SaveUpdate func(stats Stats) +} + +type Stats struct { + Files int + Directories int + Other int + Bytes uint64 } func NewArchiver(be backend.Server, key *Key) (*Archiver, error) { @@ -25,6 +37,9 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) { arch.Error = func(string, os.FileInfo, error) error { return err } // allow all files arch.Filter = func(string, os.FileInfo) bool { return true } + // do nothing + arch.ScannerUpdate = func(Stats) {} + arch.SaveUpdate = func(Stats) {} arch.smap = NewStorageMap() arch.ch, err = NewContentHandler(be, key) @@ -65,10 +80,10 @@ func (arch *Archiver) SaveJSON(t backend.Type, item interface{}) (*Blob, error) return blob, nil } -func (arch *Archiver) SaveFile(node *Node) (Blobs, error) { +func (arch *Archiver) SaveFile(node *Node) error { blobs, err := arch.ch.SaveFile(node.path, uint(node.Size)) if err != nil { - return nil, arch.Error(node.path, nil, err) + return arch.Error(node.path, nil, err) } node.Content = make([]backend.ID, len(blobs)) @@ -77,23 +92,20 @@ func (arch *Archiver) SaveFile(node *Node) (Blobs, error) { arch.smap.Insert(blob) } - return blobs, err + return err } -func (arch *Archiver) ImportDir(dir string) (Tree, error) { +func (arch *Archiver) loadTree(dir string) (*Tree, error) { + // open and list path fd, err := os.Open(dir) defer fd.Close() if err != nil { - return nil, arch.Error(dir, nil, err) + return nil, err } entries, err := fd.Readdir(-1) if err != nil { - return nil, arch.Error(dir, nil, err) - } - - if len(entries) == 0 { - return nil, nil + return nil, err } tree := Tree{} @@ -107,71 +119,97 @@ func (arch *Archiver) ImportDir(dir string) (Tree, error) { node, err := NodeFromFileInfo(path, entry) if err != nil { - return nil, arch.Error(dir, entry, err) + // TODO: error processing + return nil, err } tree = append(tree, node) if entry.IsDir() { - subtree, err := arch.ImportDir(path) + node.Tree, err = arch.loadTree(path) if err != nil { return nil, err } - - blob, err := arch.SaveJSON(backend.Tree, subtree) - if err != nil { - return nil, err - } - - node.Subtree = blob.ID - - continue } - if node.Type == "file" { - _, err := arch.SaveFile(node) - if err != nil { - return nil, arch.Error(path, entry, err) - } + switch node.Type { + case "file": + arch.Stats.Files++ + arch.Stats.Bytes += node.Size + case "dir": + arch.Stats.Directories++ + default: + arch.Stats.Other++ } } - return tree, nil + arch.ScannerUpdate(arch.Stats) + + return &tree, nil } -func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) { +func (arch *Archiver) LoadTree(path string) (*Tree, error) { + fi, err := os.Lstat(path) + if err != nil { + return nil, err + } + + node, err := NodeFromFileInfo(path, fi) + if err != nil { + return nil, err + } + + if node.Type != "dir" { + arch.Stats.Files = 1 + arch.Stats.Bytes = node.Size + arch.ScannerUpdate(arch.Stats) + return &Tree{node}, nil + } + + arch.Stats.Directories = 1 + node.Tree, err = arch.loadTree(path) + if err != nil { + return nil, err + } + + arch.ScannerUpdate(arch.Stats) + + return &Tree{node}, nil +} + +func (arch *Archiver) saveTree(t *Tree) (*Blob, error) { + for _, node := range *t { + if node.Tree != nil && node.Subtree == nil { + b, err := arch.saveTree(node.Tree) + if err != nil { + return nil, err + } + node.Subtree = b.ID + arch.SaveUpdate(Stats{Directories: 1}) + } else if node.Type == "file" && len(node.Content) == 0 { + err := arch.SaveFile(node) + if err != nil { + return nil, err + } + + arch.SaveUpdate(Stats{Files: 1, Bytes: node.Size}) + } else { + arch.SaveUpdate(Stats{Other: 1}) + } + } + + blob, err := arch.SaveJSON(backend.Tree, t) + if err != nil { + return nil, err + } + + return blob, nil +} + +func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, error) { sn := NewSnapshot(dir) - fi, err := os.Lstat(dir) - if err != nil { - return nil, nil, err - } - - node, err := NodeFromFileInfo(dir, fi) - if err != nil { - return nil, nil, err - } - - if node.Type == "dir" { - tree, err := arch.ImportDir(dir) - if err != nil { - return nil, nil, err - } - - blob, err := arch.SaveJSON(backend.Tree, tree) - if err != nil { - return nil, nil, err - } - - node.Subtree = blob.ID - } else if node.Type == "file" { - _, err := arch.SaveFile(node) - if err != nil { - return nil, nil, err - } - } - - blob, err := arch.SaveJSON(backend.Tree, &Tree{node}) + blob, err := arch.saveTree(t) if err != nil { return nil, nil, err } @@ -185,5 +223,5 @@ func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) { return nil, nil, err } - return sn, blob, nil + return sn, blob.Storage, nil } diff --git a/cmd/khepri/cmd_backup.go b/cmd/khepri/cmd_backup.go index 6fdc6c761..da2fef8ab 100644 --- a/cmd/khepri/cmd_backup.go +++ b/cmd/khepri/cmd_backup.go @@ -4,11 +4,41 @@ import ( "errors" "fmt" "os" + "strings" "github.com/fd0/khepri" "github.com/fd0/khepri/backend" + "golang.org/x/crypto/ssh/terminal" ) +func format_bytes(c uint64) string { + b := float64(c) + + switch { + case c > 1<<40: + return fmt.Sprintf("%.3f TiB", b/(1<<40)) + case c > 1<<30: + return fmt.Sprintf("%.3f GiB", b/(1<<30)) + case c > 1<<20: + return fmt.Sprintf("%.3f MiB", b/(1<<20)) + case c > 1<<10: + return fmt.Sprintf("%.3f KiB", b/(1<<10)) + default: + return fmt.Sprintf("%d B", c) + } +} + +func print_tree2(indent int, t *khepri.Tree) { + for _, node := range *t { + if node.Tree != nil { + fmt.Printf("%s%s/\n", strings.Repeat(" ", indent), node.Name) + print_tree2(indent+1, node.Tree) + } else { + fmt.Printf("%s%s\n", strings.Repeat(" ", indent), node.Name) + } + } +} + func commandBackup(be backend.Server, key *khepri.Key, args []string) error { if len(args) != 1 { return errors.New("usage: backup [dir|file]") @@ -25,12 +55,49 @@ func commandBackup(be backend.Server, key *khepri.Key, args []string) error { return err } - _, blob, err := arch.Import(target) + fmt.Printf("scanning %s\n", target) + + if terminal.IsTerminal(int(os.Stdout.Fd())) { + arch.ScannerUpdate = func(stats khepri.Stats) { + fmt.Printf("\r%6d directories, %6d files, %14s", stats.Directories, stats.Files, format_bytes(stats.Bytes)) + } + } + + // TODO: add filter + // arch.Filter = func(dir string, fi os.FileInfo) bool { + // return true + // } + + t, err := arch.LoadTree(target) if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) return err } - fmt.Printf("snapshot %s saved\n", blob.Storage) + fmt.Printf("\r%6d directories, %6d files, %14s\n", arch.Stats.Directories, arch.Stats.Files, format_bytes(arch.Stats.Bytes)) + + stats := khepri.Stats{} + if terminal.IsTerminal(int(os.Stdout.Fd())) { + arch.SaveUpdate = func(s khepri.Stats) { + stats.Files += s.Files + stats.Directories += s.Directories + stats.Other += s.Other + stats.Bytes += s.Bytes + + fmt.Printf("\r%3.2f%% %d/%d directories, %d/%d files, %s/%s", + float64(stats.Bytes)/float64(arch.Stats.Bytes)*100, + stats.Directories, arch.Stats.Directories, + stats.Files, arch.Stats.Files, + format_bytes(stats.Bytes), format_bytes(arch.Stats.Bytes)) + } + } + + sn, id, err := arch.Snapshot(target, t) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + fmt.Printf("\nsnapshot %s saved: %v\n", id, sn) return nil } diff --git a/tree.go b/tree.go index ad5f64597..778388122 100644 --- a/tree.go +++ b/tree.go @@ -34,6 +34,8 @@ type Node struct { Content []backend.ID `json:"content,omitempty"` Subtree backend.ID `json:"subtree,omitempty"` + Tree *Tree `json:"-"` + path string }