more cleanup
This commit is contained in:
parent
7af7c64403
commit
0d9360a815
7 changed files with 18 additions and 33 deletions
26
archiver.go
26
archiver.go
|
@ -24,10 +24,12 @@ const (
|
||||||
maxConcurrency = 10
|
maxConcurrency = 10
|
||||||
maxConcurrencyPreload = 20
|
maxConcurrencyPreload = 20
|
||||||
|
|
||||||
// chunkerBufSize is used in pool.go
|
|
||||||
chunkerBufSize = 512 * chunker.KiB
|
chunkerBufSize = 512 * chunker.KiB
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var archiverAbortOnAllErrors = func(str string, fi os.FileInfo, err error) error { return err }
|
||||||
|
var archiverAllowAllFiles = func(string, os.FileInfo) bool { return true }
|
||||||
|
|
||||||
type Archiver struct {
|
type Archiver struct {
|
||||||
s *server.Server
|
s *server.Server
|
||||||
|
|
||||||
|
@ -37,24 +39,20 @@ type Archiver struct {
|
||||||
Filter func(item string, fi os.FileInfo) bool
|
Filter func(item string, fi os.FileInfo) bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewArchiver(s *server.Server) (*Archiver, error) {
|
func NewArchiver(s *server.Server) *Archiver {
|
||||||
var err error
|
|
||||||
arch := &Archiver{
|
arch := &Archiver{
|
||||||
s: s,
|
s: s,
|
||||||
blobToken: make(chan struct{}, maxConcurrentBlobs),
|
blobToken: make(chan struct{}, maxConcurrentBlobs),
|
||||||
}
|
}
|
||||||
|
|
||||||
// fill blob token
|
|
||||||
for i := 0; i < maxConcurrentBlobs; i++ {
|
for i := 0; i < maxConcurrentBlobs; i++ {
|
||||||
arch.blobToken <- struct{}{}
|
arch.blobToken <- struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// abort on all errors
|
arch.Error = archiverAbortOnAllErrors
|
||||||
arch.Error = func(string, os.FileInfo, error) error { return err }
|
arch.Filter = archiverAllowAllFiles
|
||||||
// allow all files
|
|
||||||
arch.Filter = func(string, os.FileInfo) bool { return true }
|
|
||||||
|
|
||||||
return arch, nil
|
return arch
|
||||||
}
|
}
|
||||||
|
|
||||||
func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error {
|
func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error {
|
||||||
|
@ -78,22 +76,18 @@ func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Re
|
||||||
}
|
}
|
||||||
|
|
||||||
func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) {
|
func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) {
|
||||||
// convert to json
|
|
||||||
data, err := json.Marshal(item)
|
data, err := json.Marshal(item)
|
||||||
// append newline
|
|
||||||
data = append(data, '\n')
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
|
||||||
// check if tree has been saved before
|
// check if tree has been saved before
|
||||||
id := backend.Hash(data)
|
id := backend.Hash(data)
|
||||||
|
|
||||||
if arch.s.Index().Has(id) {
|
if arch.s.Index().Has(id) {
|
||||||
return id, nil
|
return id, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// otherwise save the data
|
|
||||||
return arch.s.SaveJSON(pack.Tree, item)
|
return arch.s.SaveJSON(pack.Tree, item)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,7 +100,7 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// check file again
|
// check file again, since it could have disappeared by now
|
||||||
fi, err := file.Stat()
|
fi, err := file.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -116,14 +110,12 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
|
||||||
e2 := arch.Error(node.path, fi, errors.New("file was updated, using new version"))
|
e2 := arch.Error(node.path, fi, errors.New("file was updated, using new version"))
|
||||||
|
|
||||||
if e2 == nil {
|
if e2 == nil {
|
||||||
// create new node
|
|
||||||
n, err := NodeFromFileInfo(node.path, fi)
|
n, err := NodeFromFileInfo(node.path, fi)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
debug.Log("Archiver.SaveFile", "NodeFromFileInfo returned error for %v: %v", node.path, err)
|
debug.Log("Archiver.SaveFile", "NodeFromFileInfo returned error for %v: %v", node.path, err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy node
|
|
||||||
*node = *n
|
*node = *n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -121,10 +121,10 @@ func archiveDirectory(b testing.TB) {
|
||||||
key := SetupKey(b, server, "geheim")
|
key := SetupKey(b, server, "geheim")
|
||||||
server.SetKey(key)
|
server.SetKey(key)
|
||||||
|
|
||||||
arch, err := restic.NewArchiver(server)
|
arch := restic.NewArchiver(server)
|
||||||
OK(b, err)
|
|
||||||
|
|
||||||
_, id, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil)
|
_, id, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil)
|
||||||
|
OK(b, err)
|
||||||
|
|
||||||
b.Logf("snapshot archived as %v", id)
|
b.Logf("snapshot archived as %v", id)
|
||||||
}
|
}
|
||||||
|
@ -238,8 +238,7 @@ func BenchmarkLoadTree(t *testing.B) {
|
||||||
s.SetKey(key)
|
s.SetKey(key)
|
||||||
|
|
||||||
// archive a few files
|
// archive a few files
|
||||||
arch, err := restic.NewArchiver(s)
|
arch := restic.NewArchiver(s)
|
||||||
OK(t, err)
|
|
||||||
sn, _, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil)
|
sn, _, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil)
|
||||||
OK(t, err)
|
OK(t, err)
|
||||||
t.Logf("archived snapshot %v", sn.ID())
|
t.Logf("archived snapshot %v", sn.ID())
|
||||||
|
|
|
@ -16,8 +16,7 @@ func TestCache(t *testing.T) {
|
||||||
_, err := restic.NewCache(server)
|
_, err := restic.NewCache(server)
|
||||||
OK(t, err)
|
OK(t, err)
|
||||||
|
|
||||||
arch, err := restic.NewArchiver(server)
|
arch := restic.NewArchiver(server)
|
||||||
OK(t, err)
|
|
||||||
|
|
||||||
// archive some files, this should automatically cache all blobs from the snapshot
|
// archive some files, this should automatically cache all blobs from the snapshot
|
||||||
_, _, err = arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil)
|
_, _, err = arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil)
|
||||||
|
|
|
@ -252,10 +252,7 @@ func (cmd CmdBackup) Execute(args []string) error {
|
||||||
// return true
|
// return true
|
||||||
// }
|
// }
|
||||||
|
|
||||||
arch, err := restic.NewArchiver(s)
|
arch := restic.NewArchiver(s)
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "err: %v\n", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
arch.Error = func(dir string, fi os.FileInfo, err error) error {
|
arch.Error = func(dir string, fi os.FileInfo, err error) error {
|
||||||
// TODO: make ignoring errors configurable
|
// TODO: make ignoring errors configurable
|
||||||
|
|
|
@ -19,11 +19,11 @@ type Restorer struct {
|
||||||
Filter func(item string, dstpath string, node *Node) bool
|
Filter func(item string, dstpath string, node *Node) bool
|
||||||
}
|
}
|
||||||
|
|
||||||
var abortOnAllErrors = func(str string, node *Node, err error) error { return err }
|
var restorerAbortOnAllErrors = func(str string, node *Node, err error) error { return err }
|
||||||
|
|
||||||
// NewRestorer creates a restorer preloaded with the content from the snapshot id.
|
// NewRestorer creates a restorer preloaded with the content from the snapshot id.
|
||||||
func NewRestorer(s *server.Server, id backend.ID) (*Restorer, error) {
|
func NewRestorer(s *server.Server, id backend.ID) (*Restorer, error) {
|
||||||
r := &Restorer{s: s, Error: abortOnAllErrors}
|
r := &Restorer{s: s, Error: restorerAbortOnAllErrors}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
|
|
@ -50,8 +50,7 @@ func SetupKey(t testing.TB, s *server.Server, password string) *server.Key {
|
||||||
}
|
}
|
||||||
|
|
||||||
func SnapshotDir(t testing.TB, server *server.Server, path string, parent backend.ID) *restic.Snapshot {
|
func SnapshotDir(t testing.TB, server *server.Server, path string, parent backend.ID) *restic.Snapshot {
|
||||||
arch, err := restic.NewArchiver(server)
|
arch := restic.NewArchiver(server)
|
||||||
OK(t, err)
|
|
||||||
sn, _, err := arch.Snapshot(nil, []string{path}, parent)
|
sn, _, err := arch.Snapshot(nil, []string{path}, parent)
|
||||||
OK(t, err)
|
OK(t, err)
|
||||||
return sn
|
return sn
|
||||||
|
|
|
@ -22,8 +22,7 @@ func TestWalkTree(t *testing.T) {
|
||||||
server.SetKey(key)
|
server.SetKey(key)
|
||||||
|
|
||||||
// archive a few files
|
// archive a few files
|
||||||
arch, err := restic.NewArchiver(server)
|
arch := restic.NewArchiver(server)
|
||||||
OK(t, err)
|
|
||||||
sn, _, err := arch.Snapshot(nil, dirs, nil)
|
sn, _, err := arch.Snapshot(nil, dirs, nil)
|
||||||
OK(t, err)
|
OK(t, err)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue