forked from TrueCloudLab/restic
commit
5e152b7753
8 changed files with 82 additions and 30 deletions
|
@ -105,7 +105,7 @@ func (cmd CmdDump) DumpIndexes() error {
|
|||
for id := range cmd.repo.List(backend.Index, done) {
|
||||
fmt.Printf("index_id: %v\n", id)
|
||||
|
||||
idx, err := repository.LoadIndex(cmd.repo, id.String())
|
||||
idx, err := repository.LoadIndex(cmd.repo, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -71,7 +71,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
|
|||
cmd.global.Printf(" loading index %v\n", i)
|
||||
|
||||
debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
|
||||
idx, err := repository.LoadIndex(cmd.repo, indexID.String())
|
||||
idx, err := repository.LoadIndex(cmd.repo, indexID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -235,31 +235,84 @@ func (b *Local) Remove(t backend.Type, name string) error {
|
|||
return os.Remove(fn)
|
||||
}
|
||||
|
||||
func isFile(fi os.FileInfo) bool {
|
||||
return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
|
||||
}
|
||||
|
||||
func readdir(d string) (fileInfos []os.FileInfo, err error) {
|
||||
f, e := os.Open(d)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
|
||||
defer func() {
|
||||
e := f.Close()
|
||||
if err == nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
|
||||
return f.Readdir(-1)
|
||||
}
|
||||
|
||||
// listDir returns a list of all files in d.
|
||||
func listDir(d string) (filenames []string, err error) {
|
||||
fileInfos, err := readdir(d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, fi := range fileInfos {
|
||||
if isFile(fi) {
|
||||
filenames = append(filenames, fi.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
}
|
||||
|
||||
// listDirs returns a list of all files in directories within d.
|
||||
func listDirs(dir string) (filenames []string, err error) {
|
||||
fileInfos, err := readdir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, fi := range fileInfos {
|
||||
if !fi.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
files, err := listDir(filepath.Join(dir, fi.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
filenames = append(filenames, files...)
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine is started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
var pattern string
|
||||
lister := listDir
|
||||
if t == backend.Data {
|
||||
pattern = filepath.Join(dirname(b.p, t, ""), "*", "*")
|
||||
} else {
|
||||
pattern = filepath.Join(dirname(b.p, t, ""), "*")
|
||||
lister = listDirs
|
||||
}
|
||||
|
||||
ch := make(chan string)
|
||||
matches, err := filepath.Glob(pattern)
|
||||
items, err := lister(filepath.Join(dirname(b.p, t, "")))
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
for i := range matches {
|
||||
matches[i] = filepath.Base(matches[i])
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, m := range matches {
|
||||
for _, m := range items {
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
|
|
@ -83,12 +83,12 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
|
|||
|
||||
worker := func(id backend.ID, done <-chan struct{}) error {
|
||||
debug.Log("LoadIndex", "worker got index %v", id)
|
||||
idx, err := repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeIndex)
|
||||
idx, err := repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeIndex)
|
||||
if err == repository.ErrOldIndexFormat {
|
||||
debug.Log("LoadIndex", "index %v has old format", id.Str())
|
||||
hints = append(hints, ErrOldIndexFormat{id})
|
||||
|
||||
idx, err = repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeOldIndex)
|
||||
idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
|
|
@ -557,15 +557,10 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
|
|||
}
|
||||
|
||||
// LoadIndexWithDecoder loads the index and decodes it with fn.
|
||||
func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
|
||||
func LoadIndexWithDecoder(repo *Repository, id backend.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
|
||||
debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8])
|
||||
|
||||
idxID, err := backend.ParseID(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, err := repo.LoadAndDecrypt(backend.Index, idxID)
|
||||
buf, err := repo.LoadAndDecrypt(backend.Index, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -576,7 +571,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
|
|||
return nil, err
|
||||
}
|
||||
|
||||
idx.id = idxID
|
||||
idx.id = id
|
||||
|
||||
return idx, nil
|
||||
}
|
||||
|
@ -588,7 +583,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
|
|||
func ConvertIndex(repo *Repository, id backend.ID) (backend.ID, error) {
|
||||
debug.Log("ConvertIndex", "checking index %v", id.Str())
|
||||
|
||||
idx, err := LoadIndexWithDecoder(repo, id.String(), DecodeOldIndex)
|
||||
idx, err := LoadIndexWithDecoder(repo, id, DecodeOldIndex)
|
||||
if err != nil {
|
||||
debug.Log("ConvertIndex", "LoadIndexWithDecoder(%v) returned error: %v", id.Str(), err)
|
||||
return id, err
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"restic/backend"
|
||||
"restic/debug"
|
||||
)
|
||||
|
||||
func closeIfOpen(ch chan struct{}) {
|
||||
|
@ -75,12 +76,14 @@ func FilesInParallel(repo backend.Lister, t backend.Type, n uint, f ParallelWork
|
|||
}
|
||||
|
||||
// ParallelWorkFuncParseID converts a function that takes a backend.ID to a
|
||||
// function that takes a string.
|
||||
// function that takes a string. Filenames that do not parse as a backend.ID
|
||||
// are ignored.
|
||||
func ParallelWorkFuncParseID(f ParallelIDWorkFunc) ParallelWorkFunc {
|
||||
return func(s string, done <-chan struct{}) error {
|
||||
id, err := backend.ParseID(s)
|
||||
if err != nil {
|
||||
return err
|
||||
debug.Log("repository.ParallelWorkFuncParseID", "invalid ID %q: %v", id, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
return f(id, done)
|
||||
|
|
|
@ -366,7 +366,7 @@ func (r *Repository) LoadIndex() error {
|
|||
errCh := make(chan error, 1)
|
||||
indexes := make(chan *Index)
|
||||
|
||||
worker := func(id string, done <-chan struct{}) error {
|
||||
worker := func(id backend.ID, done <-chan struct{}) error {
|
||||
idx, err := LoadIndex(r, id)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -382,7 +382,8 @@ func (r *Repository) LoadIndex() error {
|
|||
|
||||
go func() {
|
||||
defer close(indexes)
|
||||
errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism, worker)
|
||||
errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism,
|
||||
ParallelWorkFuncParseID(worker))
|
||||
}()
|
||||
|
||||
for idx := range indexes {
|
||||
|
@ -397,14 +398,14 @@ func (r *Repository) LoadIndex() error {
|
|||
}
|
||||
|
||||
// LoadIndex loads the index id from backend and returns it.
|
||||
func LoadIndex(repo *Repository, id string) (*Index, error) {
|
||||
func LoadIndex(repo *Repository, id backend.ID) (*Index, error) {
|
||||
idx, err := LoadIndexWithDecoder(repo, id, DecodeIndex)
|
||||
if err == nil {
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
if err == ErrOldIndexFormat {
|
||||
fmt.Fprintf(os.Stderr, "index %v has old format\n", id[:10])
|
||||
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
|
||||
return LoadIndexWithDecoder(repo, id, DecodeOldIndex)
|
||||
}
|
||||
|
||||
|
|
|
@ -292,7 +292,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
|
|||
packEntries := make(map[backend.ID]map[backend.ID]struct{})
|
||||
|
||||
for id := range repo.List(backend.Index, nil) {
|
||||
idx, err := repository.LoadIndex(repo, id.String())
|
||||
idx, err := repository.LoadIndex(repo, id)
|
||||
OK(t, err)
|
||||
|
||||
for pb := range idx.Each(nil) {
|
||||
|
|
Loading…
Reference in a new issue