Merge pull request #476 from restic/fix-475

Ignore invalid index files
This commit is contained in:
Alexander Neumann 2016-02-25 17:56:49 +01:00
commit 5e152b7753
8 changed files with 82 additions and 30 deletions

View file

@ -105,7 +105,7 @@ func (cmd CmdDump) DumpIndexes() error {
for id := range cmd.repo.List(backend.Index, done) {
fmt.Printf("index_id: %v\n", id)
idx, err := repository.LoadIndex(cmd.repo, id.String())
idx, err := repository.LoadIndex(cmd.repo, id)
if err != nil {
return err
}

View file

@ -71,7 +71,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
cmd.global.Printf(" loading index %v\n", i)
debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
idx, err := repository.LoadIndex(cmd.repo, indexID.String())
idx, err := repository.LoadIndex(cmd.repo, indexID)
if err != nil {
return err
}

View file

@ -235,31 +235,84 @@ func (b *Local) Remove(t backend.Type, name string) error {
return os.Remove(fn)
}
func isFile(fi os.FileInfo) bool {
return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}
func readdir(d string) (fileInfos []os.FileInfo, err error) {
f, e := os.Open(d)
if e != nil {
return nil, e
}
defer func() {
e := f.Close()
if err == nil {
err = e
}
}()
return f.Readdir(-1)
}
// listDir returns a list of all files in d.
func listDir(d string) (filenames []string, err error) {
fileInfos, err := readdir(d)
if err != nil {
return nil, err
}
for _, fi := range fileInfos {
if isFile(fi) {
filenames = append(filenames, fi.Name())
}
}
return filenames, nil
}
// listDirs returns a list of all files in directories within d.
func listDirs(dir string) (filenames []string, err error) {
fileInfos, err := readdir(dir)
if err != nil {
return nil, err
}
for _, fi := range fileInfos {
if !fi.IsDir() {
continue
}
files, err := listDir(filepath.Join(dir, fi.Name()))
if err != nil {
continue
}
filenames = append(filenames, files...)
}
return filenames, nil
}
// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
var pattern string
lister := listDir
if t == backend.Data {
pattern = filepath.Join(dirname(b.p, t, ""), "*", "*")
} else {
pattern = filepath.Join(dirname(b.p, t, ""), "*")
lister = listDirs
}
ch := make(chan string)
matches, err := filepath.Glob(pattern)
items, err := lister(filepath.Join(dirname(b.p, t, "")))
if err != nil {
close(ch)
return ch
}
for i := range matches {
matches[i] = filepath.Base(matches[i])
}
go func() {
defer close(ch)
for _, m := range matches {
for _, m := range items {
if m == "" {
continue
}

View file

@ -83,12 +83,12 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
worker := func(id backend.ID, done <-chan struct{}) error {
debug.Log("LoadIndex", "worker got index %v", id)
idx, err := repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeIndex)
idx, err := repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeIndex)
if err == repository.ErrOldIndexFormat {
debug.Log("LoadIndex", "index %v has old format", id.Str())
hints = append(hints, ErrOldIndexFormat{id})
idx, err = repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeOldIndex)
idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
}
if err != nil {

View file

@ -557,15 +557,10 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
}
// LoadIndexWithDecoder loads the index and decodes it with fn.
func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
func LoadIndexWithDecoder(repo *Repository, id backend.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8])
idxID, err := backend.ParseID(id)
if err != nil {
return nil, err
}
buf, err := repo.LoadAndDecrypt(backend.Index, idxID)
buf, err := repo.LoadAndDecrypt(backend.Index, id)
if err != nil {
return nil, err
}
@ -576,7 +571,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
return nil, err
}
idx.id = idxID
idx.id = id
return idx, nil
}
@ -588,7 +583,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
func ConvertIndex(repo *Repository, id backend.ID) (backend.ID, error) {
debug.Log("ConvertIndex", "checking index %v", id.Str())
idx, err := LoadIndexWithDecoder(repo, id.String(), DecodeOldIndex)
idx, err := LoadIndexWithDecoder(repo, id, DecodeOldIndex)
if err != nil {
debug.Log("ConvertIndex", "LoadIndexWithDecoder(%v) returned error: %v", id.Str(), err)
return id, err

View file

@ -4,6 +4,7 @@ import (
"sync"
"restic/backend"
"restic/debug"
)
func closeIfOpen(ch chan struct{}) {
@ -75,12 +76,14 @@ func FilesInParallel(repo backend.Lister, t backend.Type, n uint, f ParallelWork
}
// ParallelWorkFuncParseID converts a function that takes a backend.ID to a
// function that takes a string.
// function that takes a string. Filenames that do not parse as a backend.ID
// are ignored.
func ParallelWorkFuncParseID(f ParallelIDWorkFunc) ParallelWorkFunc {
return func(s string, done <-chan struct{}) error {
id, err := backend.ParseID(s)
if err != nil {
return err
debug.Log("repository.ParallelWorkFuncParseID", "invalid ID %q: %v", id, err)
return nil
}
return f(id, done)

View file

@ -366,7 +366,7 @@ func (r *Repository) LoadIndex() error {
errCh := make(chan error, 1)
indexes := make(chan *Index)
worker := func(id string, done <-chan struct{}) error {
worker := func(id backend.ID, done <-chan struct{}) error {
idx, err := LoadIndex(r, id)
if err != nil {
return err
@ -382,7 +382,8 @@ func (r *Repository) LoadIndex() error {
go func() {
defer close(indexes)
errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism, worker)
errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism,
ParallelWorkFuncParseID(worker))
}()
for idx := range indexes {
@ -397,14 +398,14 @@ func (r *Repository) LoadIndex() error {
}
// LoadIndex loads the index id from backend and returns it.
func LoadIndex(repo *Repository, id string) (*Index, error) {
func LoadIndex(repo *Repository, id backend.ID) (*Index, error) {
idx, err := LoadIndexWithDecoder(repo, id, DecodeIndex)
if err == nil {
return idx, nil
}
if err == ErrOldIndexFormat {
fmt.Fprintf(os.Stderr, "index %v has old format\n", id[:10])
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
return LoadIndexWithDecoder(repo, id, DecodeOldIndex)
}

View file

@ -292,7 +292,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
packEntries := make(map[backend.ID]map[backend.ID]struct{})
for id := range repo.List(backend.Index, nil) {
idx, err := repository.LoadIndex(repo, id.String())
idx, err := repository.LoadIndex(repo, id)
OK(t, err)
for pb := range idx.Each(nil) {