diff --git a/archiver.go b/archiver.go index 2f8078733..a8e94e516 100644 --- a/archiver.go +++ b/archiver.go @@ -350,6 +350,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str // ignore dir nodes with errors if dir.Error() != nil { + fmt.Fprintf(os.Stderr, "error walking dir %v: %v\n", dir.Path(), dir.Error()) dir.Result() <- nil p.Report(Stat{Errors: 1}) continue @@ -365,6 +366,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str // if we get a nil pointer here, an error has happened while // processing this entry. Ignore it for now. if res == nil { + debug.Log("Archiver.dirWorker", "got nil result?") continue } @@ -373,7 +375,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str tree.Insert(node) if node.Type == "dir" { - debug.Log("Archiver.dirWorker", "got tree node for %s: %v", node.path, node.blobs) + debug.Log("Archiver.dirWorker", "got tree node for %s: %v", node.path, node.Subtree) if node.Subtree.IsNull() { panic("invalid null subtree ID") diff --git a/pipe/pipe.go b/pipe/pipe.go index 6aa4f3588..45bb8a905 100644 --- a/pipe/pipe.go +++ b/pipe/pipe.go @@ -82,7 +82,7 @@ var errCancelled = errors.New("walk cancelled") // dirs). If false is returned, files are ignored and dirs are not even walked. type SelectFunc func(item string, fi os.FileInfo) bool -func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs chan<- Job, res chan<- Result) { +func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs chan<- Job, res chan<- Result) (excluded bool) { debug.Log("pipe.walk", "start on %q, basedir %q", dir, basedir) relpath, err := filepath.Rel(basedir, dir) @@ -92,7 +92,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs info, err := os.Lstat(dir) if err != nil { - debug.Log("pipe.walk", "error for %v: %v", dir, err) + debug.Log("pipe.walk", "error for %v: %v, res %p", dir, err, res) select { case jobs <- Dir{basedir: basedir, path: relpath, info: info, error: err, result: res}: case <-done: @@ -101,11 +101,13 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs } if !selectFunc(dir, info) { - debug.Log("pipe.walk", "file %v excluded by filter", dir) + debug.Log("pipe.walk", "file %v excluded by filter, res %p", dir, res) + excluded = true return } if !info.IsDir() { + debug.Log("pipe.walk", "sending file job for %v, res %p", dir, res) select { case jobs <- Entry{info: info, basedir: basedir, path: relpath, result: res}: case <-done: @@ -116,7 +118,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs debug.RunHook("pipe.readdirnames", dir) names, err := readDirNames(dir) if err != nil { - debug.Log("pipe.walk", "Readdirnames(%v) returned error: %v", dir, err) + debug.Log("pipe.walk", "Readdirnames(%v) returned error: %v, res %p", dir, err, res) select { case <-done: case jobs <- Dir{basedir: basedir, path: relpath, info: info, error: err, result: res}: @@ -143,6 +145,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs entries = append(entries, ch) if statErr != nil { + debug.Log("pipe.walk", "sending file job for %v, err %v, res %p", subpath, err, res) select { case jobs <- Entry{info: fi, error: statErr, basedir: basedir, path: filepath.Join(relpath, name), result: ch}: case <-done: @@ -158,11 +161,13 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs walk(basedir, subpath, selectFunc, done, jobs, ch) } - debug.Log("pipe.walk", "sending dirjob for %q, basedir %q", dir, basedir) + debug.Log("pipe.walk", "sending dirjob for %q, basedir %q, res %p", dir, basedir, res) select { case jobs <- Dir{basedir: basedir, path: relpath, info: info, Entries: entries, result: res}: case <-done: } + + return } // cleanupPath is used to clean a path. For a normal path, a slice with just @@ -174,7 +179,16 @@ func cleanupPath(path string) ([]string, error) { return []string{path}, nil } - return readDirNames(path) + paths, err := readDirNames(path) + if err != nil { + return nil, err + } + + for i, p := range paths { + paths[i] = filepath.Join(path, p) + } + + return paths, nil } // Walk sends a Job for each file and directory it finds below the paths. When @@ -203,12 +217,18 @@ func Walk(walkPaths []string, selectFunc SelectFunc, done chan struct{}, jobs ch for _, path := range paths { debug.Log("pipe.Walk", "start walker for %v", path) ch := make(chan Result, 1) - walk(filepath.Dir(path), path, selectFunc, done, jobs, ch) + excluded := walk(filepath.Dir(path), path, selectFunc, done, jobs, ch) + + if excluded { + debug.Log("pipe.Walk", "walker for %v done, it was excluded by the filter", path) + continue + } + entries = append(entries, ch) debug.Log("pipe.Walk", "walker for %v done", path) } - debug.Log("pipe.Walk", "sending root node") + debug.Log("pipe.Walk", "sending root node, res %p", res) select { case <-done: return