Factor Lister into own file, write tests and fix
This commit is contained in:
Nick Craig-Wood
parent
c2d0e86431
commit
dbfa7031d2
3 changed files with 614 additions and 272 deletions
280
fs/fs.go
280
fs/fs.go
|
|
@ -9,8 +9,6 @@ import (
|
|||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -95,10 +93,8 @@ func Register(info *RegInfo) {
|
|||
fsRegistry = append(fsRegistry, info)
|
||||
}
|
||||
|
||||
// Fs is the interface a cloud storage system must provide
|
||||
type Fs interface {
|
||||
Info
|
||||
|
||||
// ListFser is the interface for listing a remote Fs
|
||||
type ListFser interface {
|
||||
// List the objects and directories of the Fs starting from dir
|
||||
//
|
||||
// dir should be "" to start from the root, and should not
|
||||
|
|
@ -107,6 +103,12 @@ type Fs interface {
|
|||
// This should return ErrDirNotFound (using out.SetError())
|
||||
// if the directory isn't found.
|
||||
List(out ListOpts, dir string)
|
||||
}
|
||||
|
||||
// Fs is the interface a cloud storage system must provide
|
||||
type Fs interface {
|
||||
Info
|
||||
ListFser
|
||||
|
||||
// NewFsObject finds the Object at remote. Returns nil if can't be found
|
||||
NewFsObject(remote string) Object
|
||||
|
|
@ -288,272 +290,6 @@ type ListOpts interface {
|
|||
IsFinished() bool
|
||||
}
|
||||
|
||||
// listerResult is returned by the lister methods
|
||||
type listerResult struct {
|
||||
Obj Object
|
||||
Dir *Dir
|
||||
Err error
|
||||
}
|
||||
|
||||
// Lister objects are used for controlling listing of Fs objects
|
||||
type Lister struct {
|
||||
mu sync.RWMutex
|
||||
buffer int
|
||||
abort bool
|
||||
results chan listerResult
|
||||
finished sync.Once
|
||||
level int
|
||||
filter *Filter
|
||||
}
|
||||
|
||||
// NewLister creates a Lister object.
|
||||
//
|
||||
// The default channel buffer size will be Config.Checkers unless
|
||||
// overridden with SetBuffer. The default level will be infinite.
|
||||
func NewLister() *Lister {
|
||||
o := &Lister{}
|
||||
return o.SetLevel(-1).SetBuffer(Config.Checkers)
|
||||
}
|
||||
|
||||
// Start starts a go routine listing the Fs passed in. It returns the
|
||||
// same Lister that was passed in for convenience.
|
||||
func (o *Lister) Start(f Fs, dir string) *Lister {
|
||||
o.results = make(chan listerResult, o.buffer)
|
||||
go func() {
|
||||
f.List(o, dir)
|
||||
}()
|
||||
return o
|
||||
}
|
||||
|
||||
// SetLevel sets the level to recurse to. It returns same Lister that
|
||||
// was passed in for convenience. If Level is < 0 then it sets it to
|
||||
// infinite. Must be called before Start().
|
||||
func (o *Lister) SetLevel(level int) *Lister {
|
||||
if level < 0 {
|
||||
o.level = MaxLevel
|
||||
} else {
|
||||
o.level = level
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// SetFilter sets the Filter that is in use. It defaults to no
|
||||
// filtering. Must be called before Start().
|
||||
func (o *Lister) SetFilter(filter *Filter) *Lister {
|
||||
o.filter = filter
|
||||
return o
|
||||
}
|
||||
|
||||
// Level gets the recursion level for this listing.
|
||||
//
|
||||
// Fses may ignore this, but should implement it for improved efficiency if possible.
|
||||
//
|
||||
// Level 1 means list just the contents of the directory
|
||||
//
|
||||
// Each returned item must have less than level `/`s in.
|
||||
func (o *Lister) Level() int {
|
||||
return o.level
|
||||
}
|
||||
|
||||
// SetBuffer sets the channel buffer size in use. Must be called
|
||||
// before Start().
|
||||
func (o *Lister) SetBuffer(buffer int) *Lister {
|
||||
if buffer < 1 {
|
||||
buffer = 1
|
||||
}
|
||||
o.buffer = buffer
|
||||
return o
|
||||
}
|
||||
|
||||
// Buffer gets the channel buffer size in use
|
||||
func (o *Lister) Buffer() int {
|
||||
return o.buffer
|
||||
}
|
||||
|
||||
// Add an object to the output.
|
||||
// If the function returns true, the operation has been aborted.
|
||||
// Multiple goroutines can safely add objects concurrently.
|
||||
func (o *Lister) Add(obj Object) (abort bool) {
|
||||
o.mu.RLock()
|
||||
defer o.mu.RUnlock()
|
||||
if o.abort {
|
||||
return true
|
||||
}
|
||||
o.results <- listerResult{Obj: obj}
|
||||
return false
|
||||
}
|
||||
|
||||
// AddDir will a directory to the output.
|
||||
// If the function returns true, the operation has been aborted.
|
||||
// Multiple goroutines can safely add objects concurrently.
|
||||
func (o *Lister) AddDir(dir *Dir) (abort bool) {
|
||||
o.mu.RLock()
|
||||
defer o.mu.RUnlock()
|
||||
if o.abort {
|
||||
return true
|
||||
}
|
||||
remote := dir.Name
|
||||
remote = strings.Trim(remote, "/")
|
||||
dir.Name = remote
|
||||
// Check the level and ignore if too high
|
||||
slashes := strings.Count(remote, "/")
|
||||
if slashes >= o.level {
|
||||
return false
|
||||
}
|
||||
// Check if directory is included
|
||||
if !o.IncludeDirectory(remote) {
|
||||
return false
|
||||
}
|
||||
o.results <- listerResult{Dir: dir}
|
||||
return false
|
||||
}
|
||||
|
||||
// IncludeDirectory returns whether this directory should be
|
||||
// included in the listing (and recursed into or not).
|
||||
func (o *Lister) IncludeDirectory(remote string) bool {
|
||||
if o.filter == nil {
|
||||
return true
|
||||
}
|
||||
return o.filter.IncludeDirectory(remote)
|
||||
}
|
||||
|
||||
// SetError will set an error state, and will cause the listing to
|
||||
// be aborted.
|
||||
// Multiple goroutines can set the error state concurrently,
|
||||
// but only the first will be returned to the caller.
|
||||
func (o *Lister) SetError(err error) {
|
||||
o.mu.RLock()
|
||||
if err != nil && !o.abort {
|
||||
o.results <- listerResult{Err: err}
|
||||
}
|
||||
o.mu.RUnlock()
|
||||
o.Finished()
|
||||
}
|
||||
|
||||
// Finished should be called when listing is finished
|
||||
func (o *Lister) Finished() {
|
||||
o.finished.Do(func() {
|
||||
o.mu.Lock()
|
||||
o.abort = true
|
||||
close(o.results)
|
||||
o.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// IsFinished returns whether the directory listing is finished or not
|
||||
func (o *Lister) IsFinished() bool {
|
||||
o.mu.RLock()
|
||||
defer o.mu.RUnlock()
|
||||
return o.abort
|
||||
}
|
||||
|
||||
// Get an object from the listing.
|
||||
// Will return either an object or a directory, never both.
|
||||
// Will return (nil, nil, nil) when all objects have been returned.
|
||||
func (o *Lister) Get() (Object, *Dir, error) {
|
||||
select {
|
||||
case r := <-o.results:
|
||||
return r.Obj, r.Dir, r.Err
|
||||
}
|
||||
}
|
||||
|
||||
// Get all the objects and dirs from the listing.
|
||||
func (o *Lister) GetAll() (objs []Object, dirs []*Dir, err error) {
|
||||
for {
|
||||
obj, dir, err := o.Get()
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, nil, err
|
||||
case obj != nil:
|
||||
objs = append(objs, obj)
|
||||
case dir != nil:
|
||||
dirs = append(dirs, dir)
|
||||
default:
|
||||
return objs, dirs, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetObject will return an object from the listing.
|
||||
// It will skip over any directories.
|
||||
// Will return (nil, nil) when all objects have been returned.
|
||||
func (o *Lister) GetObject() (Object, error) {
|
||||
for {
|
||||
obj, dir, err := o.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if we are finished
|
||||
if dir == nil && obj == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// Ignore directories
|
||||
if dir != nil {
|
||||
continue
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetObjects will return a slice of object from the listing.
|
||||
// It will skip over any directories.
|
||||
func (o *Lister) GetObjects() (objs []Object, err error) {
|
||||
for {
|
||||
obj, dir, err := o.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if we are finished
|
||||
if dir == nil && obj == nil {
|
||||
break
|
||||
}
|
||||
if obj != nil {
|
||||
objs = append(objs, obj)
|
||||
}
|
||||
}
|
||||
return objs, nil
|
||||
}
|
||||
|
||||
// GetDir will return a directory from the listing.
|
||||
// It will skip over any objects.
|
||||
// Will return (nil, nil) when all objects have been returned.
|
||||
func (o *Lister) GetDir() (*Dir, error) {
|
||||
for {
|
||||
obj, dir, err := o.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if we are finished
|
||||
if dir == nil && obj == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// Ignore objects
|
||||
if obj != nil {
|
||||
continue
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetDirs will return a slice of directories from the listing.
|
||||
// It will skip over any objects.
|
||||
func (o *Lister) GetDirs() (dirs []*Dir, err error) {
|
||||
for {
|
||||
obj, dir, err := o.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if we are finished
|
||||
if dir == nil && obj == nil {
|
||||
break
|
||||
}
|
||||
if dir != nil {
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
}
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// Objects is a slice of Object~s
|
||||
type Objects []Object
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue