KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

Factor Lister into own file, write tests and fix

Browse source
This commit is contained in:
Nick Craig-Wood 2016-05-07 17:12:34 +01:00
parent c2d0e86431
commit dbfa7031d2
3 changed files with 614 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

280
fs/fs.go
View file

@ -9,8 +9,6 @@ import (
"path/filepath" "path/filepath"
"regexp" "regexp"
"sort" "sort"
"strings"
"sync"
"time" "time"
) )
@ -95,10 +93,8 @@ func Register(info *RegInfo) {
fsRegistry = append(fsRegistry, info) fsRegistry = append(fsRegistry, info)
} }
// Fs is the interface a cloud storage system must provide // ListFser is the interface for listing a remote Fs
type Fs interface { type ListFser interface {
Info
// List the objects and directories of the Fs starting from dir // List the objects and directories of the Fs starting from dir
// //
// dir should be "" to start from the root, and should not // dir should be "" to start from the root, and should not
@ -107,6 +103,12 @@ type Fs interface {
// This should return ErrDirNotFound (using out.SetError()) // This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found. // if the directory isn't found.
List(out ListOpts, dir string) List(out ListOpts, dir string)
}
// Fs is the interface a cloud storage system must provide
type Fs interface {
Info
ListFser
// NewFsObject finds the Object at remote. Returns nil if can't be found // NewFsObject finds the Object at remote. Returns nil if can't be found
NewFsObject(remote string) Object NewFsObject(remote string) Object
@ -288,272 +290,6 @@ type ListOpts interface {
IsFinished() bool IsFinished() bool
} }
// listerResult is returned by the lister methods
type listerResult struct {
Obj Object
Dir *Dir
Err error
}
// Lister objects are used for controlling listing of Fs objects
type Lister struct {
mu sync.RWMutex
buffer int
abort bool
results chan listerResult
finished sync.Once
level int
filter *Filter
}
// NewLister creates a Lister object.
//
// The default channel buffer size will be Config.Checkers unless
// overridden with SetBuffer. The default level will be infinite.
func NewLister() *Lister {
o := &Lister{}
return o.SetLevel(-1).SetBuffer(Config.Checkers)
}
// Start starts a go routine listing the Fs passed in. It returns the
// same Lister that was passed in for convenience.
func (o *Lister) Start(f Fs, dir string) *Lister {
o.results = make(chan listerResult, o.buffer)
go func() {
f.List(o, dir)
}()
return o
}
// SetLevel sets the level to recurse to. It returns same Lister that
// was passed in for convenience. If Level is < 0 then it sets it to
// infinite. Must be called before Start().
func (o *Lister) SetLevel(level int) *Lister {
if level < 0 {
o.level = MaxLevel
} else {
o.level = level
}
return o
}
// SetFilter sets the Filter that is in use. It defaults to no
// filtering. Must be called before Start().
func (o *Lister) SetFilter(filter *Filter) *Lister {
o.filter = filter
return o
}
// Level gets the recursion level for this listing.
//
// Fses may ignore this, but should implement it for improved efficiency if possible.
//
// Level 1 means list just the contents of the directory
//
// Each returned item must have less than level `/`s in.
func (o *Lister) Level() int {
return o.level
}
// SetBuffer sets the channel buffer size in use. Must be called
// before Start().
func (o *Lister) SetBuffer(buffer int) *Lister {
if buffer < 1 {
buffer = 1
}
o.buffer = buffer
return o
}
// Buffer gets the channel buffer size in use
func (o *Lister) Buffer() int {
return o.buffer
}
// Add an object to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (o *Lister) Add(obj Object) (abort bool) {
o.mu.RLock()
defer o.mu.RUnlock()
if o.abort {
return true
}
o.results <- listerResult{Obj: obj}
return false
}
// AddDir will a directory to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (o *Lister) AddDir(dir *Dir) (abort bool) {
o.mu.RLock()
defer o.mu.RUnlock()
if o.abort {
return true
}
remote := dir.Name
remote = strings.Trim(remote, "/")
dir.Name = remote
// Check the level and ignore if too high
slashes := strings.Count(remote, "/")
if slashes >= o.level {
return false
}
// Check if directory is included
if !o.IncludeDirectory(remote) {
return false
}
o.results <- listerResult{Dir: dir}
return false
}
// IncludeDirectory returns whether this directory should be
// included in the listing (and recursed into or not).
func (o *Lister) IncludeDirectory(remote string) bool {
if o.filter == nil {
return true
}
return o.filter.IncludeDirectory(remote)
}
// SetError will set an error state, and will cause the listing to
// be aborted.
// Multiple goroutines can set the error state concurrently,
// but only the first will be returned to the caller.
func (o *Lister) SetError(err error) {
o.mu.RLock()
if err != nil && !o.abort {
o.results <- listerResult{Err: err}
}
o.mu.RUnlock()
o.Finished()
}
// Finished should be called when listing is finished
func (o *Lister) Finished() {
o.finished.Do(func() {
o.mu.Lock()
o.abort = true
close(o.results)
o.mu.Unlock()
})
}
// IsFinished returns whether the directory listing is finished or not
func (o *Lister) IsFinished() bool {
o.mu.RLock()
defer o.mu.RUnlock()
return o.abort
}
// Get an object from the listing.
// Will return either an object or a directory, never both.
// Will return (nil, nil, nil) when all objects have been returned.
func (o *Lister) Get() (Object, *Dir, error) {
select {
case r := <-o.results:
return r.Obj, r.Dir, r.Err
}
}
// Get all the objects and dirs from the listing.
func (o *Lister) GetAll() (objs []Object, dirs []*Dir, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, nil, err
case obj != nil:
objs = append(objs, obj)
case dir != nil:
dirs = append(dirs, dir)
default:
return objs, dirs, nil
}
}
}
// GetObject will return an object from the listing.
// It will skip over any directories.
// Will return (nil, nil) when all objects have been returned.
func (o *Lister) GetObject() (Object, error) {
for {
obj, dir, err := o.Get()
if err != nil {
return nil, err
}
// Check if we are finished
if dir == nil && obj == nil {
return nil, nil
}
// Ignore directories
if dir != nil {
continue
}
return obj, nil
}
}
// GetObjects will return a slice of object from the listing.
// It will skip over any directories.
func (o *Lister) GetObjects() (objs []Object, err error) {
for {
obj, dir, err := o.Get()
if err != nil {
return nil, err
}
// Check if we are finished
if dir == nil && obj == nil {
break
}
if obj != nil {
objs = append(objs, obj)
}
}
return objs, nil
}
// GetDir will return a directory from the listing.
// It will skip over any objects.
// Will return (nil, nil) when all objects have been returned.
func (o *Lister) GetDir() (*Dir, error) {
for {
obj, dir, err := o.Get()
if err != nil {
return nil, err
}
// Check if we are finished
if dir == nil && obj == nil {
return nil, nil
}
// Ignore objects
if obj != nil {
continue
}
return dir, nil
}
}
// GetDirs will return a slice of directories from the listing.
// It will skip over any objects.
func (o *Lister) GetDirs() (dirs []*Dir, err error) {
for {
obj, dir, err := o.Get()
if err != nil {
return nil, err
}
// Check if we are finished
if dir == nil && obj == nil {
break
}
if dir != nil {
dirs = append(dirs, dir)
}
}
return dirs, nil
}
// Objects is a slice of Object~s // Objects is a slice of Object~s
type Objects []Object type Objects []Object

256
fs/lister.go Normal file
View file

@ -0,0 +1,256 @@
// This file implements the Lister object
package fs
import "sync"
// listerResult is returned by the lister methods
type listerResult struct {
Obj Object
Dir *Dir
Err error
}
// Lister objects are used for conniltrolling listing of Fs objects
type Lister struct {
mu sync.RWMutex
buffer int
abort bool
results chan listerResult
finished sync.Once
level int
filter *Filter
}
// NewLister creates a Lister object.
//
// The default channel buffer size will be Config.Checkers unless
// overridden with SetBuffer. The default level will be infinite.
func NewLister() *Lister {
o := &Lister{}
return o.SetLevel(-1).SetBuffer(Config.Checkers)
}
// Start starts a go routine listing the Fs passed in. It returns the
// same Lister that was passed in for convenience.
func (o *Lister) Start(f ListFser, dir string) *Lister {
o.results = make(chan listerResult, o.buffer)
go func() {
f.List(o, dir)
}()
return o
}
// SetLevel sets the level to recurse to. It returns same Lister that
// was passed in for convenience. If Level is < 0 then it sets it to
// infinite. Must be called before Start().
func (o *Lister) SetLevel(level int) *Lister {
if level < 0 {
o.level = MaxLevel
} else {
o.level = level
}
return o
}
// SetFilter sets the Filter that is in use. It defaults to no
// filtering. Must be called before Start().
func (o *Lister) SetFilter(filter *Filter) *Lister {
o.filter = filter
return o
}
// Level gets the recursion level for this listing.
//
// Fses may ignore this, but should implement it for improved efficiency if possible.
//
// Level 1 means list just the contents of the directory
//
// Each returned item must have less than level `/`s in.
func (o *Lister) Level() int {
return o.level
}
// SetBuffer sets the channel buffer size in use. Must be called
// before Start().
func (o *Lister) SetBuffer(buffer int) *Lister {
if buffer < 1 {
buffer = 1
}
o.buffer = buffer
return o
}
// Buffer gets the channel buffer size in use
func (o *Lister) Buffer() int {
return o.buffer
}
// Add an object to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (o *Lister) Add(obj Object) (abort bool) {
o.mu.RLock()
defer o.mu.RUnlock()
if o.abort {
return true
}
o.results <- listerResult{Obj: obj}
return false
}
// AddDir will a directory to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (o *Lister) AddDir(dir *Dir) (abort bool) {
o.mu.RLock()
defer o.mu.RUnlock()
if o.abort {
return true
}
o.results <- listerResult{Dir: dir}
return false
}
// IncludeDirectory returns whether this directory should be
// included in the listing (and recursed into or not).
func (o *Lister) IncludeDirectory(remote string) bool {
if o.filter == nil {
return true
}
return o.filter.IncludeDirectory(remote)
}
// SetError will set an error state, and will cause the listing to
// be aborted.
// Multiple goroutines can set the error state concurrently,
// but only the first will be returned to the caller.
func (o *Lister) SetError(err error) {
o.mu.RLock()
if err != nil && !o.abort {
o.results <- listerResult{Err: err}
}
o.mu.RUnlock()
o.Finished()
}
// Finished should be called when listing is finished
func (o *Lister) Finished() {
o.finished.Do(func() {
o.mu.Lock()
o.abort = true
close(o.results)
o.mu.Unlock()
})
}
// IsFinished returns whether the directory listing is finished or not
func (o *Lister) IsFinished() bool {
o.mu.RLock()
defer o.mu.RUnlock()
return o.abort
}
// Get an object from the listing.
// Will return either an object or a directory, never both.
// Will return (nil, nil, nil) when all objects have been returned.
func (o *Lister) Get() (Object, *Dir, error) {
select {
case r := <-o.results:
return r.Obj, r.Dir, r.Err
}
}
// GetAll gets all the objects and dirs from the listing.
func (o *Lister) GetAll() (objs []Object, dirs []*Dir, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, nil, err
case obj != nil:
objs = append(objs, obj)
case dir != nil:
dirs = append(dirs, dir)
default:
return objs, dirs, nil
}
}
}
// GetObject will return an object from the listing.
// It will skip over any directories.
// Will return (nil, nil) when all objects have been returned.
func (o *Lister) GetObject() (Object, error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
return obj, nil
case dir != nil:
// ignore
default:
return nil, nil
}
}
}
// GetObjects will return a slice of object from the listing.
// It will skip over any directories.
func (o *Lister) GetObjects() (objs []Object, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
objs = append(objs, obj)
case dir != nil:
// ignore
default:
return objs, nil
}
}
}
// GetDir will return a directory from the listing.
// It will skip over any objects.
// Will return (nil, nil) when all objects have been returned.
func (o *Lister) GetDir() (*Dir, error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
// ignore
case dir != nil:
return dir, nil
default:
return nil, nil
}
}
}
// GetDirs will return a slice of directories from the listing.
// It will skip over any objects.
func (o *Lister) GetDirs() (dirs []*Dir, err error) {
for {
obj, dir, err := o.Get()
switch {
case err != nil:
return nil, err
case obj != nil:
// ignore
case dir != nil:
dirs = append(dirs, dir)
default:
return dirs, nil
}
}
}
// Check interface
var _ ListOpts = (*Lister)(nil)

350
fs/lister_test.go Normal file
View file

@ -0,0 +1,350 @@
package fs
import (
"errors"
"io"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestListerNew(t *testing.T) {
o := NewLister()
assert.Equal(t, Config.Checkers, o.buffer)
assert.Equal(t, false, o.abort)
assert.Equal(t, MaxLevel, o.level)
}
var errNotImpl = errors.New("Not implemented")
type mockObject string
func (o mockObject) String() string { return string(o) }
func (o mockObject) Fs() Info { return nil }
func (o mockObject) Remote() string { return string(o) }
func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl }
func (o mockObject) ModTime() (t time.Time) { return t }
func (o mockObject) Size() int64 { return 0 }
func (o mockObject) Storable() bool { return true }
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
func (o mockObject) Open() (io.ReadCloser, error) { return nil, errNotImpl }
func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl }
func (o mockObject) Remove() error { return errNotImpl }
type mockFs struct {
listFn func(o ListOpts, dir string)
}
func (f *mockFs) List(o ListOpts, dir string) {
defer o.Finished()
f.listFn(o, dir)
}
func TestListerStart(t *testing.T) {
f := &mockFs{}
ranList := false
f.listFn = func(o ListOpts, dir string) {
ranList = true
}
o := NewLister().Start(f, "")
objs, dirs, err := o.GetAll()
require.Nil(t, err)
assert.Len(t, objs, 0)
assert.Len(t, dirs, 0)
assert.Equal(t, true, ranList)
}
func TestListerSetLevel(t *testing.T) {
o := NewLister()
o.SetLevel(1)
assert.Equal(t, 1, o.level)
o.SetLevel(0)
assert.Equal(t, 0, o.level)
o.SetLevel(-1)
assert.Equal(t, MaxLevel, o.level)
}
func TestListerSetFilter(t *testing.T) {
filter := &Filter{}
o := NewLister().SetFilter(filter)
assert.Equal(t, filter, o.filter)
}
func TestListerLevel(t *testing.T) {
o := NewLister()
assert.Equal(t, MaxLevel, o.Level())
o.SetLevel(123)
assert.Equal(t, 123, o.Level())
}
func TestListerSetBuffer(t *testing.T) {
o := NewLister()
o.SetBuffer(2)
assert.Equal(t, 2, o.buffer)
o.SetBuffer(1)
assert.Equal(t, 1, o.buffer)
o.SetBuffer(0)
assert.Equal(t, 1, o.buffer)
o.SetBuffer(-1)
assert.Equal(t, 1, o.buffer)
}
func TestListerBuffer(t *testing.T) {
o := NewLister()
assert.Equal(t, Config.Checkers, o.Buffer())
o.SetBuffer(123)
assert.Equal(t, 123, o.Buffer())
}
func TestListerAdd(t *testing.T) {
f := &mockFs{}
objs := []Object{
mockObject("1"),
mockObject("2"),
}
f.listFn = func(o ListOpts, dir string) {
for _, obj := range objs {
assert.Equal(t, false, o.Add(obj))
}
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
require.Nil(t, err)
assert.Equal(t, objs, gotObjs)
assert.Len(t, gotDirs, 0)
}
func TestListerAddDir(t *testing.T) {
f := &mockFs{}
dirs := []*Dir{
&Dir{Name: "1"},
&Dir{Name: "2"},
}
f.listFn = func(o ListOpts, dir string) {
for _, dir := range dirs {
assert.Equal(t, false, o.AddDir(dir))
}
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
require.Nil(t, err)
assert.Len(t, gotObjs, 0)
assert.Equal(t, dirs, gotDirs)
}
func TestListerIncludeDirectory(t *testing.T) {
o := NewLister()
assert.Equal(t, true, o.IncludeDirectory("whatever"))
filter, err := NewFilter()
require.Nil(t, err)
require.NotNil(t, filter)
require.Nil(t, filter.AddRule("!"))
require.Nil(t, filter.AddRule("+ potato/*"))
require.Nil(t, filter.AddRule("- *"))
o.SetFilter(filter)
assert.Equal(t, false, o.IncludeDirectory("floop"))
assert.Equal(t, true, o.IncludeDirectory("potato"))
assert.Equal(t, false, o.IncludeDirectory("potato/sausage"))
}
func TestListerSetError(t *testing.T) {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.Add(mockObject("1")))
o.SetError(errNotImpl)
assert.Equal(t, true, o.Add(mockObject("2")))
o.SetError(errors.New("not signalled"))
assert.Equal(t, true, o.AddDir(&Dir{Name: "2"}))
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, gotObjs)
assert.Nil(t, gotDirs)
}
func TestListerIsFinished(t *testing.T) {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.IsFinished())
o.Finished()
assert.Equal(t, true, o.IsFinished())
}
o := NewLister().Start(f, "")
gotObjs, gotDirs, err := o.GetAll()
assert.Nil(t, err)
assert.Len(t, gotObjs, 0)
assert.Len(t, gotDirs, 0)
}
func testListerGet(t *testing.T) *Lister {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.Add(mockObject("1")))
assert.Equal(t, false, o.AddDir(&Dir{Name: "2"}))
}
return NewLister().Start(f, "")
}
func TestListerGet(t *testing.T) {
o := testListerGet(t)
obj, dir, err := o.Get()
assert.Nil(t, err)
assert.Equal(t, obj.Remote(), "1")
assert.Nil(t, dir)
obj, dir, err = o.Get()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Equal(t, dir.Name, "2")
obj, dir, err = o.Get()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObject(t *testing.T) {
o := testListerGet(t)
obj, err := o.GetObject()
assert.Nil(t, err)
assert.Equal(t, obj.Remote(), "1")
obj, err = o.GetObject()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDir(t *testing.T) {
o := testListerGet(t)
dir, err := o.GetDir()
assert.Nil(t, err)
assert.Equal(t, dir.Name, "2")
dir, err = o.GetDir()
assert.Nil(t, err)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func testListerGetError(t *testing.T) *Lister {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
o.SetError(errNotImpl)
}
return NewLister().Start(f, "")
}
func TestListerGetError(t *testing.T) {
o := testListerGetError(t)
obj, dir, err := o.Get()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, obj)
assert.Nil(t, dir)
obj, dir, err = o.Get()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObjectError(t *testing.T) {
o := testListerGetError(t)
obj, err := o.GetObject()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, obj)
obj, err = o.GetObject()
assert.Nil(t, err)
assert.Nil(t, obj)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDirError(t *testing.T) {
o := testListerGetError(t)
dir, err := o.GetDir()
assert.Equal(t, err, errNotImpl)
assert.Nil(t, dir)
dir, err = o.GetDir()
assert.Nil(t, err)
assert.Nil(t, dir)
assert.Equal(t, true, o.IsFinished())
}
func testListerGetAll(t *testing.T) (*Lister, []Object, []*Dir) {
objs := []Object{
mockObject("1f"),
mockObject("2f"),
mockObject("3f"),
}
dirs := []*Dir{
&Dir{Name: "1d"},
&Dir{Name: "2d"},
}
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
assert.Equal(t, false, o.Add(objs[0]))
assert.Equal(t, false, o.Add(objs[1]))
assert.Equal(t, false, o.AddDir(dirs[0]))
assert.Equal(t, false, o.Add(objs[2]))
assert.Equal(t, false, o.AddDir(dirs[1]))
}
return NewLister().Start(f, ""), objs, dirs
}
func TestListerGetAll(t *testing.T) {
o, objs, dirs := testListerGetAll(t)
gotObjs, gotDirs, err := o.GetAll()
assert.Nil(t, err)
assert.Equal(t, objs, gotObjs)
assert.Equal(t, dirs, gotDirs)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObjects(t *testing.T) {
o, objs, _ := testListerGetAll(t)
gotObjs, err := o.GetObjects()
assert.Nil(t, err)
assert.Equal(t, objs, gotObjs)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDirs(t *testing.T) {
o, _, dirs := testListerGetAll(t)
gotDirs, err := o.GetDirs()
assert.Nil(t, err)
assert.Equal(t, dirs, gotDirs)
assert.Equal(t, true, o.IsFinished())
}
func testListerGetAllError(t *testing.T) *Lister {
f := &mockFs{}
f.listFn = func(o ListOpts, dir string) {
o.SetError(errNotImpl)
}
return NewLister().Start(f, "")
}
func TestListerGetAllError(t *testing.T) {
o := testListerGetAllError(t)
gotObjs, gotDirs, err := o.GetAll()
assert.Equal(t, errNotImpl, err)
assert.Len(t, gotObjs, 0)
assert.Len(t, gotDirs, 0)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetObjectsError(t *testing.T) {
o := testListerGetAllError(t)
gotObjs, err := o.GetObjects()
assert.Equal(t, errNotImpl, err)
assert.Len(t, gotObjs, 0)
assert.Equal(t, true, o.IsFinished())
}
func TestListerGetDirsError(t *testing.T) {
o := testListerGetAllError(t)
gotDirs, err := o.GetDirs()
assert.Equal(t, errNotImpl, err)
assert.Len(t, gotDirs, 0)
assert.Equal(t, true, o.IsFinished())
}