fs: make an in memory object for short transfers
This commit is contained in:
Nick Craig-Wood
parent
a1f8318b29
commit
11332a19a0
2 changed files with 364 additions and 1 deletions
187
fs/object.go
187
fs/object.go
|
|
@ -1,6 +1,12 @@
|
|||
package fs
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NewStaticObjectInfo returns a static ObjectInfo
|
||||
// If hashes is nil and fs is not nil, the hash map will be replaced with
|
||||
|
|
@ -48,3 +54,182 @@ func (i *staticObjectInfo) Hash(h HashType) (string, error) {
|
|||
}
|
||||
return "", ErrHashUnsupported
|
||||
}
|
||||
|
||||
// MemoryFs is an in memory Fs, it only supports FsInfo and Put
|
||||
var MemoryFs memoryFs
|
||||
|
||||
// memoryFs is an in memory fs
|
||||
type memoryFs struct{}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (memoryFs) Name() string { return "memory" }
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (memoryFs) Root() string { return "" }
|
||||
|
||||
// String returns a description of the FS
|
||||
func (memoryFs) String() string { return "memory" }
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (memoryFs) Precision() time.Duration { return time.Nanosecond }
|
||||
|
||||
// Returns the supported hash types of the filesystem
|
||||
func (memoryFs) Hashes() HashSet { return SupportedHashes }
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (memoryFs) Features() *Features { return &Features{} }
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (memoryFs) List(dir string) (entries DirEntries, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error ErrorObjectNotFound.
|
||||
func (memoryFs) NewObject(remote string) (Object, error) {
|
||||
return nil, ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (memoryFs) Put(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) {
|
||||
o := NewMemoryObject(src.Remote(), src.ModTime(), nil)
|
||||
return o, o.Update(in, src, options...)
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (memoryFs) Mkdir(dir string) error {
|
||||
return errors.New("memoryFs: can't make directory")
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (memoryFs) Rmdir(dir string) error {
|
||||
return ErrorDirNotFound
|
||||
}
|
||||
|
||||
var _ Fs = MemoryFs
|
||||
|
||||
// MemoryObject is an in memory object
|
||||
type MemoryObject struct {
|
||||
remote string
|
||||
modTime time.Time
|
||||
content []byte
|
||||
}
|
||||
|
||||
// NewMemoryObject returns an in memory Object with the modTime and content passed in
|
||||
func NewMemoryObject(remote string, modTime time.Time, content []byte) *MemoryObject {
|
||||
return &MemoryObject{
|
||||
remote: remote,
|
||||
modTime: modTime,
|
||||
content: content,
|
||||
}
|
||||
}
|
||||
|
||||
// Content returns the underlying buffer
|
||||
func (o *MemoryObject) Content() []byte {
|
||||
return o.content
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *MemoryObject) Fs() Info {
|
||||
return MemoryFs
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *MemoryObject) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// String returns a description of the Object
|
||||
func (o *MemoryObject) String() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
func (o *MemoryObject) ModTime() time.Time {
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *MemoryObject) Size() int64 {
|
||||
return int64(len(o.content))
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (o *MemoryObject) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the requested hash of the contents
|
||||
func (o *MemoryObject) Hash(h HashType) (string, error) {
|
||||
hash, err := NewMultiHasherTypes(HashSet(h))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = hash.Write(o.content)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hash.Sums()[h], nil
|
||||
}
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *MemoryObject) SetModTime(modTime time.Time) error {
|
||||
o.modTime = modTime
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *MemoryObject) Open(options ...OpenOption) (io.ReadCloser, error) {
|
||||
content := o.content
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *RangeOption:
|
||||
content = o.content[x.Start:x.End]
|
||||
case *SeekOption:
|
||||
content = o.content[x.Offset:]
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ioutil.NopCloser(bytes.NewBuffer(content)), nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// This re-uses the internal buffer if at all possible.
|
||||
func (o *MemoryObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) (err error) {
|
||||
size := src.Size()
|
||||
if size == 0 {
|
||||
o.content = nil
|
||||
} else if size < 0 || int64(cap(o.content)) < size {
|
||||
o.content, err = ioutil.ReadAll(in)
|
||||
} else {
|
||||
o.content = o.content[:size]
|
||||
_, err = io.ReadFull(in, o.content)
|
||||
}
|
||||
o.modTime = src.ModTime()
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove this object
|
||||
func (o *MemoryObject) Remove() error {
|
||||
return errors.New("memoryObject.Remove not supported")
|
||||
}
|
||||
|
|
|
|||
178
fs/object_test.go
Normal file
178
fs/object_test.go
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
package fs_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestStaticObject(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
now := time.Now()
|
||||
remote := "path/to/object"
|
||||
size := int64(1024)
|
||||
|
||||
o := fs.NewStaticObjectInfo(remote, now, size, true, nil, r.Flocal)
|
||||
|
||||
assert.Equal(t, r.Flocal, o.Fs())
|
||||
assert.Equal(t, remote, o.Remote())
|
||||
assert.Equal(t, remote, o.String())
|
||||
assert.Equal(t, now, o.ModTime())
|
||||
assert.Equal(t, size, o.Size())
|
||||
assert.Equal(t, true, o.Storable())
|
||||
|
||||
hash, err := o.Hash(fs.HashMD5)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "", hash)
|
||||
|
||||
o = fs.NewStaticObjectInfo(remote, now, size, true, nil, nil)
|
||||
_, err = o.Hash(fs.HashMD5)
|
||||
assert.Equal(t, fs.ErrHashUnsupported, err)
|
||||
|
||||
hs := map[fs.HashType]string{
|
||||
fs.HashMD5: "potato",
|
||||
}
|
||||
o = fs.NewStaticObjectInfo(remote, now, size, true, hs, nil)
|
||||
hash, err = o.Hash(fs.HashMD5)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "potato", hash)
|
||||
_, err = o.Hash(fs.HashSHA1)
|
||||
assert.Equal(t, fs.ErrHashUnsupported, err)
|
||||
|
||||
}
|
||||
|
||||
func TestMemoryFs(t *testing.T) {
|
||||
f := fs.MemoryFs
|
||||
assert.Equal(t, "memory", f.Name())
|
||||
assert.Equal(t, "", f.Root())
|
||||
assert.Equal(t, "memory", f.String())
|
||||
assert.Equal(t, time.Nanosecond, f.Precision())
|
||||
assert.Equal(t, fs.SupportedHashes, f.Hashes())
|
||||
assert.Equal(t, &fs.Features{}, f.Features())
|
||||
|
||||
entries, err := f.List("")
|
||||
assert.NoError(t, err)
|
||||
assert.Nil(t, entries)
|
||||
|
||||
o, err := f.NewObject("obj")
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
assert.Nil(t, o)
|
||||
|
||||
buf := bytes.NewBufferString("potato")
|
||||
now := time.Now()
|
||||
src := fs.NewStaticObjectInfo("remote", now, int64(buf.Len()), true, nil, nil)
|
||||
o, err = f.Put(buf, src)
|
||||
assert.NoError(t, err)
|
||||
hash, err := o.Hash(fs.HashSHA1)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "3e2e95f5ad970eadfa7e17eaf73da97024aa5359", hash)
|
||||
|
||||
err = f.Mkdir("dir")
|
||||
assert.Error(t, err)
|
||||
|
||||
err = f.Rmdir("dir")
|
||||
assert.Error(t, fs.ErrorDirNotFound)
|
||||
}
|
||||
|
||||
func TestMemoryObject(t *testing.T) {
|
||||
remote := "path/to/object"
|
||||
now := time.Now()
|
||||
content := []byte("potatoXXXXXXXXXXXXX")
|
||||
content = content[:6] // make some extra cap
|
||||
|
||||
o := fs.NewMemoryObject(remote, now, content)
|
||||
|
||||
assert.Equal(t, content, o.Content())
|
||||
assert.Equal(t, fs.MemoryFs, o.Fs())
|
||||
assert.Equal(t, remote, o.Remote())
|
||||
assert.Equal(t, remote, o.String())
|
||||
assert.Equal(t, now, o.ModTime())
|
||||
assert.Equal(t, int64(len(content)), o.Size())
|
||||
assert.Equal(t, true, o.Storable())
|
||||
|
||||
hash, err := o.Hash(fs.HashMD5)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "8ee2027983915ec78acc45027d874316", hash)
|
||||
|
||||
hash, err = o.Hash(fs.HashSHA1)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "3e2e95f5ad970eadfa7e17eaf73da97024aa5359", hash)
|
||||
|
||||
newNow := now.Add(time.Minute)
|
||||
err = o.SetModTime(newNow)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, newNow, o.ModTime())
|
||||
|
||||
checkOpen := func(rc io.ReadCloser, expected string) {
|
||||
actual, err := ioutil.ReadAll(rc)
|
||||
assert.NoError(t, err)
|
||||
err = rc.Close()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expected, string(actual))
|
||||
}
|
||||
|
||||
checkContent := func(o fs.Object, expected string) {
|
||||
rc, err := o.Open()
|
||||
assert.NoError(t, err)
|
||||
checkOpen(rc, expected)
|
||||
}
|
||||
|
||||
checkContent(o, string(content))
|
||||
|
||||
rc, err := o.Open(&fs.RangeOption{Start: 1, End: 3})
|
||||
assert.NoError(t, err)
|
||||
checkOpen(rc, "ot")
|
||||
|
||||
rc, err = o.Open(&fs.SeekOption{Offset: 3})
|
||||
assert.NoError(t, err)
|
||||
checkOpen(rc, "ato")
|
||||
|
||||
// check it fits within the buffer
|
||||
newNow = now.Add(2 * time.Minute)
|
||||
newContent := bytes.NewBufferString("Rutabaga")
|
||||
assert.True(t, newContent.Len() < cap(content)) // fits within cap(content)
|
||||
src := fs.NewStaticObjectInfo(remote, newNow, int64(newContent.Len()), true, nil, nil)
|
||||
err = o.Update(newContent, src)
|
||||
assert.NoError(t, err)
|
||||
checkContent(o, "Rutabaga")
|
||||
assert.Equal(t, newNow, o.ModTime())
|
||||
assert.Equal(t, "Rutaba", string(content)) // check we re-used the buffer
|
||||
|
||||
// not within the buffer
|
||||
newStr := "0123456789"
|
||||
newStr = newStr + newStr + newStr + newStr + newStr + newStr + newStr + newStr + newStr + newStr
|
||||
newContent = bytes.NewBufferString(newStr)
|
||||
assert.True(t, newContent.Len() > cap(content)) // does not fit within cap(content)
|
||||
src = fs.NewStaticObjectInfo(remote, newNow, int64(newContent.Len()), true, nil, nil)
|
||||
err = o.Update(newContent, src)
|
||||
assert.NoError(t, err)
|
||||
checkContent(o, newStr)
|
||||
assert.Equal(t, "Rutaba", string(content)) // check we didn't re-use the buffer
|
||||
|
||||
// now try streaming
|
||||
newStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
newContent = bytes.NewBufferString(newStr)
|
||||
src = fs.NewStaticObjectInfo(remote, newNow, -1, true, nil, nil)
|
||||
err = o.Update(newContent, src)
|
||||
assert.NoError(t, err)
|
||||
checkContent(o, newStr)
|
||||
|
||||
// and zero length
|
||||
newStr = ""
|
||||
newContent = bytes.NewBufferString(newStr)
|
||||
src = fs.NewStaticObjectInfo(remote, newNow, 0, true, nil, nil)
|
||||
err = o.Update(newContent, src)
|
||||
assert.NoError(t, err)
|
||||
checkContent(o, newStr)
|
||||
|
||||
err = o.Remove()
|
||||
assert.Error(t, err)
|
||||
}
|
||||
Loading…
Reference in a new issue