alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

backend: Remove GetReader

Browse source
This commit is contained in:
Alexander Neumann 2016-01-24 01:00:27 +01:00
parent 2c3a6a6fa9
commit adbe9e2e1c
13 changed files with 26 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
backend/generic.go
View file

@ -3,7 +3,6 @@ package backend
import (
"crypto/sha256"
"errors"
"io"
)
const (
@ -82,39 +81,3 @@ outer:
return IDSize, nil
}
// wrap around io.LimitedReader that implements io.ReadCloser
type blobReader struct {
cl io.Closer
rd io.Reader
closed bool
}
func (l *blobReader) Read(p []byte) (int, error) {
n, err := l.rd.Read(p)
if err == io.EOF {
l.Close()
}
return n, err
}
func (l *blobReader) Close() error {
if l == nil {
return nil
}
if !l.closed {
err := l.cl.Close()
l.closed = true
return err
}
return nil
}
// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
// implements the Close() method.
func LimitReadCloser(r io.ReadCloser, n int64) *blobReader {
return &blobReader{cl: r, rd: io.LimitReader(r, n)}
}

4
backend/interface.go
View file

@ -25,10 +25,6 @@ type Backend interface {
// has been called on the returned Blob.
Create() (Blob, error)
// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length.
GetReader(t Type, name string, offset, length uint) (io.ReadCloser, error)
// Test a boolean value whether a Blob with the name and type exists.
Test(t Type, name string) (bool, error)

7
backend/local/backend_test.go
View file

@ -44,13 +44,6 @@ func TestLocalBackendConfig(t *testing.T) {
test.TestConfig(t)
}
func TestLocalBackendGetReader(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGetReader(t)
}
func TestLocalBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

27
backend/local/local.go
View file

@ -196,33 +196,6 @@ func dirname(base string, t backend.Type, name string) string {
return filepath.Join(base, n)
}
// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length. If length is 0, the reader reads until EOF.
func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
f, err := os.Open(filename(b.p, t, name))
if err != nil {
return nil, err
}
b.mu.Lock()
open, _ := b.open[filename(b.p, t, name)]
b.open[filename(b.p, t, name)] = append(open, f)
b.mu.Unlock()
if offset > 0 {
_, err = f.Seek(int64(offset), 0)
if err != nil {
return nil, err
}
}
if length == 0 {
return f, nil
}
return backend.LimitReadCloser(f, int64(length)), nil
}
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {

7
backend/mem/backend_test.go
View file

@ -44,13 +44,6 @@ func TestMemBackendConfig(t *testing.T) {
test.TestConfig(t)
}
func TestMemBackendGetReader(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGetReader(t)
}
func TestMemBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

36
backend/mem/mem_backend.go
View file

@ -41,10 +41,6 @@ func New() *MemoryBackend {
return memCreate(be)
}
be.MockBackend.GetReaderFn = func(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
return memGetReader(be, t, name, offset, length)
}
be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
return memLoad(be, h, p, off)
}
@ -133,38 +129,6 @@ func memCreate(be *MemoryBackend) (backend.Blob, error) {
return blob, nil
}
func memGetReader(be *MemoryBackend, t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
be.m.Lock()
defer be.m.Unlock()
if t == backend.Config {
name = ""
}
debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length)
if _, ok := be.data[entry{t, name}]; !ok {
return nil, errors.New("no such data")
}
buf := be.data[entry{t, name}]
if offset > uint(len(buf)) {
return nil, errors.New("offset beyond end of file")
}
buf = buf[offset:]
if length > 0 {
if length > uint(len(buf)) {
length = uint(len(buf))
}
buf = buf[:length]
}
return backend.ReadCloser(bytes.NewReader(buf)), nil
}
func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) {
be.m.Lock()
defer be.m.Unlock()

32
backend/mock_backend.go
View file

@ -1,23 +1,19 @@
package backend
import (
"errors"
"io"
)
import "errors"
// MockBackend implements a backend whose functions can be specified. This
// should only be used for tests.
type MockBackend struct {
CloseFn func() error
CreateFn func() (Blob, error)
LoadFn func(h Handle, p []byte, off int64) (int, error)
StatFn func(h Handle) (BlobInfo, error)
GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error)
ListFn func(Type, <-chan struct{}) <-chan string
RemoveFn func(Type, string) error
TestFn func(Type, string) (bool, error)
DeleteFn func() error
LocationFn func() string
CloseFn func() error
CreateFn func() (Blob, error)
LoadFn func(h Handle, p []byte, off int64) (int, error)
StatFn func(h Handle) (BlobInfo, error)
ListFn func(Type, <-chan struct{}) <-chan string
RemoveFn func(Type, string) error
TestFn func(Type, string) (bool, error)
DeleteFn func() error
LocationFn func() string
}
func (m *MockBackend) Close() error {
@ -60,14 +56,6 @@ func (m *MockBackend) Stat(h Handle) (BlobInfo, error) {
return m.StatFn(h)
}
func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) {
if m.GetReaderFn == nil {
return nil, errors.New("not implemented")
}
return m.GetReaderFn(t, name, offset, len)
}
func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string {
if m.ListFn == nil {
ch := make(chan string)

7
backend/s3/backend_test.go
View file

@ -44,13 +44,6 @@ func TestS3BackendConfig(t *testing.T) {
test.TestConfig(t)
}
func TestS3BackendGetReader(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGetReader(t)
}
func TestS3BackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

25
backend/s3/s3.go
View file

@ -145,31 +145,6 @@ func (be *S3Backend) Create() (backend.Blob, error) {
return &blob, nil
}
// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length. If length is 0, the reader reads until EOF.
func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
debug.Log("s3.GetReader", "%v %v, offset %v len %v", t, name, offset, length)
path := s3path(t, name)
obj, err := be.client.GetObject(be.bucketname, path)
if err != nil {
debug.Log("s3.GetReader", " err %v", err)
return nil, err
}
if offset > 0 {
_, err = obj.Seek(int64(offset), 0)
if err != nil {
return nil, err
}
}
if length == 0 {
return obj, nil
}
return backend.LimitReadCloser(obj, int64(length)), nil
}
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) {

7
backend/sftp/backend_test.go
View file

@ -44,13 +44,6 @@ func TestSftpBackendConfig(t *testing.T) {
test.TestConfig(t)
}
func TestSftpBackendGetReader(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGetReader(t)
}
func TestSftpBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

22
backend/sftp/sftp.go
View file

@ -344,28 +344,6 @@ func (r *SFTP) dirname(t backend.Type, name string) string {
return Join(r.p, n)
}
// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length. If length is 0, the reader reads until EOF.
func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
f, err := r.c.Open(r.filename(t, name))
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(int64(offset), 0)
if err != nil {
return nil, err
}
}
if length == 0 {
return f, nil
}
return backend.LimitReadCloser(f, int64(length)), nil
}
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) {

7
backend/test/backend_test.go
View file

@ -44,13 +44,6 @@ func TestTestBackendConfig(t *testing.T) {
test.TestConfig(t)
}
func TestTestBackendGetReader(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGetReader(t)
}
func TestTestBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

96
backend/test/tests.go
View file

@ -152,7 +152,7 @@ func TestConfig(t testing.TB) {
var testString = "Config"
// create config and read it back
_, err := b.GetReader(backend.Config, "", 0, 0)
_, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
@ -175,76 +175,17 @@ func TestConfig(t testing.TB) {
// try accessing the config with different names, should all return the
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
rd, err := b.GetReader(backend.Config, name, 0, 0)
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
if err != nil {
t.Fatalf("unable to read config with name %q: %v", name, err)
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
t.Fatalf("read config error: %v", err)
}
err = rd.Close()
if err != nil {
t.Fatalf("close error: %v", err)
}
if string(buf) != testString {
t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf))
}
}
}
// TestGetReader tests various ways the GetReader function can be called.
func TestGetReader(t testing.TB) {
b := open(t)
defer close(t)
length := rand.Intn(1<<24) + 2000
data := make([]byte, length)
_, err := io.ReadFull(crand.Reader, data)
OK(t, err)
blob, err := b.Create()
OK(t, err)
id := backend.Hash(data)
_, err = blob.Write([]byte(data))
OK(t, err)
OK(t, blob.Finalize(backend.Data, id.String()))
for i := 0; i < 500; i++ {
l := rand.Intn(length + 2000)
o := rand.Intn(length + 2000)
d := data
if o < len(d) {
d = d[o:]
} else {
o = len(d)
d = d[:0]
}
if l > 0 && l < len(d) {
d = d[:l]
}
rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l))
OK(t, err)
buf, err := ioutil.ReadAll(rd)
OK(t, err)
if !bytes.Equal(buf, d) {
t.Fatalf("data not equal")
}
}
OK(t, b.Remove(backend.Data, id.String()))
}
// TestLoad tests the backend's Load function.
func TestLoad(t testing.TB) {
b := open(t)
@ -360,12 +301,8 @@ func TestWrite(t testing.TB) {
name := fmt.Sprintf("%s-%d", id, i)
OK(t, blob.Finalize(backend.Data, name))
rd, err := b.GetReader(backend.Data, name, 0, 0)
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil)
OK(t, err)
buf, err := ioutil.ReadAll(rd)
OK(t, err)
if len(buf) != len(data) {
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
}
@ -436,12 +373,13 @@ func TestBackend(t testing.TB) {
OK(t, err)
Assert(t, !ret, "blob was found to exist before creating")
// try to open not existing blob
_, err = b.GetReader(tpe, id.String(), 0, 0)
// try to stat a not existing blob
h := backend.Handle{Type: tpe, Name: id.String()}
_, err = b.Stat(h)
Assert(t, err != nil, "blob data could be extracted before creation")
// try to read not existing blob
_, err = b.GetReader(tpe, id.String(), 0, 1)
_, err = b.Load(h, nil, 0)
Assert(t, err != nil, "blob reader could be obtained before creation")
// try to get string out, should fail
@ -454,24 +392,22 @@ func TestBackend(t testing.TB) {
for _, test := range testStrings {
store(t, b, tpe, []byte(test.data))
// test GetReader()
rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data)))
// test Load()
h := backend.Handle{Type: tpe, Name: test.id}
buf, err := backend.LoadAll(b, h, nil)
OK(t, err)
Assert(t, rd != nil, "GetReader() returned nil")
read(t, rd, []byte(test.data))
OK(t, rd.Close())
Equals(t, test.data, string(buf))
// try to read it out with an offset and a length
start := 1
end := len(test.data) - 2
length := end - start
rd, err = b.GetReader(tpe, test.id, uint(start), uint(length))
OK(t, err)
Assert(t, rd != nil, "GetReader() returned nil")
read(t, rd, []byte(test.data[start:end]))
OK(t, rd.Close())
buf2 := make([]byte, length)
n, err := b.Load(h, buf2, int64(start))
OK(t, err)
Equals(t, length, n)
Equals(t, test.data[start:end], string(buf2))
}
// test adding the first file again