From adbe9e2e1ceb29eef2880fa06343f6fae39eb6b9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 01:00:27 +0100 Subject: [PATCH] backend: Remove GetReader --- backend/generic.go | 37 -------------- backend/interface.go | 4 -- backend/local/backend_test.go | 7 --- backend/local/local.go | 27 ---------- backend/mem/backend_test.go | 7 --- backend/mem/mem_backend.go | 36 ------------- backend/mock_backend.go | 32 ++++-------- backend/s3/backend_test.go | 7 --- backend/s3/s3.go | 25 --------- backend/sftp/backend_test.go | 7 --- backend/sftp/sftp.go | 22 -------- backend/test/backend_test.go | 7 --- backend/test/tests.go | 96 ++++++----------------------------- 13 files changed, 26 insertions(+), 288 deletions(-) diff --git a/backend/generic.go b/backend/generic.go index 4c736a6b2..d8c72b9e0 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -3,7 +3,6 @@ package backend import ( "crypto/sha256" "errors" - "io" ) const ( @@ -82,39 +81,3 @@ outer: return IDSize, nil } - -// wrap around io.LimitedReader that implements io.ReadCloser -type blobReader struct { - cl io.Closer - rd io.Reader - closed bool -} - -func (l *blobReader) Read(p []byte) (int, error) { - n, err := l.rd.Read(p) - if err == io.EOF { - l.Close() - } - - return n, err -} - -func (l *blobReader) Close() error { - if l == nil { - return nil - } - - if !l.closed { - err := l.cl.Close() - l.closed = true - return err - } - - return nil -} - -// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also -// implements the Close() method. -func LimitReadCloser(r io.ReadCloser, n int64) *blobReader { - return &blobReader{cl: r, rd: io.LimitReader(r, n)} -} diff --git a/backend/interface.go b/backend/interface.go index f215c6733..24424dd96 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -25,10 +25,6 @@ type Backend interface { // has been called on the returned Blob. Create() (Blob, error) - // GetReader returns an io.ReadCloser for the Blob with the given name of - // type t at offset and length. - GetReader(t Type, name string, offset, length uint) (io.ReadCloser, error) - // Test a boolean value whether a Blob with the name and type exists. Test(t Type, name string) (bool, error) diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 8cc244e85..c6032be6b 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -44,13 +44,6 @@ func TestLocalBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestLocalBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestLocalBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/local/local.go b/backend/local/local.go index 554710d37..0d12e4109 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -196,33 +196,6 @@ func dirname(base string, t backend.Type, name string) string { return filepath.Join(base, n) } -// GetReader returns an io.ReadCloser for the Blob with the given name of -// type t at offset and length. If length is 0, the reader reads until EOF. -func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - f, err := os.Open(filename(b.p, t, name)) - if err != nil { - return nil, err - } - - b.mu.Lock() - open, _ := b.open[filename(b.p, t, name)] - b.open[filename(b.p, t, name)] = append(open, f) - b.mu.Unlock() - - if offset > 0 { - _, err = f.Seek(int64(offset), 0) - if err != nil { - return nil, err - } - } - - if length == 0 { - return f, nil - } - - return backend.LimitReadCloser(f, int64(length)), nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { diff --git a/backend/mem/backend_test.go b/backend/mem/backend_test.go index 233b30f60..681d0a680 100644 --- a/backend/mem/backend_test.go +++ b/backend/mem/backend_test.go @@ -44,13 +44,6 @@ func TestMemBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestMemBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestMemBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 6e80735e3..b999efd1d 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -41,10 +41,6 @@ func New() *MemoryBackend { return memCreate(be) } - be.MockBackend.GetReaderFn = func(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - return memGetReader(be, t, name, offset, length) - } - be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) { return memLoad(be, h, p, off) } @@ -133,38 +129,6 @@ func memCreate(be *MemoryBackend) (backend.Blob, error) { return blob, nil } -func memGetReader(be *MemoryBackend, t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - be.m.Lock() - defer be.m.Unlock() - - if t == backend.Config { - name = "" - } - - debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length) - - if _, ok := be.data[entry{t, name}]; !ok { - return nil, errors.New("no such data") - } - - buf := be.data[entry{t, name}] - if offset > uint(len(buf)) { - return nil, errors.New("offset beyond end of file") - } - - buf = buf[offset:] - - if length > 0 { - if length > uint(len(buf)) { - length = uint(len(buf)) - } - - buf = buf[:length] - } - - return backend.ReadCloser(bytes.NewReader(buf)), nil -} - func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { be.m.Lock() defer be.m.Unlock() diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 4d142f846..1075292d6 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -1,23 +1,19 @@ package backend -import ( - "errors" - "io" -) +import "errors" // MockBackend implements a backend whose functions can be specified. This // should only be used for tests. type MockBackend struct { - CloseFn func() error - CreateFn func() (Blob, error) - LoadFn func(h Handle, p []byte, off int64) (int, error) - StatFn func(h Handle) (BlobInfo, error) - GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error) - ListFn func(Type, <-chan struct{}) <-chan string - RemoveFn func(Type, string) error - TestFn func(Type, string) (bool, error) - DeleteFn func() error - LocationFn func() string + CloseFn func() error + CreateFn func() (Blob, error) + LoadFn func(h Handle, p []byte, off int64) (int, error) + StatFn func(h Handle) (BlobInfo, error) + ListFn func(Type, <-chan struct{}) <-chan string + RemoveFn func(Type, string) error + TestFn func(Type, string) (bool, error) + DeleteFn func() error + LocationFn func() string } func (m *MockBackend) Close() error { @@ -60,14 +56,6 @@ func (m *MockBackend) Stat(h Handle) (BlobInfo, error) { return m.StatFn(h) } -func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { - if m.GetReaderFn == nil { - return nil, errors.New("not implemented") - } - - return m.GetReaderFn(t, name, offset, len) -} - func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string { if m.ListFn == nil { ch := make(chan string) diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index d0d15f51a..cc981cf18 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -44,13 +44,6 @@ func TestS3BackendConfig(t *testing.T) { test.TestConfig(t) } -func TestS3BackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestS3BackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 444fe2a73..a46cdb22d 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -145,31 +145,6 @@ func (be *S3Backend) Create() (backend.Blob, error) { return &blob, nil } -// GetReader returns an io.ReadCloser for the Blob with the given name of -// type t at offset and length. If length is 0, the reader reads until EOF. -func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - debug.Log("s3.GetReader", "%v %v, offset %v len %v", t, name, offset, length) - path := s3path(t, name) - obj, err := be.client.GetObject(be.bucketname, path) - if err != nil { - debug.Log("s3.GetReader", " err %v", err) - return nil, err - } - - if offset > 0 { - _, err = obj.Seek(int64(offset), 0) - if err != nil { - return nil, err - } - } - - if length == 0 { - return obj, nil - } - - return backend.LimitReadCloser(obj, int64(length)), nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index ef4ac9129..217f34b11 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -44,13 +44,6 @@ func TestSftpBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestSftpBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestSftpBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index f23ec1db5..e40a3cd00 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -344,28 +344,6 @@ func (r *SFTP) dirname(t backend.Type, name string) string { return Join(r.p, n) } -// GetReader returns an io.ReadCloser for the Blob with the given name of -// type t at offset and length. If length is 0, the reader reads until EOF. -func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - f, err := r.c.Open(r.filename(t, name)) - if err != nil { - return nil, err - } - - if offset > 0 { - _, err = f.Seek(int64(offset), 0) - if err != nil { - return nil, err - } - } - - if length == 0 { - return f, nil - } - - return backend.LimitReadCloser(f, int64(length)), nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index 44b330a0a..733fecaaa 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -44,13 +44,6 @@ func TestTestBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestTestBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestTestBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/test/tests.go b/backend/test/tests.go index bd12fdc4a..009ef1172 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -152,7 +152,7 @@ func TestConfig(t testing.TB) { var testString = "Config" // create config and read it back - _, err := b.GetReader(backend.Config, "", 0, 0) + _, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil) if err == nil { t.Fatalf("did not get expected error for non-existing config") } @@ -175,76 +175,17 @@ func TestConfig(t testing.TB) { // try accessing the config with different names, should all return the // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - rd, err := b.GetReader(backend.Config, name, 0, 0) + buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil) if err != nil { t.Fatalf("unable to read config with name %q: %v", name, err) } - buf, err := ioutil.ReadAll(rd) - if err != nil { - t.Fatalf("read config error: %v", err) - } - - err = rd.Close() - if err != nil { - t.Fatalf("close error: %v", err) - } - if string(buf) != testString { t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf)) } } } -// TestGetReader tests various ways the GetReader function can be called. -func TestGetReader(t testing.TB) { - b := open(t) - defer close(t) - - length := rand.Intn(1<<24) + 2000 - - data := make([]byte, length) - _, err := io.ReadFull(crand.Reader, data) - OK(t, err) - - blob, err := b.Create() - OK(t, err) - - id := backend.Hash(data) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) - - for i := 0; i < 500; i++ { - l := rand.Intn(length + 2000) - o := rand.Intn(length + 2000) - - d := data - if o < len(d) { - d = d[o:] - } else { - o = len(d) - d = d[:0] - } - - if l > 0 && l < len(d) { - d = d[:l] - } - - rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l)) - OK(t, err) - buf, err := ioutil.ReadAll(rd) - OK(t, err) - - if !bytes.Equal(buf, d) { - t.Fatalf("data not equal") - } - } - - OK(t, b.Remove(backend.Data, id.String())) -} - // TestLoad tests the backend's Load function. func TestLoad(t testing.TB) { b := open(t) @@ -360,12 +301,8 @@ func TestWrite(t testing.TB) { name := fmt.Sprintf("%s-%d", id, i) OK(t, blob.Finalize(backend.Data, name)) - rd, err := b.GetReader(backend.Data, name, 0, 0) + buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil) OK(t, err) - - buf, err := ioutil.ReadAll(rd) - OK(t, err) - if len(buf) != len(data) { t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) } @@ -436,12 +373,13 @@ func TestBackend(t testing.TB) { OK(t, err) Assert(t, !ret, "blob was found to exist before creating") - // try to open not existing blob - _, err = b.GetReader(tpe, id.String(), 0, 0) + // try to stat a not existing blob + h := backend.Handle{Type: tpe, Name: id.String()} + _, err = b.Stat(h) Assert(t, err != nil, "blob data could be extracted before creation") // try to read not existing blob - _, err = b.GetReader(tpe, id.String(), 0, 1) + _, err = b.Load(h, nil, 0) Assert(t, err != nil, "blob reader could be obtained before creation") // try to get string out, should fail @@ -454,24 +392,22 @@ func TestBackend(t testing.TB) { for _, test := range testStrings { store(t, b, tpe, []byte(test.data)) - // test GetReader() - rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data))) + // test Load() + h := backend.Handle{Type: tpe, Name: test.id} + buf, err := backend.LoadAll(b, h, nil) OK(t, err) - Assert(t, rd != nil, "GetReader() returned nil") - - read(t, rd, []byte(test.data)) - OK(t, rd.Close()) + Equals(t, test.data, string(buf)) // try to read it out with an offset and a length start := 1 end := len(test.data) - 2 length := end - start - rd, err = b.GetReader(tpe, test.id, uint(start), uint(length)) - OK(t, err) - Assert(t, rd != nil, "GetReader() returned nil") - read(t, rd, []byte(test.data[start:end])) - OK(t, rd.Close()) + buf2 := make([]byte, length) + n, err := b.Load(h, buf2, int64(start)) + OK(t, err) + Equals(t, length, n) + Equals(t, test.data[start:end], string(buf2)) } // test adding the first file again