From 72586f17d439202f9b153a666ac88a8c1262741b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 29 Jul 2022 10:54:49 +0300 Subject: [PATCH] shard: fix `GetRange` for objects stored in the write-cache Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/get.go | 14 +-- pkg/local_object_storage/shard/range.go | 22 ++++- pkg/local_object_storage/shard/range_test.go | 93 ++++++++++++++++++++ 3 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 pkg/local_object_storage/shard/range_test.go diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 2647e74d9..70ce128c4 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -87,8 +87,12 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) { return res.Object(), nil } + wc := func(c writecache.Cache) (*objectSDK.Object, error) { + return c.Get(prm.addr) + } + skipMeta := prm.skipMeta || s.GetMode().NoMetabase() - obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small) + obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small, wc) return GetRes{ obj: obj, @@ -97,16 +101,16 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) { } // fetchObjectData looks through writeCache and blobStor to find object. -func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small storFetcher) (*objectSDK.Object, bool, error) { +func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small storFetcher, wc func(w writecache.Cache) (*objectSDK.Object, error)) (*objectSDK.Object, bool, error) { var ( err error res *objectSDK.Object ) if s.hasWriteCache() { - res, err = s.writeCache.Get(addr) - if err == nil { - return res, false, nil + res, err := wc(s.writeCache) + if err == nil || IsErrOutOfRange(err) { + return res, false, err } if writecache.IsErrNotFound(err) { diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 84b7dda97..23f5d3edd 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -3,6 +3,8 @@ package shard import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" + apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) @@ -102,8 +104,26 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) { return obj, nil } + wc := func(c writecache.Cache) (*object.Object, error) { + res, err := c.Get(prm.addr) + if err != nil { + return nil, err + } + + payload := res.Payload() + from := rng.GetOffset() + to := from + rng.GetLength() + if uint64(len(payload)) < to { + return nil, apistatus.ObjectOutOfRange{} + } + + obj := object.New() + obj.SetPayload(payload[from:to]) + return obj, nil + } + skipMeta := prm.skipMeta || s.GetMode().NoMetabase() - obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small) + obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small, wc) return RngRes{ obj: obj, diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go new file mode 100644 index 000000000..fda085f8d --- /dev/null +++ b/pkg/local_object_storage/shard/range_test.go @@ -0,0 +1,93 @@ +package shard_test + +import ( + "testing" + + "github.com/nspcc-dev/neo-go/pkg/util/slice" + "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" + apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" + objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/stretchr/testify/require" +) + +func TestShard_GetRange(t *testing.T) { + t.Run("without write cache", func(t *testing.T) { + testShardGetRange(t, false) + }) + + t.Run("with write cache", func(t *testing.T) { + testShardGetRange(t, true) + }) +} + +func testShardGetRange(t *testing.T, hasWriteCache bool) { + type testCase struct { + hasErr bool + name string + payloadSize int + rng *objectSDK.Range + } + + const ( + writeCacheMaxSize = 1024 + smallObjectSize = 2048 + ) + + newRange := func(off, ln uint64) *objectSDK.Range { + rng := objectSDK.NewRange() + rng.SetOffset(off) + rng.SetLength(ln) + return rng + } + + testCases := []testCase{ + {false, "small object, good", 1024, newRange(11, 123)}, + {true, "small object, out of range", 1024, newRange(10, 1020)}, + {false, "big object, good", 2048, newRange(11, 123)}, + {true, "big object, out of range", 2048, newRange(100, 2000)}, + } + + if hasWriteCache { + testCases = append(testCases, + testCase{false, "object in write-cache, good", 100, newRange(2, 18)}, + testCase{true, "object in write-cache, out of range", 100, newRange(4, 99)}) + } + + sh := newCustomShard(t, t.TempDir(), hasWriteCache, + []writecache.Option{writecache.WithMaxMemSize(0), writecache.WithMaxObjectSize(writeCacheMaxSize)}, + []blobstor.Option{blobstor.WithSmallSizeLimit(smallObjectSize)}) + defer releaseShard(sh, t) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + obj := generateObject(t) + addAttribute(obj, "foo", "bar") + addPayload(obj, tc.payloadSize) + + addr := object.AddressOf(obj) + payload := slice.Copy(obj.Payload()) + + var putPrm shard.PutPrm + putPrm.SetObject(obj) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + var rngPrm shard.RngPrm + rngPrm.SetAddress(addr) + rngPrm.SetRange(tc.rng.GetOffset(), tc.rng.GetLength()) + + res, err := sh.GetRange(rngPrm) + if tc.hasErr { + require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) + } else { + require.Equal(t, + payload[tc.rng.GetOffset():tc.rng.GetOffset()+tc.rng.GetLength()], + res.Object().Payload()) + } + }) + } +}