forked from TrueCloudLab/frostfs-node
shard: fix GetRange
for objects stored in the write-cache
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
parent
07465849a4
commit
72586f17d4
3 changed files with 123 additions and 6 deletions
|
@ -87,8 +87,12 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) {
|
||||||
return res.Object(), nil
|
return res.Object(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
wc := func(c writecache.Cache) (*objectSDK.Object, error) {
|
||||||
|
return c.Get(prm.addr)
|
||||||
|
}
|
||||||
|
|
||||||
skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
|
skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
|
||||||
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
|
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small, wc)
|
||||||
|
|
||||||
return GetRes{
|
return GetRes{
|
||||||
obj: obj,
|
obj: obj,
|
||||||
|
@ -97,16 +101,16 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchObjectData looks through writeCache and blobStor to find object.
|
// fetchObjectData looks through writeCache and blobStor to find object.
|
||||||
func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small storFetcher) (*objectSDK.Object, bool, error) {
|
func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small storFetcher, wc func(w writecache.Cache) (*objectSDK.Object, error)) (*objectSDK.Object, bool, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
res *objectSDK.Object
|
res *objectSDK.Object
|
||||||
)
|
)
|
||||||
|
|
||||||
if s.hasWriteCache() {
|
if s.hasWriteCache() {
|
||||||
res, err = s.writeCache.Get(addr)
|
res, err := wc(s.writeCache)
|
||||||
if err == nil {
|
if err == nil || IsErrOutOfRange(err) {
|
||||||
return res, false, nil
|
return res, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if writecache.IsErrNotFound(err) {
|
if writecache.IsErrNotFound(err) {
|
||||||
|
|
|
@ -3,6 +3,8 @@ package shard
|
||||||
import (
|
import (
|
||||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
|
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
|
||||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
|
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
|
||||||
|
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
|
||||||
|
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
@ -102,8 +104,26 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) {
|
||||||
return obj, nil
|
return obj, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
wc := func(c writecache.Cache) (*object.Object, error) {
|
||||||
|
res, err := c.Get(prm.addr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
payload := res.Payload()
|
||||||
|
from := rng.GetOffset()
|
||||||
|
to := from + rng.GetLength()
|
||||||
|
if uint64(len(payload)) < to {
|
||||||
|
return nil, apistatus.ObjectOutOfRange{}
|
||||||
|
}
|
||||||
|
|
||||||
|
obj := object.New()
|
||||||
|
obj.SetPayload(payload[from:to])
|
||||||
|
return obj, nil
|
||||||
|
}
|
||||||
|
|
||||||
skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
|
skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
|
||||||
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
|
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small, wc)
|
||||||
|
|
||||||
return RngRes{
|
return RngRes{
|
||||||
obj: obj,
|
obj: obj,
|
||||||
|
|
93
pkg/local_object_storage/shard/range_test.go
Normal file
93
pkg/local_object_storage/shard/range_test.go
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
package shard_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neo-go/pkg/util/slice"
|
||||||
|
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
||||||
|
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
|
||||||
|
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
|
||||||
|
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
|
||||||
|
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
|
||||||
|
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestShard_GetRange(t *testing.T) {
|
||||||
|
t.Run("without write cache", func(t *testing.T) {
|
||||||
|
testShardGetRange(t, false)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("with write cache", func(t *testing.T) {
|
||||||
|
testShardGetRange(t, true)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testShardGetRange(t *testing.T, hasWriteCache bool) {
|
||||||
|
type testCase struct {
|
||||||
|
hasErr bool
|
||||||
|
name string
|
||||||
|
payloadSize int
|
||||||
|
rng *objectSDK.Range
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
writeCacheMaxSize = 1024
|
||||||
|
smallObjectSize = 2048
|
||||||
|
)
|
||||||
|
|
||||||
|
newRange := func(off, ln uint64) *objectSDK.Range {
|
||||||
|
rng := objectSDK.NewRange()
|
||||||
|
rng.SetOffset(off)
|
||||||
|
rng.SetLength(ln)
|
||||||
|
return rng
|
||||||
|
}
|
||||||
|
|
||||||
|
testCases := []testCase{
|
||||||
|
{false, "small object, good", 1024, newRange(11, 123)},
|
||||||
|
{true, "small object, out of range", 1024, newRange(10, 1020)},
|
||||||
|
{false, "big object, good", 2048, newRange(11, 123)},
|
||||||
|
{true, "big object, out of range", 2048, newRange(100, 2000)},
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasWriteCache {
|
||||||
|
testCases = append(testCases,
|
||||||
|
testCase{false, "object in write-cache, good", 100, newRange(2, 18)},
|
||||||
|
testCase{true, "object in write-cache, out of range", 100, newRange(4, 99)})
|
||||||
|
}
|
||||||
|
|
||||||
|
sh := newCustomShard(t, t.TempDir(), hasWriteCache,
|
||||||
|
[]writecache.Option{writecache.WithMaxMemSize(0), writecache.WithMaxObjectSize(writeCacheMaxSize)},
|
||||||
|
[]blobstor.Option{blobstor.WithSmallSizeLimit(smallObjectSize)})
|
||||||
|
defer releaseShard(sh, t)
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
obj := generateObject(t)
|
||||||
|
addAttribute(obj, "foo", "bar")
|
||||||
|
addPayload(obj, tc.payloadSize)
|
||||||
|
|
||||||
|
addr := object.AddressOf(obj)
|
||||||
|
payload := slice.Copy(obj.Payload())
|
||||||
|
|
||||||
|
var putPrm shard.PutPrm
|
||||||
|
putPrm.SetObject(obj)
|
||||||
|
|
||||||
|
_, err := sh.Put(putPrm)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var rngPrm shard.RngPrm
|
||||||
|
rngPrm.SetAddress(addr)
|
||||||
|
rngPrm.SetRange(tc.rng.GetOffset(), tc.rng.GetLength())
|
||||||
|
|
||||||
|
res, err := sh.GetRange(rngPrm)
|
||||||
|
if tc.hasErr {
|
||||||
|
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
|
||||||
|
} else {
|
||||||
|
require.Equal(t,
|
||||||
|
payload[tc.rng.GetOffset():tc.rng.GetOffset()+tc.rng.GetLength()],
|
||||||
|
res.Object().Payload())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue