TrueCloudLab/frostfs-node
19
1
Fork 27
Code Issues 91 Pull requests 8 Projects Releases 1 Activity Actions

[#1686] blobstor: Add generic tests

Browse source 
This tests check that each blobstor component behaves similarly when
same methods are being used. It is intended to serve as a specification
for all future components.

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-08-22 17:16:35 +03:00 committed by fyrchik
parent b2d4cc556e
commit 0b95a21701
11 changed files with 518 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go Normal file
View file

@ -0,0 +1,32 @@
package blobovniczatree
import (
"os"
"path/filepath"
"strconv"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"go.uber.org/zap/zaptest"
)
func TestGeneric(t *testing.T) {
const maxObjectSize = 1 << 16
defer func() { _ = os.RemoveAll(t.Name()) }()
var n int
newTree := func(t *testing.T) common.Storage {
dir := filepath.Join(t.Name(), strconv.Itoa(n))
return NewBlobovniczaTree(
WithLogger(zaptest.NewLogger(t)),
WithObjectSizeLimit(maxObjectSize),
WithBlobovniczaShallowWidth(2),
WithBlobovniczaShallowDepth(2),
WithRootPath(dir),
WithBlobovniczaSize(1<<20))
}
blobstortest.TestAll(t, newTree, 1024, maxObjectSize)
}

2
pkg/local_object_storage/blobstor/blobovniczatree/get.go
View file

@ -143,5 +143,5 @@ func (b *Blobovniczas) getObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.G
return common.GetRes{}, fmt.Errorf("could not unmarshal the object: %w", err)
}
return common.GetRes{Object: obj}, nil
return common.GetRes{Object: obj, RawData: data}, nil
}

5
pkg/local_object_storage/blobstor/blobovniczatree/iterate.go
View file

@ -26,11 +26,16 @@ func (b *Blobovniczas) Iterate(prm common.IteratePrm) (common.IterateRes, error)
return fmt.Errorf("could not decompress object data: %w", err)
}
if prm.Handler != nil {
return prm.Handler(common.IterationElement{
Address: elem.Address(),
ObjectData: data,
StorageID: []byte(p),
})
}
return prm.LazyHandler(elem.Address(), func() ([]byte, error) {
return data, err
})
})
subPrm.DecodeAddresses()

2
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -222,7 +222,7 @@ func (t *FSTree) Put(prm common.PutPrm) (common.PutRes, error) {
if !prm.DontCompress {
prm.RawData = t.Compress(prm.RawData)
}
return common.PutRes{}, os.WriteFile(p, prm.RawData, t.Permissions)
return common.PutRes{StorageID: []byte{}}, os.WriteFile(p, prm.RawData, t.Permissions)
}
// PutStream puts executes handler on a file opened for write.

26
pkg/local_object_storage/blobstor/fstree/generic_test.go Normal file
View file

@ -0,0 +1,26 @@
package fstree
import (
"os"
"path/filepath"
"strconv"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
)
func TestGeneric(t *testing.T) {
defer func() { _ = os.RemoveAll(t.Name()) }()
var n int
newTree := func(t *testing.T) common.Storage {
dir := filepath.Join(t.Name(), strconv.Itoa(n))
return New(
WithPath(dir),
WithDepth(2),
WithDirNameLen(2))
}
blobstortest.TestAll(t, newTree, 2048, 16*1024)
}

91
pkg/local_object_storage/blobstor/internal/blobstortest/common.go Normal file
View file

@ -0,0 +1,91 @@
package blobstortest
import (
"math/rand"
"testing"
objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
// Constructor constructs blobstor component.
// Each call must create a component using different file-system path.
type Constructor = func(t *testing.T) common.Storage
// objectDesc is a helper structure to avoid multiple `Marshal` invokes during tests.
type objectDesc struct {
obj *objectSDK.Object
addr oid.Address
raw []byte
storageID []byte
}
func TestAll(t *testing.T, cons Constructor, min, max uint64) {
t.Run("get", func(t *testing.T) {
TestGet(t, cons, min, max)
})
t.Run("get range", func(t *testing.T) {
TestGetRange(t, cons, min, max)
})
t.Run("delete", func(t *testing.T) {
TestDelete(t, cons, min, max)
})
t.Run("exists", func(t *testing.T) {
TestExists(t, cons, min, max)
})
t.Run("iterate", func(t *testing.T) {
TestIterate(t, cons, min, max)
})
}
func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objectDesc {
objects := make([]objectDesc, count)
for i := range objects {
objects[i].obj = NewObject(min + uint64(rand.Intn(int(max-min+1)))) // not too large
objects[i].addr = objectCore.AddressOf(objects[i].obj)
raw, err := objects[i].obj.Marshal()
require.NoError(t, err)
objects[i].raw = raw
}
for i := range objects {
var prm common.PutPrm
prm.Address = objects[i].addr
prm.Object = objects[i].obj
prm.RawData = objects[i].raw
putRes, err := s.Put(prm)
require.NoError(t, err)
objects[i].storageID = putRes.StorageID
}
return objects
}
// NewObject creates a regular object of specified size with a random payload.
func NewObject(sz uint64) *objectSDK.Object {
raw := objectSDK.New()
raw.SetID(oidtest.ID())
raw.SetContainerID(cidtest.ID())
payload := make([]byte, sz)
rand.Read(payload)
raw.SetPayload(payload)
// fit the binary size to the required
data, _ := raw.Marshal()
if ln := uint64(len(data)); ln > sz {
raw.SetPayload(raw.Payload()[:sz-(ln-sz)])
}
return raw
}

82
pkg/local_object_storage/blobstor/internal/blobstortest/delete.go Normal file
View file

@ -0,0 +1,82 @@
package blobstortest
import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestDelete(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Init())
t.Cleanup(func() { require.NoError(t, s.Close()) })
objects := prepare(t, 4, s, min, max)
t.Run("delete non-existent", func(t *testing.T) {
var prm common.DeletePrm
prm.Address = oidtest.Address()
_, err := s.Delete(prm)
require.Error(t, err, new(apistatus.ObjectNotFound))
})
t.Run("with storage ID", func(t *testing.T) {
var prm common.DeletePrm
prm.Address = objects[0].addr
prm.StorageID = objects[0].storageID
_, err := s.Delete(prm)
require.NoError(t, err)
t.Run("exists fail", func(t *testing.T) {
prm := common.ExistsPrm{Address: oidtest.Address()}
res, err := s.Exists(prm)
require.NoError(t, err)
require.False(t, res.Exists)
})
t.Run("get fail", func(t *testing.T) {
prm := common.GetPrm{Address: oidtest.Address()}
_, err := s.Get(prm)
require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
})
t.Run("getrange fail", func(t *testing.T) {
prm := common.GetRangePrm{Address: oidtest.Address()}
_, err := s.GetRange(prm)
require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
})
})
t.Run("without storage ID", func(t *testing.T) {
var prm common.DeletePrm
prm.Address = objects[1].addr
_, err := s.Delete(prm)
require.NoError(t, err)
})
t.Run("delete twice", func(t *testing.T) {
var prm common.DeletePrm
prm.Address = objects[2].addr
prm.StorageID = objects[2].storageID
_, err := s.Delete(prm)
require.NoError(t, err)
_, err = s.Delete(prm)
require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
})
t.Run("non-deleted object is still available", func(t *testing.T) {
var prm common.GetPrm
prm.Address = objects[3].addr
prm.Raw = true
res, err := s.Get(prm)
require.NoError(t, err)
require.Equal(t, objects[3].raw, res.RawData)
})
}

31
pkg/local_object_storage/blobstor/internal/blobstortest/exists.go Normal file
View file

@ -0,0 +1,31 @@
package blobstortest
import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestExists(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Init())
t.Cleanup(func() { require.NoError(t, s.Close()) })
objects := prepare(t, 1, s, min, max)
t.Run("missing object", func(t *testing.T) {
prm := common.ExistsPrm{Address: oidtest.Address()}
res, err := s.Exists(prm)
require.NoError(t, err)
require.False(t, res.Exists)
})
var prm common.ExistsPrm
prm.Address = objects[0].addr
res, err := s.Exists(prm)
require.NoError(t, err)
require.True(t, res.Exists)
}

50
pkg/local_object_storage/blobstor/internal/blobstortest/get.go Normal file
View file

@ -0,0 +1,50 @@
package blobstortest
import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestGet(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Init())
t.Cleanup(func() { require.NoError(t, s.Close()) })
objects := prepare(t, 2, s, min, max)
t.Run("missing object", func(t *testing.T) {
gPrm := common.GetPrm{Address: oidtest.Address()}
_, err := s.Get(gPrm)
require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
})
for i := range objects {
var gPrm common.GetPrm
gPrm.Address = objects[i].addr
// With storage ID.
gPrm.StorageID = objects[i].storageID
res, err := s.Get(gPrm)
require.NoError(t, err)
require.Equal(t, objects[i].obj, res.Object)
// Without storage ID.
gPrm.StorageID = nil
res, err = s.Get(gPrm)
require.NoError(t, err)
require.Equal(t, objects[i].obj, res.Object)
// With raw flag.
gPrm.StorageID = objects[i].storageID
gPrm.Raw = true
res, err = s.Get(gPrm)
require.NoError(t, err)
require.Equal(t, objects[i].raw, res.RawData)
}
}

84
pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go Normal file
View file

@ -0,0 +1,84 @@
package blobstortest
import (
"math"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestGetRange(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Init())
t.Cleanup(func() { require.NoError(t, s.Close()) })
objects := prepare(t, 1, s, min, max)
t.Run("missing object", func(t *testing.T) {
gPrm := common.GetRangePrm{Address: oidtest.Address()}
_, err := s.GetRange(gPrm)
require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
})
payload := objects[0].obj.Payload()
var start, stop uint64 = 11, 100
if uint64(len(payload)) < stop {
panic("unexpected: invalid test object generated")
}
var gPrm common.GetRangePrm
gPrm.Address = objects[0].addr
gPrm.Range.SetOffset(start)
gPrm.Range.SetLength(stop - start)
t.Run("without storage ID", func(t *testing.T) {
// Without storage ID.
res, err := s.GetRange(gPrm)
require.NoError(t, err)
require.Equal(t, payload[start:stop], res.Data)
})
t.Run("with storage ID", func(t *testing.T) {
gPrm.StorageID = objects[0].storageID
res, err := s.GetRange(gPrm)
require.NoError(t, err)
require.Equal(t, payload[start:stop], res.Data)
})
t.Run("offset > len(payload)", func(t *testing.T) {
gPrm.Range.SetOffset(uint64(len(payload) + 10))
gPrm.Range.SetLength(10)
_, err := s.GetRange(gPrm)
require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
})
t.Run("offset + length > len(payload)", func(t *testing.T) {
gPrm.Range.SetOffset(10)
gPrm.Range.SetLength(uint64(len(payload)))
_, err := s.GetRange(gPrm)
require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
})
t.Run("length is negative when converted to int64", func(t *testing.T) {
gPrm.Range.SetOffset(0)
gPrm.Range.SetLength(1 << 63)
_, err := s.GetRange(gPrm)
require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
})
t.Run("offset + length overflow uint64", func(t *testing.T) {
gPrm.Range.SetOffset(10)
gPrm.Range.SetLength(math.MaxUint64 - 2)
_, err := s.GetRange(gPrm)
require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
})
}

111
pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go Normal file
View file

@ -0,0 +1,111 @@
package blobstortest
import (
"errors"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
func TestIterate(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Init())
t.Cleanup(func() { require.NoError(t, s.Close()) })
objects := prepare(t, 10, s, min, max)
// Delete random object to ensure it is not iterated over.
const delID = 2
var delPrm common.DeletePrm
delPrm.Address = objects[2].addr
delPrm.StorageID = objects[2].storageID
_, err := s.Delete(delPrm)
require.NoError(t, err)
objects = append(objects[:delID], objects[delID+1:]...)
t.Run("normal handler", func(t *testing.T) {
seen := make(map[string]objectDesc)
var iterPrm common.IteratePrm
iterPrm.Handler = func(elem common.IterationElement) error {
seen[elem.Address.String()] = objectDesc{
addr: elem.Address,
raw: elem.ObjectData,
storageID: elem.StorageID,
}
return nil
}
_, err := s.Iterate(iterPrm)
require.NoError(t, err)
require.Equal(t, len(objects), len(seen))
for i := range objects {
d, ok := seen[objects[i].addr.String()]
require.True(t, ok)
require.Equal(t, objects[i].raw, d.raw)
require.Equal(t, objects[i].addr, d.addr)
require.Equal(t, objects[i].storageID, d.storageID)
}
})
t.Run("lazy handler", func(t *testing.T) {
seen := make(map[string]func() ([]byte, error))
var iterPrm common.IteratePrm
iterPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error {
seen[addr.String()] = f
return nil
}
_, err := s.Iterate(iterPrm)
require.NoError(t, err)
require.Equal(t, len(objects), len(seen))
for i := range objects {
f, ok := seen[objects[i].addr.String()]
require.True(t, ok)
data, err := f()
require.NoError(t, err)
require.Equal(t, objects[i].raw, data)
}
})
t.Run("ignore errors doesn't work for logical errors", func(t *testing.T) {
seen := make(map[string]objectDesc)
var n int
var logicErr = errors.New("logic error")
var iterPrm common.IteratePrm
iterPrm.IgnoreErrors = true
iterPrm.Handler = func(elem common.IterationElement) error {
seen[elem.Address.String()] = objectDesc{
addr: elem.Address,
raw: elem.ObjectData,
storageID: elem.StorageID,
}
n++
if n == len(objects)/2 {
return logicErr
}
return nil
}
_, err := s.Iterate(iterPrm)
require.Equal(t, err, logicErr)
require.Equal(t, len(objects)/2, len(seen))
for i := range objects {
d, ok := seen[objects[i].addr.String()]
if ok {
n--
require.Equal(t, objects[i].raw, d.raw)
require.Equal(t, objects[i].addr, d.addr)
require.Equal(t, objects[i].storageID, d.storageID)
}
}
require.Equal(t, 0, n)
})
}