[#1686] blobstor: Add generic tests

This tests check that each blobstor component behaves similarly when
same methods are being used. It is intended to serve as a specification
for all future components.

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>

pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go

@@ -0,0 +1,32 @@
+package blobovniczatree
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestGeneric(t *testing.T) {
+	const maxObjectSize = 1 << 16
+
+	defer func() { _ = os.RemoveAll(t.Name()) }()
+
+	var n int
+	newTree := func(t *testing.T) common.Storage {
+		dir := filepath.Join(t.Name(), strconv.Itoa(n))
+		return NewBlobovniczaTree(
+			WithLogger(zaptest.NewLogger(t)),
+			WithObjectSizeLimit(maxObjectSize),
+			WithBlobovniczaShallowWidth(2),
+			WithBlobovniczaShallowDepth(2),
+			WithRootPath(dir),
+			WithBlobovniczaSize(1<<20))
+	}
+
+	blobstortest.TestAll(t, newTree, 1024, maxObjectSize)
+}

pkg/local_object_storage/blobstor/blobovniczatree/get.go

@@ -143,5 +143,5 @@ func (b *Blobovniczas) getObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.G
 		return common.GetRes{}, fmt.Errorf("could not unmarshal the object: %w", err)
 	}
 
-	return common.GetRes{Object: obj}, nil
+	return common.GetRes{Object: obj, RawData: data}, nil
 }

pkg/local_object_storage/blobstor/blobovniczatree/iterate.go

@@ -26,10 +26,15 @@ func (b *Blobovniczas) Iterate(prm common.IteratePrm) (common.IterateRes, error)
 				return fmt.Errorf("could not decompress object data: %w", err)
 			}
 
-			return prm.Handler(common.IterationElement{
+			if prm.Handler != nil {
-				Address:    elem.Address(),
+				return prm.Handler(common.IterationElement{
-				ObjectData: data,
+					Address:    elem.Address(),
-				StorageID:  []byte(p),
+					ObjectData: data,
+					StorageID:  []byte(p),
+				})
+			}
+			return prm.LazyHandler(elem.Address(), func() ([]byte, error) {
+				return data, err
 			})
 		})
 		subPrm.DecodeAddresses()

pkg/local_object_storage/blobstor/fstree/fstree.go

@@ -222,7 +222,7 @@ func (t *FSTree) Put(prm common.PutPrm) (common.PutRes, error) {
 	if !prm.DontCompress {
 		prm.RawData = t.Compress(prm.RawData)
 	}
-	return common.PutRes{}, os.WriteFile(p, prm.RawData, t.Permissions)
+	return common.PutRes{StorageID: []byte{}}, os.WriteFile(p, prm.RawData, t.Permissions)
 }
 
 // PutStream puts executes handler on a file opened for write.

pkg/local_object_storage/blobstor/fstree/generic_test.go

@@ -0,0 +1,26 @@
+package fstree
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
+)
+
+func TestGeneric(t *testing.T) {
+	defer func() { _ = os.RemoveAll(t.Name()) }()
+
+	var n int
+	newTree := func(t *testing.T) common.Storage {
+		dir := filepath.Join(t.Name(), strconv.Itoa(n))
+		return New(
+			WithPath(dir),
+			WithDepth(2),
+			WithDirNameLen(2))
+	}
+
+	blobstortest.TestAll(t, newTree, 2048, 16*1024)
+}

pkg/local_object_storage/blobstor/internal/blobstortest/common.go

@@ -0,0 +1,91 @@
+package blobstortest
+
+import (
+	"math/rand"
+	"testing"
+
+	objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
+	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+	oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+)
+
+// Constructor constructs blobstor component.
+// Each call must create a component using different file-system path.
+type Constructor = func(t *testing.T) common.Storage
+
+// objectDesc is a helper structure to avoid multiple `Marshal` invokes during tests.
+type objectDesc struct {
+	obj       *objectSDK.Object
+	addr      oid.Address
+	raw       []byte
+	storageID []byte
+}
+
+func TestAll(t *testing.T, cons Constructor, min, max uint64) {
+	t.Run("get", func(t *testing.T) {
+		TestGet(t, cons, min, max)
+	})
+	t.Run("get range", func(t *testing.T) {
+		TestGetRange(t, cons, min, max)
+	})
+	t.Run("delete", func(t *testing.T) {
+		TestDelete(t, cons, min, max)
+	})
+	t.Run("exists", func(t *testing.T) {
+		TestExists(t, cons, min, max)
+	})
+	t.Run("iterate", func(t *testing.T) {
+		TestIterate(t, cons, min, max)
+	})
+}
+
+func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objectDesc {
+	objects := make([]objectDesc, count)
+
+	for i := range objects {
+		objects[i].obj = NewObject(min + uint64(rand.Intn(int(max-min+1)))) // not too large
+		objects[i].addr = objectCore.AddressOf(objects[i].obj)
+
+		raw, err := objects[i].obj.Marshal()
+		require.NoError(t, err)
+		objects[i].raw = raw
+	}
+
+	for i := range objects {
+		var prm common.PutPrm
+		prm.Address = objects[i].addr
+		prm.Object = objects[i].obj
+		prm.RawData = objects[i].raw
+
+		putRes, err := s.Put(prm)
+		require.NoError(t, err)
+
+		objects[i].storageID = putRes.StorageID
+	}
+
+	return objects
+}
+
+// NewObject creates a regular object of specified size with a random payload.
+func NewObject(sz uint64) *objectSDK.Object {
+	raw := objectSDK.New()
+
+	raw.SetID(oidtest.ID())
+	raw.SetContainerID(cidtest.ID())
+
+	payload := make([]byte, sz)
+	rand.Read(payload)
+	raw.SetPayload(payload)
+
+	// fit the binary size to the required
+	data, _ := raw.Marshal()
+	if ln := uint64(len(data)); ln > sz {
+		raw.SetPayload(raw.Payload()[:sz-(ln-sz)])
+	}
+
+	return raw
+}

pkg/local_object_storage/blobstor/internal/blobstortest/delete.go

@@ -0,0 +1,82 @@
+package blobstortest
+
+import (
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
+	oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDelete(t *testing.T, cons Constructor, min, max uint64) {
+	s := cons(t)
+	require.NoError(t, s.Open(false))
+	require.NoError(t, s.Init())
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	objects := prepare(t, 4, s, min, max)
+
+	t.Run("delete non-existent", func(t *testing.T) {
+		var prm common.DeletePrm
+		prm.Address = oidtest.Address()
+
+		_, err := s.Delete(prm)
+		require.Error(t, err, new(apistatus.ObjectNotFound))
+	})
+
+	t.Run("with storage ID", func(t *testing.T) {
+		var prm common.DeletePrm
+		prm.Address = objects[0].addr
+		prm.StorageID = objects[0].storageID
+
+		_, err := s.Delete(prm)
+		require.NoError(t, err)
+
+		t.Run("exists fail", func(t *testing.T) {
+			prm := common.ExistsPrm{Address: oidtest.Address()}
+			res, err := s.Exists(prm)
+			require.NoError(t, err)
+			require.False(t, res.Exists)
+		})
+		t.Run("get fail", func(t *testing.T) {
+			prm := common.GetPrm{Address: oidtest.Address()}
+			_, err := s.Get(prm)
+			require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
+		})
+		t.Run("getrange fail", func(t *testing.T) {
+			prm := common.GetRangePrm{Address: oidtest.Address()}
+			_, err := s.GetRange(prm)
+			require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
+		})
+	})
+	t.Run("without storage ID", func(t *testing.T) {
+		var prm common.DeletePrm
+		prm.Address = objects[1].addr
+
+		_, err := s.Delete(prm)
+		require.NoError(t, err)
+	})
+
+	t.Run("delete twice", func(t *testing.T) {
+		var prm common.DeletePrm
+		prm.Address = objects[2].addr
+		prm.StorageID = objects[2].storageID
+
+		_, err := s.Delete(prm)
+		require.NoError(t, err)
+
+		_, err = s.Delete(prm)
+		require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
+	})
+
+	t.Run("non-deleted object is still available", func(t *testing.T) {
+		var prm common.GetPrm
+		prm.Address = objects[3].addr
+		prm.Raw = true
+
+		res, err := s.Get(prm)
+		require.NoError(t, err)
+		require.Equal(t, objects[3].raw, res.RawData)
+	})
+}

pkg/local_object_storage/blobstor/internal/blobstortest/exists.go

@@ -0,0 +1,31 @@
+package blobstortest
+
+import (
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExists(t *testing.T, cons Constructor, min, max uint64) {
+	s := cons(t)
+	require.NoError(t, s.Open(false))
+	require.NoError(t, s.Init())
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	objects := prepare(t, 1, s, min, max)
+
+	t.Run("missing object", func(t *testing.T) {
+		prm := common.ExistsPrm{Address: oidtest.Address()}
+		res, err := s.Exists(prm)
+		require.NoError(t, err)
+		require.False(t, res.Exists)
+	})
+
+	var prm common.ExistsPrm
+	prm.Address = objects[0].addr
+	res, err := s.Exists(prm)
+	require.NoError(t, err)
+	require.True(t, res.Exists)
+}

pkg/local_object_storage/blobstor/internal/blobstortest/get.go

@@ -0,0 +1,50 @@
+package blobstortest
+
+import (
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
+	oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGet(t *testing.T, cons Constructor, min, max uint64) {
+	s := cons(t)
+	require.NoError(t, s.Open(false))
+	require.NoError(t, s.Init())
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	objects := prepare(t, 2, s, min, max)
+
+	t.Run("missing object", func(t *testing.T) {
+		gPrm := common.GetPrm{Address: oidtest.Address()}
+		_, err := s.Get(gPrm)
+		require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
+	})
+
+	for i := range objects {
+		var gPrm common.GetPrm
+		gPrm.Address = objects[i].addr
+
+		// With storage ID.
+		gPrm.StorageID = objects[i].storageID
+		res, err := s.Get(gPrm)
+		require.NoError(t, err)
+		require.Equal(t, objects[i].obj, res.Object)
+
+		// Without storage ID.
+		gPrm.StorageID = nil
+		res, err = s.Get(gPrm)
+		require.NoError(t, err)
+		require.Equal(t, objects[i].obj, res.Object)
+
+		// With raw flag.
+		gPrm.StorageID = objects[i].storageID
+		gPrm.Raw = true
+
+		res, err = s.Get(gPrm)
+		require.NoError(t, err)
+		require.Equal(t, objects[i].raw, res.RawData)
+	}
+}

pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go

@@ -0,0 +1,84 @@
+package blobstortest
+
+import (
+	"math"
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
+	oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetRange(t *testing.T, cons Constructor, min, max uint64) {
+	s := cons(t)
+	require.NoError(t, s.Open(false))
+	require.NoError(t, s.Init())
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	objects := prepare(t, 1, s, min, max)
+
+	t.Run("missing object", func(t *testing.T) {
+		gPrm := common.GetRangePrm{Address: oidtest.Address()}
+		_, err := s.GetRange(gPrm)
+		require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
+	})
+
+	payload := objects[0].obj.Payload()
+
+	var start, stop uint64 = 11, 100
+	if uint64(len(payload)) < stop {
+		panic("unexpected: invalid test object generated")
+	}
+
+	var gPrm common.GetRangePrm
+	gPrm.Address = objects[0].addr
+	gPrm.Range.SetOffset(start)
+	gPrm.Range.SetLength(stop - start)
+
+	t.Run("without storage ID", func(t *testing.T) {
+		// Without storage ID.
+		res, err := s.GetRange(gPrm)
+		require.NoError(t, err)
+		require.Equal(t, payload[start:stop], res.Data)
+	})
+
+	t.Run("with storage ID", func(t *testing.T) {
+		gPrm.StorageID = objects[0].storageID
+		res, err := s.GetRange(gPrm)
+		require.NoError(t, err)
+		require.Equal(t, payload[start:stop], res.Data)
+	})
+
+	t.Run("offset > len(payload)", func(t *testing.T) {
+		gPrm.Range.SetOffset(uint64(len(payload) + 10))
+		gPrm.Range.SetLength(10)
+
+		_, err := s.GetRange(gPrm)
+		require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
+	})
+
+	t.Run("offset + length > len(payload)", func(t *testing.T) {
+		gPrm.Range.SetOffset(10)
+		gPrm.Range.SetLength(uint64(len(payload)))
+
+		_, err := s.GetRange(gPrm)
+		require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
+	})
+
+	t.Run("length is negative when converted to int64", func(t *testing.T) {
+		gPrm.Range.SetOffset(0)
+		gPrm.Range.SetLength(1 << 63)
+
+		_, err := s.GetRange(gPrm)
+		require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
+	})
+
+	t.Run("offset + length overflow uint64", func(t *testing.T) {
+		gPrm.Range.SetOffset(10)
+		gPrm.Range.SetLength(math.MaxUint64 - 2)
+
+		_, err := s.GetRange(gPrm)
+		require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange))
+	})
+}

pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go

@@ -0,0 +1,111 @@
+package blobstortest
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIterate(t *testing.T, cons Constructor, min, max uint64) {
+	s := cons(t)
+	require.NoError(t, s.Open(false))
+	require.NoError(t, s.Init())
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	objects := prepare(t, 10, s, min, max)
+
+	// Delete random object to ensure it is not iterated over.
+	const delID = 2
+	var delPrm common.DeletePrm
+	delPrm.Address = objects[2].addr
+	delPrm.StorageID = objects[2].storageID
+	_, err := s.Delete(delPrm)
+	require.NoError(t, err)
+
+	objects = append(objects[:delID], objects[delID+1:]...)
+
+	t.Run("normal handler", func(t *testing.T) {
+		seen := make(map[string]objectDesc)
+
+		var iterPrm common.IteratePrm
+		iterPrm.Handler = func(elem common.IterationElement) error {
+			seen[elem.Address.String()] = objectDesc{
+				addr:      elem.Address,
+				raw:       elem.ObjectData,
+				storageID: elem.StorageID,
+			}
+			return nil
+		}
+
+		_, err := s.Iterate(iterPrm)
+		require.NoError(t, err)
+		require.Equal(t, len(objects), len(seen))
+		for i := range objects {
+			d, ok := seen[objects[i].addr.String()]
+			require.True(t, ok)
+			require.Equal(t, objects[i].raw, d.raw)
+			require.Equal(t, objects[i].addr, d.addr)
+			require.Equal(t, objects[i].storageID, d.storageID)
+		}
+	})
+
+	t.Run("lazy handler", func(t *testing.T) {
+		seen := make(map[string]func() ([]byte, error))
+
+		var iterPrm common.IteratePrm
+		iterPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error {
+			seen[addr.String()] = f
+			return nil
+		}
+
+		_, err := s.Iterate(iterPrm)
+		require.NoError(t, err)
+		require.Equal(t, len(objects), len(seen))
+		for i := range objects {
+			f, ok := seen[objects[i].addr.String()]
+			require.True(t, ok)
+
+			data, err := f()
+			require.NoError(t, err)
+			require.Equal(t, objects[i].raw, data)
+		}
+	})
+
+	t.Run("ignore errors doesn't work for logical errors", func(t *testing.T) {
+		seen := make(map[string]objectDesc)
+
+		var n int
+		var logicErr = errors.New("logic error")
+		var iterPrm common.IteratePrm
+		iterPrm.IgnoreErrors = true
+		iterPrm.Handler = func(elem common.IterationElement) error {
+			seen[elem.Address.String()] = objectDesc{
+				addr:      elem.Address,
+				raw:       elem.ObjectData,
+				storageID: elem.StorageID,
+			}
+			n++
+			if n == len(objects)/2 {
+				return logicErr
+			}
+			return nil
+		}
+
+		_, err := s.Iterate(iterPrm)
+		require.Equal(t, err, logicErr)
+		require.Equal(t, len(objects)/2, len(seen))
+		for i := range objects {
+			d, ok := seen[objects[i].addr.String()]
+			if ok {
+				n--
+				require.Equal(t, objects[i].raw, d.raw)
+				require.Equal(t, objects[i].addr, d.addr)
+				require.Equal(t, objects[i].storageID, d.storageID)
+			}
+		}
+		require.Equal(t, 0, n)
+	})
+}