From e020fe55979bb8d1a56db2af3b91ea7af579aac2 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 14 Apr 2021 11:50:21 +0300 Subject: [PATCH] [#477] engine: Add test for missing link issue There were no unit tests of storage engine. This commit adds first test to reproduce missing link ID in split info at `engine.Head(raw)` request. Engine tests uses some constructors from metabase tests, so it is better to locate such functions in common package at local_object_storage. Signed-off-by: Alex Vanin --- .../engine/engine_test.go | 121 ++++++++++++++++++ pkg/local_object_storage/engine/head_test.go | 67 ++++++++++ 2 files changed, 188 insertions(+) create mode 100644 pkg/local_object_storage/engine/engine_test.go create mode 100644 pkg/local_object_storage/engine/head_test.go diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go new file mode 100644 index 000000000..e1cfd5400 --- /dev/null +++ b/pkg/local_object_storage/engine/engine_test.go @@ -0,0 +1,121 @@ +package engine + +import ( + "crypto/sha256" + "fmt" + "math/rand" + "sync" + "testing" + + "github.com/nspcc-dev/neofs-api-go/pkg" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/nspcc-dev/neofs-node/pkg/util/test" + "github.com/nspcc-dev/tzhash/tz" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func testNewEngineWithShards(shards ...*shard.Shard) *StorageEngine { + engine := &StorageEngine{ + cfg: &cfg{ + log: zap.L(), + }, + mtx: new(sync.RWMutex), + shards: make(map[string]*shard.Shard, len(shards)), + } + + for _, s := range shards { + engine.shards[s.ID().String()] = s + } + + return engine +} + +func testNewShard(t *testing.T, id int) *shard.Shard { + sid, err := generateShardID() + require.NoError(t, err) + + s := shard.New( + shard.WithID(sid), + shard.WithLogger(zap.L()), + shard.WithBlobStorOptions( + blobstor.WithRootPath(fmt.Sprintf("%s.%d.blobstor", t.Name(), id)), + blobstor.WithBlobovniczaShallowWidth(2), + blobstor.WithBlobovniczaShallowDepth(2), + blobstor.WithRootPerm(0700), + ), + shard.WithMetaBaseOptions( + meta.WithPath(fmt.Sprintf("%s.%d.metabase", t.Name(), id)), + meta.WithPermissions(0700), + )) + + require.NoError(t, s.Open()) + require.NoError(t, s.Init()) + + return s +} + +func testCID() *container.ID { + cs := [sha256.Size]byte{} + _, _ = rand.Read(cs[:]) + + id := container.NewID() + id.SetSHA256(cs) + + return id +} + +func testOID() *objectSDK.ID { + cs := [sha256.Size]byte{} + _, _ = rand.Read(cs[:]) + + id := objectSDK.NewID() + id.SetSHA256(cs) + + return id +} + +func generateRawObjectWithCID(t *testing.T, cid *container.ID) *object.RawObject { + version := pkg.NewVersion() + version.SetMajor(2) + version.SetMinor(1) + + w, err := owner.NEO3WalletFromPublicKey(&test.DecodeKey(-1).PublicKey) + require.NoError(t, err) + + ownerID := owner.NewID() + ownerID.SetNeo3Wallet(w) + + csum := new(pkg.Checksum) + csum.SetSHA256(sha256.Sum256(w.Bytes())) + + csumTZ := new(pkg.Checksum) + csumTZ.SetTillichZemor(tz.Sum(csum.Sum())) + + obj := object.NewRaw() + obj.SetID(testOID()) + obj.SetOwnerID(ownerID) + obj.SetContainerID(cid) + obj.SetVersion(version) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + obj.SetPayload([]byte{1, 2, 3, 4, 5}) + + return obj +} + +func addAttribute(obj *object.RawObject, key, val string) { + attr := objectSDK.NewAttribute() + attr.SetKey(key) + attr.SetValue(val) + + attrs := obj.Attributes() + attrs = append(attrs, attr) + obj.SetAttributes(attrs...) +} diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go new file mode 100644 index 000000000..f90f96be5 --- /dev/null +++ b/pkg/local_object_storage/engine/head_test.go @@ -0,0 +1,67 @@ +package engine + +import ( + "os" + "testing" + + objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/stretchr/testify/require" +) + +func TestHeadRaw(t *testing.T) { + defer os.RemoveAll(t.Name()) + + cid := testCID() + splitID := objectSDK.NewSplitID() + + parent := generateRawObjectWithCID(t, cid) + addAttribute(parent, "foo", "bar") + + parentAddr := objectSDK.NewAddress() + parentAddr.SetContainerID(cid) + parentAddr.SetObjectID(parent.ID()) + + child := generateRawObjectWithCID(t, cid) + child.SetParent(parent.Object().SDK()) + child.SetParentID(parent.ID()) + child.SetSplitID(splitID) + + link := generateRawObjectWithCID(t, cid) + link.SetParent(parent.Object().SDK()) + link.SetParentID(parent.ID()) + link.SetChildren(child.ID()) + link.SetSplitID(splitID) + + t.Run("virtual object split in different shards", func(t *testing.T) { + s1 := testNewShard(t, 1) + s2 := testNewShard(t, 2) + + e := testNewEngineWithShards(s1, s2) + defer e.Close() + + putPrmLeft := new(shard.PutPrm).WithObject(child.Object()) + putPrmLink := new(shard.PutPrm).WithObject(link.Object()) + + // put most left object in one shard + _, err := s1.Put(putPrmLeft) + require.NoError(t, err) + + // put link object in another shard + _, err = s2.Put(putPrmLink) + require.NoError(t, err) + + // head with raw flag should return SplitInfoError + headPrm := new(HeadPrm).WithAddress(parentAddr).WithRaw(true) + _, err = e.Head(headPrm) + require.Error(t, err) + + si, ok := err.(*objectSDK.SplitInfoError) + require.True(t, ok) + + // SplitInfoError should contain info from both shards + require.Equal(t, splitID, si.SplitInfo().SplitID()) + require.Equal(t, child.ID(), si.SplitInfo().LastPart()) + require.Equal(t, link.ID(), si.SplitInfo().Link()) + }) +}