From 6bec50963ba261c968280e0d516b0817c811b39c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 1 Dec 2020 16:58:34 +0300 Subject: [PATCH] [#222] Add tests for shards Signed-off-by: Alex Vanin --- pkg/local_object_storage/shard/delete_test.go | 82 +++++++++++ pkg/local_object_storage/shard/get_test.go | 65 +++++++++ pkg/local_object_storage/shard/head_test.go | 44 ++++++ pkg/local_object_storage/shard/inhume_test.go | 57 ++++++++ pkg/local_object_storage/shard/shard_test.go | 129 ++++++++++++++++++ 5 files changed, 377 insertions(+) create mode 100644 pkg/local_object_storage/shard/delete_test.go create mode 100644 pkg/local_object_storage/shard/get_test.go create mode 100644 pkg/local_object_storage/shard/head_test.go create mode 100644 pkg/local_object_storage/shard/inhume_test.go create mode 100644 pkg/local_object_storage/shard/shard_test.go diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go new file mode 100644 index 0000000000..91a36d4e32 --- /dev/null +++ b/pkg/local_object_storage/shard/delete_test.go @@ -0,0 +1,82 @@ +package shard_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/stretchr/testify/require" +) + +func TestShard_Delete(t *testing.T) { + sh := newShard(t, false) + shWC := newShard(t, true) + + defer func() { + releaseShard(sh, t) + releaseShard(shWC, t) + }() + + t.Run("without write cache", func(t *testing.T) { + testShardDelete(t, sh) + }) + + t.Run("with write cache", func(t *testing.T) { + testShardDelete(t, shWC) + }) +} + +func testShardDelete(t *testing.T, sh *shard.Shard) { + cid := generateCID() + + obj := generateRawObjectWithCID(t, cid) + addAttribute(obj, "foo", "bar") + + putPrm := new(shard.PutPrm) + getPrm := new(shard.GetPrm) + + t.Run("big object", func(t *testing.T) { + addPayload(obj, 1<<20) + + putPrm.WithObject(obj.Object()) + getPrm.WithAddress(obj.Object().Address()) + + delPrm := new(shard.DeletePrm) + delPrm.WithAddress(obj.Object().Address()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + _, err = sh.Get(getPrm) + require.NoError(t, err) + + _, err = sh.Delete(delPrm) + require.NoError(t, err) + + _, err = sh.Get(getPrm) + require.EqualError(t, err, object.ErrNotFound.Error()) + }) + + t.Run("small object", func(t *testing.T) { + obj.SetID(generateOID()) + addPayload(obj, 1<<5) + + putPrm.WithObject(obj.Object()) + getPrm.WithAddress(obj.Object().Address()) + + delPrm := new(shard.DeletePrm) + delPrm.WithAddress(obj.Object().Address()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + _, err = sh.Get(getPrm) + require.NoError(t, err) + + _, err = sh.Delete(delPrm) + require.NoError(t, err) + + _, err = sh.Get(getPrm) + require.EqualError(t, err, object.ErrNotFound.Error()) + }) +} diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go new file mode 100644 index 0000000000..a6839381aa --- /dev/null +++ b/pkg/local_object_storage/shard/get_test.go @@ -0,0 +1,65 @@ +package shard_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/stretchr/testify/require" +) + +func TestShard_Get(t *testing.T) { + sh := newShard(t, false) + shWC := newShard(t, true) + + defer func() { + releaseShard(sh, t) + releaseShard(shWC, t) + }() + + t.Run("without write cache", func(t *testing.T) { + testShardGet(t, sh) + }) + + t.Run("with write cache", func(t *testing.T) { + testShardGet(t, shWC) + }) +} + +func testShardGet(t *testing.T, sh *shard.Shard) { + obj := generateRawObject(t) + addAttribute(obj, "foo", "bar") + + putPrm := new(shard.PutPrm) + getPrm := new(shard.GetPrm) + + t.Run("small object", func(t *testing.T) { + addPayload(obj, 1<<5) + + putPrm.WithObject(obj.Object()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + getPrm.WithAddress(obj.Object().Address()) + + res, err := sh.Get(getPrm) + require.NoError(t, err) + require.Equal(t, obj.Object(), res.Object()) + }) + + t.Run("big object", func(t *testing.T) { + obj.SetID(generateOID()) + addPayload(obj, 1<<20) // big obj + + putPrm.WithObject(obj.Object()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + getPrm.WithAddress(obj.Object().Address()) + + res, err := sh.Get(getPrm) + require.NoError(t, err) + require.Equal(t, obj.Object(), res.Object()) + }) +} diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go new file mode 100644 index 0000000000..7ec0108f02 --- /dev/null +++ b/pkg/local_object_storage/shard/head_test.go @@ -0,0 +1,44 @@ +package shard_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/stretchr/testify/require" +) + +func TestShard_Head(t *testing.T) { + sh := newShard(t, false) + shWC := newShard(t, true) + + defer func() { + releaseShard(sh, t) + releaseShard(shWC, t) + }() + + t.Run("without write cache", func(t *testing.T) { + testShardHead(t, sh) + }) + + t.Run("with write cache", func(t *testing.T) { + testShardHead(t, shWC) + }) +} + +func testShardHead(t *testing.T, sh *shard.Shard) { + obj := generateRawObject(t) + addAttribute(obj, "foo", "bar") + + putPrm := new(shard.PutPrm) + putPrm.WithObject(obj.Object()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + headPrm := new(shard.HeadPrm) + headPrm.WithAddress(obj.Object().Address()) + + res, err := sh.Head(headPrm) + require.NoError(t, err) + require.Equal(t, obj.Object(), res.Object()) +} diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go new file mode 100644 index 0000000000..1c3b3c4a49 --- /dev/null +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -0,0 +1,57 @@ +package shard_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/stretchr/testify/require" +) + +func TestShard_Inhume(t *testing.T) { + sh := newShard(t, false) + shWC := newShard(t, true) + + defer func() { + releaseShard(sh, t) + releaseShard(shWC, t) + }() + + t.Run("without write cache", func(t *testing.T) { + testShardInhume(t, sh) + }) + + t.Run("with write cache", func(t *testing.T) { + testShardInhume(t, shWC) + }) +} + +func testShardInhume(t *testing.T, sh *shard.Shard) { + cid := generateCID() + + obj := generateRawObjectWithCID(t, cid) + addAttribute(obj, "foo", "bar") + + ts := generateRawObjectWithCID(t, cid) + + putPrm := new(shard.PutPrm) + putPrm.WithObject(obj.Object()) + + inhPrm := new(shard.InhumePrm) + inhPrm.WithTarget(obj.Object().Address(), ts.Object().Address()) + + getPrm := new(shard.GetPrm) + getPrm.WithAddress(obj.Object().Address()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + _, err = sh.Get(getPrm) + require.NoError(t, err) + + _, err = sh.Inhume(inhPrm) + require.NoError(t, err) + + _, err = sh.Get(getPrm) + require.EqualError(t, err, object.ErrAlreadyRemoved.Error()) +} diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go new file mode 100644 index 0000000000..1e08c659e8 --- /dev/null +++ b/pkg/local_object_storage/shard/shard_test.go @@ -0,0 +1,129 @@ +package shard_test + +import ( + "crypto/rand" + "crypto/sha256" + "os" + "path" + "testing" + + "github.com/nspcc-dev/neofs-api-go/pkg" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" + meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase/v2" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/nspcc-dev/neofs-node/pkg/util/test" + "github.com/nspcc-dev/tzhash/tz" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { + rootPath := t.Name() + if enableWriteCache { + rootPath = path.Join(rootPath, "wc") + } else { + rootPath = path.Join(rootPath, "nowc") + } + + opts := []shard.Option{ + shard.WithLogger(zap.L()), + shard.WithBlobStorOptions( + blobstor.WithRootPath(path.Join(rootPath, "blob")), + blobstor.WithBlobovniczaShallowWidth(2), + blobstor.WithBlobovniczaShallowDepth(2), + ), + shard.WithMetaBaseOptions( + meta.WithPath(path.Join(rootPath, "meta")), + ), + shard.WithWriteCache(enableWriteCache), + shard.WithWriteCacheOptions( + blobstor.WithRootPath(path.Join(rootPath, "wcache")), + ), + } + + sh := shard.New(opts...) + + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init()) + + return sh +} + +func releaseShard(s *shard.Shard, t testing.TB) { + s.Close() + os.RemoveAll(t.Name()) +} + +func generateRawObject(t *testing.T) *object.RawObject { + return generateRawObjectWithCID(t, generateCID()) +} + +func generateRawObjectWithCID(t *testing.T, cid *container.ID) *object.RawObject { + version := pkg.NewVersion() + version.SetMajor(2) + version.SetMinor(1) + + w, err := owner.NEO3WalletFromPublicKey(&test.DecodeKey(-1).PublicKey) + require.NoError(t, err) + + ownerID := owner.NewID() + ownerID.SetNeo3Wallet(w) + + csum := new(pkg.Checksum) + csum.SetSHA256(sha256.Sum256(w.Bytes())) + + csumTZ := new(pkg.Checksum) + csumTZ.SetTillichZemor(tz.Sum(csum.Sum())) + + obj := object.NewRaw() + obj.SetID(generateOID()) + obj.SetOwnerID(ownerID) + obj.SetContainerID(cid) + obj.SetVersion(version) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + + return obj +} + +func addAttribute(obj *object.RawObject, key, val string) { + attr := objectSDK.NewAttribute() + attr.SetKey(key) + attr.SetValue(val) + + attrs := obj.Attributes() + attrs = append(attrs, attr) + obj.SetAttributes(attrs...) +} + +func addPayload(obj *object.RawObject, size int) { + buf := make([]byte, size) + _, _ = rand.Read(buf) + + obj.SetPayload(buf) + obj.SetPayloadSize(uint64(size)) +} + +func generateCID() *container.ID { + cs := [sha256.Size]byte{} + _, _ = rand.Read(cs[:]) + + id := container.NewID() + id.SetSHA256(cs) + + return id +} + +func generateOID() *objectSDK.ID { + cs := [sha256.Size]byte{} + _, _ = rand.Read(cs[:]) + + id := objectSDK.NewID() + id.SetSHA256(cs) + + return id +}