From 6c4a1699ef57ef0bbf46b4ab1d083e7586a86631 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 17 Mar 2023 10:56:37 +0300 Subject: [PATCH] [#145] shard-gc: Expired locked unit test Added unit test that verifies that GC deletes expired locked objects in one epoch. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc_test.go | 121 +++++++++++++++++++ pkg/local_object_storage/shard/shard_test.go | 6 +- 2 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 pkg/local_object_storage/shard/gc_test.go diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go new file mode 100644 index 00000000..8643605e --- /dev/null +++ b/pkg/local_object_storage/shard/gc_test.go @@ -0,0 +1,121 @@ +package shard_test + +import ( + "context" + "path/filepath" + "testing" + "time" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func Test_GCDropsLockedExpiredObject(t *testing.T) { + var sh *shard.Shard + + epoch := &epochState{ + Value: 100, + } + + rootPath := t.TempDir() + opts := []shard.Option{ + shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), + shard.WithBlobStorOptions( + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(2), + blobovniczatree.WithBlobovniczaShallowWidth(2)), + Policy: func(_ *object.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + ), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epoch), + ), + shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + shard.WithExpiredLocksCallback(func(_ context.Context, a []oid.Address) { + sh.HandleExpiredLocks(a) + }), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + + return pool + }), + } + + sh = shard.New(opts...) + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init()) + + t.Cleanup(func() { + releaseShard(sh, t) + }) + + cnr := cidtest.ID() + + var objExpirationAttr objectSDK.Attribute + objExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + objExpirationAttr.SetValue("101") + + obj := generateObjectWithCID(t, cnr) + obj.SetAttributes(objExpirationAttr) + objID, _ := obj.ID() + + var lockExpirationAttr objectSDK.Attribute + lockExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + lockExpirationAttr.SetValue("103") + + lock := generateObjectWithCID(t, cnr) + lock.SetType(object.TypeLock) + lock.SetAttributes(lockExpirationAttr) + lockID, _ := lock.ID() + + var putPrm shard.PutPrm + putPrm.SetObject(obj) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + err = sh.Lock(cnr, lockID, []oid.ID{objID}) + require.NoError(t, err) + + putPrm.SetObject(lock) + _, err = sh.Put(putPrm) + require.NoError(t, err) + + epoch.Value = 105 + sh.NotificationChannel() <- shard.EventNewEpoch(epoch.Value) + + var getPrm shard.GetPrm + getPrm.SetAddress(objectCore.AddressOf(obj)) + require.Eventually(t, func() bool { + _, err = sh.Get(getPrm) + return shard.IsErrNotFound(err) + }, 3*time.Second, 1*time.Second, "expired object must be deleted") +} diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index a6da5391..2a98dabb 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -27,10 +27,12 @@ import ( "go.uber.org/zap/zaptest" ) -type epochState struct{} +type epochState struct { + Value uint64 +} func (s epochState) CurrentEpoch() uint64 { - return 0 + return s.Value } func newShard(t testing.TB, enableWriteCache bool) *shard.Shard {