diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 939241168..e6ffcd4ee 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -463,7 +463,7 @@ func (e engineWithoutNotifications) IsLocked(ctx context.Context, address oid.Ad return e.engine.IsLocked(ctx, address) } -func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { +func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID, expEpoch uint64) error { var prm engine.InhumePrm addrs := make([]oid.Address, len(toDelete)) @@ -472,7 +472,7 @@ func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Ad addrs[i].SetObject(toDelete[i]) } - prm.WithTarget(tombstone, addrs...) + prm.WithTarget(tombstone, expEpoch, addrs...) return e.engine.Inhume(ctx, prm) } diff --git a/pkg/core/object/object.go b/pkg/core/object/object.go index 9c450966c..fa3735a30 100644 --- a/pkg/core/object/object.go +++ b/pkg/core/object/object.go @@ -1,6 +1,9 @@ package object import ( + "strconv" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -21,3 +24,14 @@ func AddressOf(obj *objectSDK.Object) oid.Address { return addr } + +// ExpirationEpoch returns the expiration epoch of the object. +func ExpirationEpoch(obj *objectSDK.Object) (epoch uint64, found bool) { + for _, attr := range obj.Attributes() { + if attr.Key() == objectV2.SysAttributeExpEpoch { + epoch, err := strconv.ParseUint(attr.Value(), 10, 64) + return epoch, err == nil + } + } + return +} diff --git a/pkg/core/object/object_test.go b/pkg/core/object/object_test.go new file mode 100644 index 000000000..b990dd4b8 --- /dev/null +++ b/pkg/core/object/object_test.go @@ -0,0 +1,49 @@ +package object_test + +import ( + "strconv" + "testing" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" + "github.com/stretchr/testify/require" +) + +func TestExpirationEpoch(t *testing.T) { + obj := objecttest.Object() + + var expEpoch uint64 = 42 + expAttr := objectSDK.NewAttribute() + expAttr.SetKey(objectV2.SysAttributeExpEpoch) + expAttr.SetValue(strconv.FormatUint(expEpoch, 10)) + + t.Run("no attributes set", func(t *testing.T) { + obj.SetAttributes() + _, found := objectCore.ExpirationEpoch(obj) + require.False(t, found) + }) + t.Run("no expiration epoch attribute", func(t *testing.T) { + obj.SetAttributes(*objecttest.Attribute(), *objectSDK.NewAttribute()) + _, found := objectCore.ExpirationEpoch(obj) + require.False(t, found) + }) + t.Run("valid expiration epoch attribute", func(t *testing.T) { + obj.SetAttributes(*objecttest.Attribute(), *expAttr, *objectSDK.NewAttribute()) + epoch, found := objectCore.ExpirationEpoch(obj) + require.True(t, found) + require.Equal(t, expEpoch, epoch) + }) + t.Run("invalid expiration epoch value", func(t *testing.T) { + expAttr.SetValue("-42") + obj.SetAttributes(*objecttest.Attribute(), *expAttr, *objectSDK.NewAttribute()) + _, found := objectCore.ExpirationEpoch(obj) + require.False(t, found) + + expAttr.SetValue("qwerty") + obj.SetAttributes(*objecttest.Attribute(), *expAttr, *objectSDK.NewAttribute()) + _, found = objectCore.ExpirationEpoch(obj) + require.False(t, found) + }) +} diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 75bd15c8b..7565dfc7a 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -25,6 +25,7 @@ type InhumePrm struct { addrs []oid.Address forceRemoval bool + expEpoch uint64 } // WithTarget sets a list of objects that should be inhumed and tombstone address @@ -32,9 +33,10 @@ type InhumePrm struct { // // tombstone should not be nil, addr should not be empty. // Should not be called along with MarkAsGarbage. -func (p *InhumePrm) WithTarget(tombstone oid.Address, addrs ...oid.Address) { +func (p *InhumePrm) WithTarget(tombstone oid.Address, expEpoch uint64, addrs ...oid.Address) { p.addrs = addrs p.tombstone = &tombstone + p.expEpoch = expEpoch } // MarkAsGarbage marks an object to be physically removed from local storage. @@ -89,7 +91,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { for shardID, addrs := range addrsPerShard { if prm.tombstone != nil { - shPrm.SetTarget(*prm.tombstone, addrs...) + shPrm.SetTarget(*prm.tombstone, prm.expEpoch, addrs...) } else { shPrm.MarkAsGarbage(addrs...) } diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 8c5d28b15..cbf0d32ac 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -44,6 +44,8 @@ func TestStorageEngine_Inhume(t *testing.T) { link.SetChildren(idChild) link.SetSplitID(splitID) + const tombstoneExpEpoch = 1008 + t.Run("delete small object", func(t *testing.T) { t.Parallel() e := testNewEngine(t).setShardsNum(t, 1).prepare(t).engine @@ -53,7 +55,7 @@ func TestStorageEngine_Inhume(t *testing.T) { require.NoError(t, err) var inhumePrm InhumePrm - inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) + inhumePrm.WithTarget(tombstoneID, tombstoneExpEpoch, object.AddressOf(parent)) err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) @@ -83,7 +85,7 @@ func TestStorageEngine_Inhume(t *testing.T) { require.NoError(t, err) var inhumePrm InhumePrm - inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) + inhumePrm.WithTarget(tombstoneID, tombstoneExpEpoch, object.AddressOf(parent)) err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) @@ -127,7 +129,9 @@ func TestStorageEngine_ECInhume(t *testing.T) { require.NoError(t, Put(context.Background(), e, tombstoneObject, false)) var inhumePrm InhumePrm - inhumePrm.WithTarget(tombstoneObjectAddress, parentObjectAddress) + + const tombstoneExpEpoch = 1008 + inhumePrm.WithTarget(tombstoneObjectAddress, tombstoneExpEpoch, parentObjectAddress) err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) @@ -143,6 +147,7 @@ func TestInhumeExpiredRegularObject(t *testing.T) { const currEpoch = 42 const objectExpiresAfter = currEpoch - 1 + const tombstoneExpiresAfter = currEpoch + 1000 engine := testNewEngine(t).setShardsNumAdditionalOpts(t, 1, func(_ int) []shard.Option { return []shard.Option{ @@ -172,7 +177,7 @@ func TestInhumeExpiredRegularObject(t *testing.T) { ts.SetContainer(cnr) var prm InhumePrm - prm.WithTarget(ts, object.AddressOf(obj)) + prm.WithTarget(ts, tombstoneExpiresAfter, object.AddressOf(obj)) err := engine.Inhume(context.Background(), prm) require.NoError(t, err) }) @@ -205,6 +210,8 @@ func BenchmarkInhumeMultipart(b *testing.B) { func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { b.StopTimer() + const tombstoneExpiresAfter = 1000 // doesn't matter, just big enough + engine := testNewEngine(b, WithShardPoolSize(uint32(numObjects))). setShardsNum(b, numShards).prepare(b).engine defer func() { require.NoError(b, engine.Close(context.Background())) }() @@ -234,7 +241,7 @@ func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { ts.SetContainer(cnt) prm := InhumePrm{} - prm.WithTarget(ts, addrs...) + prm.WithTarget(ts, tombstoneExpiresAfter, addrs...) b.StartTimer() err := engine.Inhume(context.Background(), prm) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index b8c9d6b1d..c09e67631 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -40,6 +40,7 @@ func TestLockUserScenario(t *testing.T) { // 5. waits for an epoch after the lock expiration one // 6. tries to inhume the object and expects success const lockerExpiresAfter = 13 + const tombstoneExpiresAfter = 1000 cnr := cidtest.ID() tombObj := testutil.GenerateObjectWithCID(cnr) @@ -111,13 +112,15 @@ func TestLockUserScenario(t *testing.T) { // 3. var inhumePrm InhumePrm - inhumePrm.WithTarget(tombAddr, objAddr) + inhumePrm.WithTarget(tombAddr, tombstoneExpiresAfter, objAddr) var objLockedErr *apistatus.ObjectLocked err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) // 4. + a.SetValue(strconv.Itoa(tombstoneExpiresAfter)) + tombObj.SetType(objectSDK.TypeTombstone) tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) @@ -125,7 +128,7 @@ func TestLockUserScenario(t *testing.T) { err = Put(context.Background(), e, tombObj, false) require.NoError(t, err) - inhumePrm.WithTarget(tombForLockAddr, lockerAddr) + inhumePrm.WithTarget(tombForLockAddr, tombstoneExpiresAfter, lockerAddr) err = e.Inhume(context.Background(), inhumePrm) require.ErrorIs(t, err, meta.ErrLockObjectRemoval) @@ -133,7 +136,7 @@ func TestLockUserScenario(t *testing.T) { // 5. e.HandleNewEpoch(context.Background(), lockerExpiresAfter+1) - inhumePrm.WithTarget(tombAddr, objAddr) + inhumePrm.WithTarget(tombAddr, tombstoneExpiresAfter, objAddr) require.Eventually(t, func() bool { err = e.Inhume(context.Background(), inhumePrm) @@ -166,6 +169,7 @@ func TestLockExpiration(t *testing.T) { defer func() { require.NoError(t, e.Close(context.Background())) }() const lockerExpiresAfter = 13 + const tombstoneExpiresAfter = 1000 cnr := cidtest.ID() var err error @@ -197,7 +201,7 @@ func TestLockExpiration(t *testing.T) { var inhumePrm InhumePrm tombAddr := oidtest.Address() tombAddr.SetContainer(cnr) - inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj)) + inhumePrm.WithTarget(tombAddr, tombstoneExpiresAfter, objectcore.AddressOf(obj)) var objLockedErr *apistatus.ObjectLocked err = e.Inhume(context.Background(), inhumePrm) @@ -209,7 +213,7 @@ func TestLockExpiration(t *testing.T) { // 4. tombAddr = oidtest.Address() tombAddr.SetContainer(cnr) - inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj)) + inhumePrm.WithTarget(tombAddr, tombstoneExpiresAfter, objectcore.AddressOf(obj)) require.Eventually(t, func() bool { err = e.Inhume(context.Background(), inhumePrm) @@ -273,7 +277,8 @@ func TestLockForceRemoval(t *testing.T) { err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) - inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) + const tombstoneExpEpoch = 1008 + inhumePrm.WithTarget(oidtest.Address(), tombstoneExpEpoch, objectcore.AddressOf(obj)) err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 950385a29..4a5ac6ef4 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -156,11 +156,12 @@ func TestCounters(t *testing.T) { } var prm meta.InhumePrm + const tombstoneExpEpoch = 1008 for _, o := range inhumedObjs { tombAddr := oidtest.Address() tombAddr.SetContainer(o.Container()) - prm.SetTombstoneAddress(tombAddr) + prm.SetTombstoneAddress(tombAddr, tombstoneExpEpoch) prm.SetAddresses(o) res, err := db.Inhume(context.Background(), prm) @@ -301,11 +302,12 @@ func TestCounters(t *testing.T) { } var prm meta.InhumePrm + const tombstoneExpEpoch = 1008 for _, o := range inhumedObjs { tombAddr := oidtest.Address() tombAddr.SetContainer(o.Container()) - prm.SetTombstoneAddress(tombAddr) + prm.SetTombstoneAddress(tombAddr, tombstoneExpEpoch) prm.SetAddresses(o) _, err := db.Inhume(context.Background(), prm) diff --git a/pkg/local_object_storage/metabase/delete_ec_test.go b/pkg/local_object_storage/metabase/delete_ec_test.go index 884da23ff..dc6bd08c3 100644 --- a/pkg/local_object_storage/metabase/delete_ec_test.go +++ b/pkg/local_object_storage/metabase/delete_ec_test.go @@ -23,10 +23,13 @@ import ( func TestDeleteECObject_WithoutSplit(t *testing.T) { t.Parallel() + const currEpoch = 12 + const tombstoneExpEpoch = currEpoch + 1 + db := New( WithPath(filepath.Join(t.TempDir(), "metabase")), WithPermissions(0o600), - WithEpochState(epochState{uint64(12)}), + WithEpochState(epochState{uint64(currEpoch)}), ) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) @@ -81,7 +84,7 @@ func TestDeleteECObject_WithoutSplit(t *testing.T) { tombAddress.SetContainer(cnr) tombAddress.SetObject(tombstoneID) inhumePrm.SetAddresses(ecParentAddress) - inhumePrm.SetTombstoneAddress(tombAddress) + inhumePrm.SetTombstoneAddress(tombAddress, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) @@ -179,10 +182,13 @@ func TestDeleteECObject_WithSplit(t *testing.T) { func testDeleteECObjectWithSplit(t *testing.T, chunksCount int, withLinking bool) { t.Parallel() + const currEpoch = 12 + const tombstoneExpEpoch = currEpoch + 1 + db := New( WithPath(filepath.Join(t.TempDir(), "metabase")), WithPermissions(0o600), - WithEpochState(epochState{uint64(12)}), + WithEpochState(epochState{currEpoch}), ) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) @@ -288,7 +294,7 @@ func testDeleteECObjectWithSplit(t *testing.T, chunksCount int, withLinking bool tombAddress.SetContainer(cnr) tombAddress.SetObject(tombstoneID) inhumePrm.SetAddresses(inhumeAddresses...) - inhumePrm.SetTombstoneAddress(tombAddress) + inhumePrm.SetTombstoneAddress(tombAddress, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 2f23d424c..0a2e6e1d6 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -91,8 +91,9 @@ func (db *DB) IterateOverGarbage(ctx context.Context, p GarbageIterationPrm) err // TombstonedObject represents descriptor of the // object that has been covered with tombstone. type TombstonedObject struct { - addr oid.Address - tomb oid.Address + addr oid.Address + tomb oid.Address + expEpoch uint64 } // Address returns tombstoned object address. @@ -106,6 +107,11 @@ func (g TombstonedObject) Tombstone() oid.Address { return g.tomb } +// ExpirationEpoch returns an expiration epoch of a tombstoned object. +func (g TombstonedObject) ExpirationEpoch() uint64 { + return g.expEpoch +} + // TombstonedHandler is a TombstonedObject handling function. type TombstonedHandler func(object TombstonedObject) error @@ -249,7 +255,7 @@ func garbageFromKV(k []byte) (res GarbageObject, err error) { func graveFromKV(k, v []byte) (res TombstonedObject, err error) { if err = decodeAddressFromKey(&res.addr, k); err != nil { err = fmt.Errorf("decode tombstone target from key: %w", err) - } else if err = decodeAddressFromKey(&res.tomb, v); err != nil { + } else if err = decodeTombstoneKeyWithExpEpoch(&res.tomb, &res.expEpoch, v); err != nil { err = fmt.Errorf("decode tombstone address from value: %w", err) } diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index ebadecc04..00c3a2a8b 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -144,8 +144,9 @@ func TestDB_IterateDeletedObjects(t *testing.T) { addrTombstone := oidtest.Address() addrTombstone.SetContainer(cnr) + const tombstoneExpEpoch = 1008 inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) - inhumePrm.SetTombstoneAddress(addrTombstone) + inhumePrm.SetTombstoneAddress(addrTombstone, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) @@ -232,10 +233,12 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { addrTombstone.SetContainer(cnr) var inhumePrm meta.InhumePrm + + const tombstoneExpEpoch = 1008 inhumePrm.SetAddresses( object.AddressOf(obj1), object.AddressOf(obj2), object.AddressOf(obj3), object.AddressOf(obj4)) - inhumePrm.SetTombstoneAddress(addrTombstone) + inhumePrm.SetTombstoneAddress(addrTombstone, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) @@ -428,8 +431,10 @@ func TestDB_InhumeTombstones(t *testing.T) { addrTombstone := object.AddressOf(objTs) var inhumePrm meta.InhumePrm + + const tombstoneExpEpoch = 1008 inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) - inhumePrm.SetTombstoneAddress(addrTombstone) + inhumePrm.SetTombstoneAddress(addrTombstone, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 76018fb61..0eb051906 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -27,6 +27,8 @@ type InhumePrm struct { lockObjectHandling bool forceRemoval bool + + expEpoch uint64 } // DeletionInfo contains details on deleted object. @@ -119,8 +121,9 @@ func (p *InhumePrm) SetAddresses(addrs ...oid.Address) { // // addr should not be nil. // Should not be called along with SetGCMark. -func (p *InhumePrm) SetTombstoneAddress(addr oid.Address) { +func (p *InhumePrm) SetTombstoneAddress(addr oid.Address, expEpoch uint64) { p.tomb = &addr + p.expEpoch = expEpoch } // SetGCMark marks the object to be physically removed. @@ -365,7 +368,10 @@ func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Bucket, prm InhumePrm) (targetBucket *bbolt.Bucket, value []byte, err error) { if prm.tomb != nil { targetBucket = graveyardBKT - tombKey := addressKey(*prm.tomb, make([]byte, addressKeySize)) + tombKey := make([]byte, addressKeySize+epochSize) + if err = encodeTombstoneKeyWithExpEpoch(*prm.tomb, prm.expEpoch, tombKey); err != nil { + return nil, nil, fmt.Errorf("encode tombstone key with expiration epoch: %w", err) + } // it is forbidden to have a tomb-on-tomb in FrostFS, // so graveyard keys must not be addresses of tombstones @@ -376,6 +382,14 @@ func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Buck return nil, nil, fmt.Errorf("remove grave with tombstone key: %w", err) } } + // it can be a tombstone key without an expiration epoch + data = targetBucket.Get(tombKey[:addressKeySize]) + if data != nil { + err := targetBucket.Delete(tombKey[:addressKeySize]) + if err != nil { + return nil, nil, fmt.Errorf("remove grave with tombstone key: %w", err) + } + } value = tombKey } else { @@ -418,6 +432,9 @@ func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Buc func isTomb(graveyardBucket *bbolt.Bucket, addressKey []byte) bool { targetIsTomb := false + // take only address because graveyard record may have expiration epoch suffix + addressKey = addressKey[:addressKeySize] + // iterate over graveyard and check if target address // is the address of tombstone in graveyard. // tombstone must have the same container ID as key. @@ -426,7 +443,7 @@ func isTomb(graveyardBucket *bbolt.Bucket, addressKey []byte) bool { for k, v := c.Seek(containerPrefix); k != nil && bytes.HasPrefix(k, containerPrefix); k, v = c.Next() { // check if graveyard has record with key corresponding // to tombstone address (at least one) - targetIsTomb = bytes.Equal(v, addressKey) + targetIsTomb = bytes.HasPrefix(v, addressKey) if targetIsTomb { break } diff --git a/pkg/local_object_storage/metabase/inhume_ec_test.go b/pkg/local_object_storage/metabase/inhume_ec_test.go index 180713287..632bf67f2 100644 --- a/pkg/local_object_storage/metabase/inhume_ec_test.go +++ b/pkg/local_object_storage/metabase/inhume_ec_test.go @@ -94,10 +94,11 @@ func TestInhumeECObject(t *testing.T) { require.True(t, res.deletionDetails[0].Size == 5) // inhume EC parent (like Delete does) + const tombstoneExpEpoch = 1008 tombAddress.SetContainer(cnr) tombAddress.SetObject(tombstoneID) inhumePrm.SetAddresses(ecParentAddress) - inhumePrm.SetTombstoneAddress(tombAddress) + inhumePrm.SetTombstoneAddress(tombAddress, tombstoneExpEpoch) res, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) // Previously deleted chunk shouldn't be in the details, because it is marked as garbage diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index 786d10396..be0bc7388 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -50,6 +50,7 @@ func TestInhumeTombOnTomb(t *testing.T) { inhumePrm meta.InhumePrm existsPrm meta.ExistsPrm ) + const tombstoneExpEpoch = 1008 addr1.SetContainer(cnr) addr2.SetContainer(cnr) @@ -57,7 +58,7 @@ func TestInhumeTombOnTomb(t *testing.T) { addr4.SetContainer(cnr) inhumePrm.SetAddresses(addr1) - inhumePrm.SetTombstoneAddress(addr2) + inhumePrm.SetTombstoneAddress(addr2, tombstoneExpEpoch) // inhume addr1 via addr2 _, err = db.Inhume(context.Background(), inhumePrm) @@ -70,7 +71,7 @@ func TestInhumeTombOnTomb(t *testing.T) { require.True(t, client.IsErrObjectAlreadyRemoved(err)) inhumePrm.SetAddresses(addr3) - inhumePrm.SetTombstoneAddress(addr1) + inhumePrm.SetTombstoneAddress(addr1, tombstoneExpEpoch) // try to inhume addr3 via addr1 _, err = db.Inhume(context.Background(), inhumePrm) @@ -90,7 +91,7 @@ func TestInhumeTombOnTomb(t *testing.T) { require.True(t, client.IsErrObjectAlreadyRemoved(err)) inhumePrm.SetAddresses(addr1) - inhumePrm.SetTombstoneAddress(addr4) + inhumePrm.SetTombstoneAddress(addr4, tombstoneExpEpoch) // try to inhume addr1 (which is already a tombstone in graveyard) _, err = db.Inhume(context.Background(), inhumePrm) @@ -124,12 +125,14 @@ func TestInhumeLocked(t *testing.T) { } func metaInhume(db *meta.DB, target oid.Address, tomb oid.ID) error { + const tombstoneExpEpoch = 1008 + var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(target) var tombAddr oid.Address tombAddr.SetContainer(target.Container()) tombAddr.SetObject(tomb) - inhumePrm.SetTombstoneAddress(tombAddr) + inhumePrm.SetTombstoneAddress(tombAddr, tombstoneExpEpoch) _, err := db.Inhume(context.Background(), inhumePrm) return err diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 341ff9ad1..140d30522 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -62,6 +62,8 @@ func TestDB_Lock(t *testing.T) { objAddr := objectcore.AddressOf(objs[0]) lockAddr := objectcore.AddressOf(lockObj) + const tombstoneExpEpoch = 1008 + var inhumePrm meta.InhumePrm inhumePrm.SetGCMark() @@ -75,7 +77,7 @@ func TestDB_Lock(t *testing.T) { tombAddr := oidtest.Address() tombAddr.SetContainer(objAddr.Container()) - inhumePrm.SetTombstoneAddress(tombAddr) + inhumePrm.SetTombstoneAddress(tombAddr, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) @@ -93,7 +95,7 @@ func TestDB_Lock(t *testing.T) { tombAddr = oidtest.Address() tombAddr.SetContainer(objAddr.Container()) - inhumePrm.SetTombstoneAddress(tombAddr) + inhumePrm.SetTombstoneAddress(tombAddr, tombstoneExpEpoch) _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) }) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 80851f1c4..57e947f43 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -309,3 +309,59 @@ func isLockObject(tx *bbolt.Tx, idCnr cid.ID, obj oid.ID) bool { bucketNameLockers(idCnr, make([]byte, bucketKeySize)), objectKey(obj, make([]byte, objectKeySize))) } + +const NoExpirationEpoch uint64 = 0 + +// encodeTombstoneKeyWithExpEpoch encodes a tombstone key of a tombstoned +// object if the following format: tombstone address + expiration epoch. +// +// Returns an error if the buffer length isn't 32. +// +// The expiration epoch shouldn't be [NoExpirationEpoch], as tombstone keys +// are intended to have a valid expiration epoch. +// +// The use of [NoExpirationEpoch] is allowed only for test purposes. +func encodeTombstoneKeyWithExpEpoch(addr oid.Address, expEpoch uint64, dst []byte) error { + if len(dst) != addressKeySize+epochSize { + return errInvalidLength + } + + addr.Container().Encode(dst[:cidSize]) + addr.Object().Encode(dst[cidSize:addressKeySize]) + binary.LittleEndian.PutUint64(dst[addressKeySize:], expEpoch) + + return nil +} + +// decodeTombstoneKeyWithExpEpoch decodes a tombstone key of a tombstoned object. +// The tombstone key may have one of the following formats: +// - tombstone address +// - tombstone address + expiration epoch +// +// Expiration epoch is set to [NoExpirationEpoch] if the key doesn't have it. +func decodeTombstoneKeyWithExpEpoch(addr *oid.Address, expEpoch *uint64, src []byte) error { + if len(src) != addressKeySize && len(src) != addressKeySize+epochSize { + return errInvalidLength + } + + var cnt cid.ID + if err := cnt.Decode(src[:cidSize]); err != nil { + return err + } + + var obj oid.ID + if err := obj.Decode(src[cidSize:addressKeySize]); err != nil { + return err + } + + addr.SetContainer(cnt) + addr.SetObject(obj) + + if len(src) > addressKeySize { + *expEpoch = binary.LittleEndian.Uint64(src[addressKeySize:]) + } else { + *expEpoch = NoExpirationEpoch + } + + return nil +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 3136ddfcc..4a5a1ef46 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -334,6 +334,11 @@ func (s *Shard) refillLockObject(ctx context.Context, obj *objectSDK.Object) err } func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object) error { + expEpoch, ok := object.ExpirationEpoch(obj) + if !ok { + return fmt.Errorf("tombstone %s has no expiration epoch", object.AddressOf(obj)) + } + tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(obj.Payload()); err != nil { @@ -353,7 +358,7 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object var inhumePrm meta.InhumePrm - inhumePrm.SetTombstoneAddress(tombAddr) + inhumePrm.SetTombstoneAddress(tombAddr, expEpoch) inhumePrm.SetAddresses(tombMembers...) _, err := s.metaBase.Inhume(ctx, inhumePrm) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 6d2cd7137..1cf391330 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -7,6 +7,7 @@ import ( "math" "os" "path/filepath" + "strconv" "sync/atomic" "testing" @@ -19,6 +20,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -234,8 +236,15 @@ func TestRefillMetabase(t *testing.T) { } } + expirationEpoch := 1008 + + expirationAttr := *objectSDK.NewAttribute() + expirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + expirationAttr.SetValue(strconv.Itoa(expirationEpoch)) + tombObj := objecttest.Object() tombObj.SetType(objectSDK.TypeTombstone) + tombObj.SetAttributes(expirationAttr) tombstone := objecttest.Tombstone() @@ -276,6 +285,7 @@ func TestRefillMetabase(t *testing.T) { lockObj := objecttest.Object() lockObj.SetContainerID(cnrLocked) + lockObj.SetAttributes(expirationAttr) objectSDK.WriteLock(lockObj, lock) putPrm.SetObject(lockObj) @@ -286,7 +296,7 @@ func TestRefillMetabase(t *testing.T) { require.NoError(t, sh.Lock(context.Background(), cnrLocked, lockID, locked)) var inhumePrm InhumePrm - inhumePrm.SetTarget(object.AddressOf(tombObj), tombMembers...) + inhumePrm.SetTarget(object.AddressOf(tombObj), uint64(expirationEpoch), tombMembers...) _, err = sh.Inhume(context.Background(), inhumePrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index d46400869..4ca2e908d 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -20,6 +20,7 @@ type InhumePrm struct { target []oid.Address tombstone *oid.Address forceRemoval bool + expEpoch uint64 } // InhumeRes encapsulates results of inhume operation. @@ -30,9 +31,10 @@ type InhumeRes struct{} // // tombstone should not be nil, addr should not be empty. // Should not be called along with MarkAsGarbage. -func (p *InhumePrm) SetTarget(tombstone oid.Address, addrs ...oid.Address) { +func (p *InhumePrm) SetTarget(tombstone oid.Address, expEpoch uint64, addrs ...oid.Address) { p.target = addrs p.tombstone = &tombstone + p.expEpoch = expEpoch } // MarkAsGarbage marks object to be physically removed from shard. @@ -93,7 +95,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { metaPrm.SetLockObjectHandling() if prm.tombstone != nil { - metaPrm.SetTombstoneAddress(*prm.tombstone) + metaPrm.SetTombstoneAddress(*prm.tombstone, prm.expEpoch) } else { metaPrm.SetGCMark() } diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 1421f0e18..cbad41039 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -40,7 +40,8 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) var inhPrm InhumePrm - inhPrm.SetTarget(object.AddressOf(ts), object.AddressOf(obj)) + const tombstoneExpEpoch = 1008 + inhPrm.SetTarget(object.AddressOf(ts), tombstoneExpEpoch, object.AddressOf(obj)) var getPrm GetPrm getPrm.SetAddress(object.AddressOf(obj)) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 5caf3641f..964890a62 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -89,11 +89,13 @@ func TestShard_Lock(t *testing.T) { _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) + const tombstoneExpEpoch = 1008 + t.Run("inhuming locked objects", func(t *testing.T) { ts := testutil.GenerateObjectWithCID(cnr) var inhumePrm InhumePrm - inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(obj)) + inhumePrm.SetTarget(objectcore.AddressOf(ts), tombstoneExpEpoch, objectcore.AddressOf(obj)) var objLockedErr *apistatus.ObjectLocked @@ -109,7 +111,7 @@ func TestShard_Lock(t *testing.T) { ts := testutil.GenerateObjectWithCID(cnr) var inhumePrm InhumePrm - inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(lock)) + inhumePrm.SetTarget(objectcore.AddressOf(ts), tombstoneExpEpoch, objectcore.AddressOf(lock)) _, err = sh.Inhume(context.Background(), inhumePrm) require.Error(t, err) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 5230dcad0..8f79dcf95 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -314,11 +314,13 @@ func TestCounters(t *testing.T) { logic := mm.getObjectCounter(logical) custom := mm.getObjectCounter(user) + const tombstoneExpEpoch = 1008 + inhumedNumber := int(phy / 4) for _, o := range addrFromObjs(oo[:inhumedNumber]) { ts := oidtest.Address() ts.SetContainer(o.Container()) - prm.SetTarget(ts, o) + prm.SetTarget(ts, tombstoneExpEpoch, o) _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/services/object/common/writer/local.go b/pkg/services/object/common/writer/local.go index cf3d03275..ae72e9a66 100644 --- a/pkg/services/object/common/writer/local.go +++ b/pkg/services/object/common/writer/local.go @@ -2,6 +2,7 @@ package writer import ( "context" + "errors" "fmt" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -11,6 +12,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) +var errObjectHasNoExpirationEpoch = errors.New("object has no expiration epoch") + // ObjectStorage is an object storage interface. type ObjectStorage interface { // Put must save passed object @@ -18,7 +21,7 @@ type ObjectStorage interface { Put(context.Context, *objectSDK.Object, bool) error // Delete must delete passed objects // and return any appeared error. - Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error + Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID, expEpoch uint64) error // Lock must lock passed objects // and return any appeared error. Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error @@ -38,7 +41,12 @@ func (t LocalTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, met switch meta.Type() { case objectSDK.TypeTombstone: - err := t.Storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) + expEpoch, ok := objectCore.ExpirationEpoch(obj) + if !ok { + return errObjectHasNoExpirationEpoch + } + + err := t.Storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects(), expEpoch) if err != nil { return fmt.Errorf("could not delete objects from tombstone locally: %w", err) } diff --git a/scripts/populate-metabase/internal/populate.go b/scripts/populate-metabase/internal/populate.go index 4da23a295..9e2203173 100644 --- a/scripts/populate-metabase/internal/populate.go +++ b/scripts/populate-metabase/internal/populate.go @@ -185,7 +185,7 @@ func PopulateGraveyard( for addr := range addrs { prm := meta.InhumePrm{} prm.SetAddresses(addr) - prm.SetTombstoneAddress(tsAddr) + prm.SetTombstoneAddress(tsAddr, rand.Uint64()) group.Go(func() error { if _, err := db.Inhume(ctx, prm); err != nil {