TrueCloudLab/frostfs-node
18
1
Fork 27
Code Issues 91 Pull requests 8 Projects Releases 1 Activity Actions

metabase: optimize Inhume operation #1385

Open
dstepanov-yadro wants to merge 3 commits from dstepanov-yadro/frostfs-node:fix/metabase_inhume_hangs into master
pull from: dstepanov-yadro/frostfs-node:fix/metabase_inhume_hangs
merge into: TrueCloudLab:master
Conversation 0 Commits 3 Files changed 20 +108 -52
16 changed files with 108 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 861a5f1cc3 - Show all commits

8
pkg/local_object_storage/engine/lock_test.go
View file

@ -199,7 +199,9 @@ func TestLockExpiration(t *testing.T) {
require.NoError(t, err)
var inhumePrm InhumePrm
inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj))
tombAddr := oidtest.Address()
tombAddr.SetContainer(cnr)
inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj))
var objLockedErr *apistatus.ObjectLocked
_, err = e.Inhume(context.Background(), inhumePrm)
@ -209,7 +211,9 @@ func TestLockExpiration(t *testing.T) {
e.HandleNewEpoch(context.Background(), lockerExpiresAfter+1)
// 4.
inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj))
tombAddr = oidtest.Address()
tombAddr.SetContainer(cnr)
inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj))
require.Eventually(t, func() bool {
_, err = e.Inhume(context.Background(), inhumePrm)

4
pkg/local_object_storage/metabase/containers_test.go
View file

@ -67,7 +67,7 @@ func TestDB_Containers(t *testing.T) {
assertContains(cnrs, cnr)
require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.Address()))
require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.ID()))
cnrs, err = db.Containers(context.Background())
require.NoError(t, err)
@ -164,7 +164,7 @@ func TestDB_ContainerSize(t *testing.T) {
require.NoError(t, metaInhume(
db,
object.AddressOf(obj),
oidtest.Address(),
oidtest.ID(),
))
volume -= int(obj.PayloadSize())

2
pkg/local_object_storage/metabase/control_test.go
View file

@ -41,7 +41,7 @@ func TestReset(t *testing.T) {
err = putBig(db, obj)
require.NoError(t, err)
err = metaInhume(db, addrToInhume, oidtest.Address())
err = metaInhume(db, addrToInhume, oidtest.ID())
require.NoError(t, err)
assertExists(addr, true, nil)

30
pkg/local_object_storage/metabase/counter_test.go
View file

@ -156,13 +156,18 @@ func TestCounters(t *testing.T) {
}
var prm meta.InhumePrm
prm.SetTombstoneAddress(oidtest.Address())
prm.SetAddresses(inhumedObjs...)
for _, o := range inhumedObjs {
tombAddr := oidtest.Address()
tombAddr.SetContainer(o.Container())
res, err := db.Inhume(context.Background(), prm)
require.NoError(t, err)
require.Equal(t, uint64(len(inhumedObjs)), res.LogicInhumed())
require.Equal(t, uint64(len(inhumedObjs)), res.UserInhumed())
prm.SetTombstoneAddress(tombAddr)
prm.SetAddresses(o)
res, err := db.Inhume(context.Background(), prm)
require.NoError(t, err)
require.Equal(t, uint64(1), res.LogicInhumed())
require.Equal(t, uint64(1), res.UserInhumed())
}
c, err := db.ObjectCounters()
require.NoError(t, err)
@ -296,11 +301,16 @@ func TestCounters(t *testing.T) {
}
var prm meta.InhumePrm
prm.SetTombstoneAddress(oidtest.Address())
prm.SetAddresses(inhumedObjs...)
for _, o := range inhumedObjs {
tombAddr := oidtest.Address()
tombAddr.SetContainer(o.Container())
_, err := db.Inhume(context.Background(), prm)
require.NoError(t, err)
prm.SetTombstoneAddress(tombAddr)
prm.SetAddresses(o)
_, err := db.Inhume(context.Background(), prm)
require.NoError(t, err)
}
c, err := db.ObjectCounters()
require.NoError(t, err)

6
pkg/local_object_storage/metabase/delete_test.go
View file

@ -40,12 +40,12 @@ func TestDB_Delete(t *testing.T) {
// inhume parent and child so they will be on graveyard
ts := testutil.GenerateObjectWithCID(cnr)
err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts))
err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts).Object())
require.NoError(t, err)
ts = testutil.GenerateObjectWithCID(cnr)
err = metaInhume(db, object.AddressOf(parent), object.AddressOf(ts))
err = metaInhume(db, object.AddressOf(parent), object.AddressOf(ts).Object())
require.NoError(t, err)
// delete object
@ -108,7 +108,7 @@ func TestGraveOnlyDelete(t *testing.T) {
addr := oidtest.Address()
// inhume non-existent object by address
require.NoError(t, metaInhume(db, addr, oidtest.Address()))
require.NoError(t, metaInhume(db, addr, oidtest.ID()))
// delete the object data
require.NoError(t, metaDelete(db, addr))

2
pkg/local_object_storage/metabase/exists_test.go
View file

@ -37,7 +37,7 @@ func TestDB_Exists(t *testing.T) {
require.True(t, exists)
t.Run("removed object", func(t *testing.T) {
err := metaInhume(db, object.AddressOf(regular), oidtest.Address())
err := metaInhume(db, object.AddressOf(regular), oidtest.ID())
require.NoError(t, err)
exists, err := metaExists(db, object.AddressOf(regular))

3
pkg/local_object_storage/metabase/get_test.go
View file

@ -150,9 +150,8 @@ func TestDB_Get(t *testing.T) {
t.Run("get removed object", func(t *testing.T) {
obj := oidtest.Address()
ts := oidtest.Address()
require.NoError(t, metaInhume(db, obj, ts))
require.NoError(t, metaInhume(db, obj, oidtest.ID()))
_, err := metaGet(db, obj, false)
require.True(t, client.IsErrObjectAlreadyRemoved(err))

27
pkg/local_object_storage/metabase/graveyard_test.go
View file

@ -7,6 +7,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
@ -114,11 +115,12 @@ func TestDB_IterateDeletedObjects(t *testing.T) {
db := newDB(t)
defer func() { require.NoError(t, db.Close()) }()
cnr := cidtest.ID()
// generate and put 4 objects
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
obj3 := testutil.GenerateObject()
obj4 := testutil.GenerateObject()
obj1 := testutil.GenerateObjectWithCID(cnr)
obj2 := testutil.GenerateObjectWithCID(cnr)
obj3 := testutil.GenerateObjectWithCID(cnr)
obj4 := testutil.GenerateObjectWithCID(cnr)
var err error
@ -138,6 +140,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) {
// inhume with tombstone
addrTombstone := oidtest.Address()
addrTombstone.SetContainer(cnr)
inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2))
inhumePrm.SetTombstoneAddress(addrTombstone)
@ -201,11 +204,12 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) {
db := newDB(t)
defer func() { require.NoError(t, db.Close()) }()
cnr := cidtest.ID()
// generate and put 4 objects
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
obj3 := testutil.GenerateObject()
obj4 := testutil.GenerateObject()
obj1 := testutil.GenerateObjectWithCID(cnr)
obj2 := testutil.GenerateObjectWithCID(cnr)
obj3 := testutil.GenerateObjectWithCID(cnr)
obj4 := testutil.GenerateObjectWithCID(cnr)
var err error
@ -223,6 +227,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) {
// inhume with tombstone
addrTombstone := oidtest.Address()
addrTombstone.SetContainer(cnr)
var inhumePrm meta.InhumePrm
inhumePrm.SetAddresses(
@ -392,9 +397,10 @@ func TestDB_DropGraves(t *testing.T) {
db := newDB(t)
defer func() { require.NoError(t, db.Close()) }()
cnr := cidtest.ID()
// generate and put 2 objects
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
obj1 := testutil.GenerateObjectWithCID(cnr)
obj2 := testutil.GenerateObjectWithCID(cnr)
var err error
@ -406,6 +412,7 @@ func TestDB_DropGraves(t *testing.T) {
// inhume with tombstone
addrTombstone := oidtest.Address()
addrTombstone.SetContainer(cnr)
var inhumePrm meta.InhumePrm
inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2))

18
pkg/local_object_storage/metabase/inhume.go
View file

@ -143,6 +143,20 @@ func (p *InhumePrm) SetForceGCMark() {
p.forceRemoval = true
}
func (p *InhumePrm) validate() error {
if p == nil {
return nil
}
if p.tomb != nil {
for _, addr := range p.target {
if addr.Container() != p.tomb.Container() {
return fmt.Errorf("object %s and tombstone %s have different container ID", addr, p.tomb)
}
}
}
return nil
}
var errBreakBucketForEach = errors.New("bucket ForEach break")
// ErrLockObjectRemoval is returned when inhume operation is being
@ -171,6 +185,10 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) {
db.modeMtx.RLock()
defer db.modeMtx.RUnlock()
if err := prm.validate(); err != nil {
return InhumeRes{}, err
}
if db.mode.NoMetabase() {
return InhumeRes{}, ErrDegradedMode
} else if db.mode.ReadOnly() {

21
pkg/local_object_storage/metabase/inhume_test.go
View file

@ -9,6 +9,7 @@ import (
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
@ -21,12 +22,10 @@ func TestDB_Inhume(t *testing.T) {
raw := testutil.GenerateObject()
testutil.AddAttribute(raw, "foo", "bar")
tombstoneID := oidtest.Address()
err := putBig(db, raw)
require.NoError(t, err)
err = metaInhume(db, object.AddressOf(raw), tombstoneID)
err = metaInhume(db, object.AddressOf(raw), oidtest.ID())
require.NoError(t, err)
_, err = metaExists(db, object.AddressOf(raw))
@ -43,13 +42,20 @@ func TestInhumeTombOnTomb(t *testing.T) {
var (
err error
cnr = cidtest.ID()
addr1 = oidtest.Address()
addr2 = oidtest.Address()
addr3 = oidtest.Address()
addr4 = oidtest.Address()
inhumePrm meta.InhumePrm
existsPrm meta.ExistsPrm
)
addr1.SetContainer(cnr)
addr2.SetContainer(cnr)
addr3.SetContainer(cnr)
addr4.SetContainer(cnr)
inhumePrm.SetAddresses(addr1)
inhumePrm.SetTombstoneAddress(addr2)
@ -84,7 +90,7 @@ func TestInhumeTombOnTomb(t *testing.T) {
require.True(t, client.IsErrObjectAlreadyRemoved(err))
inhumePrm.SetAddresses(addr1)
inhumePrm.SetTombstoneAddress(oidtest.Address())
inhumePrm.SetTombstoneAddress(addr4)
// try to inhume addr1 (which is already a tombstone in graveyard)
_, err = db.Inhume(context.Background(), inhumePrm)
@ -117,10 +123,13 @@ func TestInhumeLocked(t *testing.T) {
require.ErrorAs(t, err, &e)
}
func metaInhume(db *meta.DB, target, tomb oid.Address) error {
func metaInhume(db *meta.DB, target oid.Address, tomb oid.ID) error {
var inhumePrm meta.InhumePrm
inhumePrm.SetAddresses(target)
inhumePrm.SetTombstoneAddress(tomb)
var tombAddr oid.Address
tombAddr.SetContainer(target.Container())
tombAddr.SetObject(tomb)
inhumePrm.SetTombstoneAddress(tombAddr)
_, err := db.Inhume(context.Background(), inhumePrm)
return err

6
pkg/local_object_storage/metabase/iterators_test.go
View file

@ -9,6 +9,7 @@ import (
object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
@ -71,11 +72,16 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) {
db := newDB(t)
defer func() { require.NoError(t, db.Close()) }()
cnr := cidtest.ID()
ts := oidtest.Address()
protected1 := oidtest.Address()
protected2 := oidtest.Address()
protectedLocked := oidtest.Address()
garbage := oidtest.Address()
ts.SetContainer(cnr)
protected1.SetContainer(cnr)
protected2.SetContainer(cnr)
protectedLocked.SetContainer(cnr)
var prm meta.InhumePrm
var err error

2
pkg/local_object_storage/metabase/list_test.go
View file

@ -110,7 +110,7 @@ func TestLisObjectsWithCursor(t *testing.T) {
err = putBig(db, obj)
require.NoError(t, err)
ts := testutil.GenerateObjectWithCID(containerID)
err = metaInhume(db, object.AddressOf(obj), object.AddressOf(ts))
err = metaInhume(db, object.AddressOf(obj), object.AddressOf(ts).Object())
require.NoError(t, err)
// add one child object (do not include parent into expected)

10
pkg/local_object_storage/metabase/lock_test.go
View file

@ -73,7 +73,9 @@ func TestDB_Lock(t *testing.T) {
_, err := db.Inhume(context.Background(), inhumePrm)
require.ErrorAs(t, err, &objLockedErr)
inhumePrm.SetTombstoneAddress(oidtest.Address())
tombAddr := oidtest.Address()
tombAddr.SetContainer(objAddr.Container())
inhumePrm.SetTombstoneAddress(tombAddr)
_, err = db.Inhume(context.Background(), inhumePrm)
require.ErrorAs(t, err, &objLockedErr)
@ -89,7 +91,9 @@ func TestDB_Lock(t *testing.T) {
_, err = db.Inhume(context.Background(), inhumePrm)
require.ErrorAs(t, err, &objLockedErr)
inhumePrm.SetTombstoneAddress(oidtest.Address())
tombAddr = oidtest.Address()
tombAddr.SetContainer(objAddr.Container())
inhumePrm.SetTombstoneAddress(tombAddr)
_, err = db.Inhume(context.Background(), inhumePrm)
require.ErrorAs(t, err, &objLockedErr)
})
@ -103,7 +107,7 @@ func TestDB_Lock(t *testing.T) {
var objLockedErr *apistatus.ObjectLocked
// try to inhume locked object using tombstone
err := metaInhume(db, objAddr, lockAddr)
err := metaInhume(db, objAddr, lockAddr.Object())
require.ErrorAs(t, err, &objLockedErr)
// free locked object

6
pkg/local_object_storage/metabase/select_test.go
View file

@ -352,11 +352,7 @@ func TestDB_SelectInhume(t *testing.T) {
object.AddressOf(raw2),
)
var tombstone oid.Address
tombstone.SetContainer(cnr)
tombstone.SetObject(oidtest.ID())
err = metaInhume(db, object.AddressOf(raw2), tombstone)
err = metaInhume(db, object.AddressOf(raw2), oidtest.ID())
require.NoError(t, err)
fs = objectSDK.SearchFilters{}

2
pkg/local_object_storage/metabase/storage_id_test.go
View file

@ -43,7 +43,7 @@ func TestDB_StorageID(t *testing.T) {
cnrID, ok := deleted.ContainerID()
require.True(t, ok)
ts := testutil.GenerateObjectWithCID(cnrID)
require.NoError(t, metaInhume(db, object.AddressOf(deleted), object.AddressOf(ts)))
require.NoError(t, metaInhume(db, object.AddressOf(deleted), object.AddressOf(ts).Object()))
// check StorageID for object without storageID
fetchedStorageID, err = metaStorageID(db, object.AddressOf(raw2))

13
pkg/local_object_storage/shard/metrics_test.go
View file

@ -17,6 +17,7 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
@ -308,17 +309,19 @@ func TestCounters(t *testing.T) {
t.Run("inhume_TS", func(t *testing.T) {
var prm InhumePrm
ts := objectcore.AddressOf(testutil.GenerateObject())
phy := mm.getObjectCounter(physical)
logic := mm.getObjectCounter(logical)
custom := mm.getObjectCounter(user)
inhumedNumber := int(phy / 4)
prm.SetTarget(ts, addrFromObjs(oo[:inhumedNumber])...)
_, err := sh.Inhume(context.Background(), prm)
require.NoError(t, err)
for _, o := range addrFromObjs(oo[:inhumedNumber]) {
ts := oidtest.Address()
ts.SetContainer(o.Container())
prm.SetTarget(ts, o)
_, err := sh.Inhume(context.Background(), prm)
require.NoError(t, err)
}
for i := range inhumedNumber {
cid, ok := oo[i].ContainerID()