forked from TrueCloudLab/frostfs-node
Evgenii Stratonikov
d77a218f7c
DropGraves() is only used to drop gravemarks after a tombstone removal. Thus, it makes sense to do Inhume() and DropGraves() in one transaction. It has less overhead and no unexpected problems in case of sudden power failure. Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
443 lines
14 KiB
Go
443 lines
14 KiB
Go
package meta
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
"github.com/stretchr/testify/require"
|
|
"go.etcd.io/bbolt"
|
|
)
|
|
|
|
func TestDeleteECObject_WithoutSplit(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db := New(
|
|
WithPath(filepath.Join(t.TempDir(), "metabase")),
|
|
WithPermissions(0o600),
|
|
WithEpochState(epochState{uint64(12)}),
|
|
)
|
|
|
|
require.NoError(t, db.Open(context.Background(), mode.ReadWrite))
|
|
require.NoError(t, db.Init(context.Background()))
|
|
defer func() { require.NoError(t, db.Close(context.Background())) }()
|
|
|
|
cnr := cidtest.ID()
|
|
ecChunk := oidtest.ID()
|
|
ecParent := oidtest.ID()
|
|
tombstoneID := oidtest.ID()
|
|
|
|
chunkObj := testutil.GenerateObjectWithCID(cnr)
|
|
chunkObj.SetID(ecChunk)
|
|
chunkObj.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
|
|
chunkObj.SetPayloadSize(uint64(10))
|
|
chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0))
|
|
|
|
// put object with EC
|
|
|
|
var prm PutPrm
|
|
prm.SetObject(chunkObj)
|
|
prm.SetStorageID([]byte("0/0"))
|
|
_, err := db.Put(context.Background(), prm)
|
|
require.NoError(t, err)
|
|
|
|
var ecChunkAddress oid.Address
|
|
ecChunkAddress.SetContainer(cnr)
|
|
ecChunkAddress.SetObject(ecChunk)
|
|
|
|
var ecParentAddress oid.Address
|
|
ecParentAddress.SetContainer(cnr)
|
|
ecParentAddress.SetObject(ecParent)
|
|
|
|
var getPrm GetPrm
|
|
|
|
getPrm.SetAddress(ecChunkAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.NoError(t, err)
|
|
|
|
var ecInfoError *objectSDK.ECInfoError
|
|
getPrm.SetAddress(ecParentAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, &ecInfoError)
|
|
require.True(t, len(ecInfoError.ECInfo().Chunks) == 1 &&
|
|
ecInfoError.ECInfo().Chunks[0].Index == 0 &&
|
|
ecInfoError.ECInfo().Chunks[0].Total == 3)
|
|
|
|
// inhume EC parent (like Delete does)
|
|
|
|
var inhumePrm InhumePrm
|
|
var tombAddress oid.Address
|
|
tombAddress.SetContainer(cnr)
|
|
tombAddress.SetObject(tombstoneID)
|
|
inhumePrm.SetAddresses(ecParentAddress)
|
|
inhumePrm.SetTombstoneAddress(tombAddress)
|
|
_, err = db.Inhume(context.Background(), inhumePrm)
|
|
require.NoError(t, err)
|
|
|
|
getPrm.SetAddress(ecParentAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
|
|
|
getPrm.SetAddress(ecChunkAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
|
|
|
// GC finds and deletes split, EC parent and EC chunk
|
|
|
|
var garbageAddresses []oid.Address
|
|
var itPrm GarbageIterationPrm
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
garbageAddresses = append(garbageAddresses, g.Address())
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
require.Equal(t, 2, len(garbageAddresses))
|
|
require.True(t, slices.Contains(garbageAddresses, ecParentAddress))
|
|
require.True(t, slices.Contains(garbageAddresses, ecChunkAddress))
|
|
|
|
var deletePrm DeletePrm
|
|
deletePrm.SetAddresses(garbageAddresses...)
|
|
_, err = db.Delete(context.Background(), deletePrm)
|
|
require.NoError(t, err)
|
|
|
|
garbageAddresses = nil
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
garbageAddresses = append(garbageAddresses, g.Address())
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
require.Equal(t, 0, len(garbageAddresses))
|
|
|
|
// after tombstone expired GC inhumes tombstone and drops graves
|
|
|
|
var tombstonedObjects []TombstonedObject
|
|
var graveyardIterationPrm GraveyardIterationPrm
|
|
graveyardIterationPrm.SetHandler(func(object TombstonedObject) error {
|
|
tombstonedObjects = append(tombstonedObjects, object)
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGraveyard(context.Background(), graveyardIterationPrm))
|
|
require.Equal(t, 2, len(tombstonedObjects))
|
|
|
|
_, err = db.InhumeTombstones(context.Background(), tombstonedObjects)
|
|
require.NoError(t, err)
|
|
|
|
// GC finds tombstone as garbage and deletes it
|
|
|
|
garbageAddresses = nil
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
garbageAddresses = append(garbageAddresses, g.Address())
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
require.Equal(t, 1, len(garbageAddresses))
|
|
require.Equal(t, tombstoneID, garbageAddresses[0].Object())
|
|
|
|
deletePrm.SetAddresses(garbageAddresses...)
|
|
_, err = db.Delete(context.Background(), deletePrm)
|
|
require.NoError(t, err)
|
|
|
|
// no more objects should left as garbage
|
|
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
require.FailNow(t, "no garbage objects should left")
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
|
|
require.NoError(t, db.boltDB.View(testVerifyNoObjectDataLeft))
|
|
|
|
require.NoError(t, testCountersAreZero(db, cnr))
|
|
}
|
|
|
|
func TestDeleteECObject_WithSplit(t *testing.T) {
|
|
t.Parallel()
|
|
for _, c := range []int{1, 2, 3} {
|
|
for _, l := range []bool{true, false} {
|
|
test := fmt.Sprintf("%d EC chunks with split info without linking object", c)
|
|
if l {
|
|
test = fmt.Sprintf("%d EC chunks with split info with linking object", c)
|
|
}
|
|
t.Run(test, func(t *testing.T) {
|
|
testDeleteECObjectWithSplit(t, c, l)
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func testDeleteECObjectWithSplit(t *testing.T, chunksCount int, withLinking bool) {
|
|
t.Parallel()
|
|
|
|
db := New(
|
|
WithPath(filepath.Join(t.TempDir(), "metabase")),
|
|
WithPermissions(0o600),
|
|
WithEpochState(epochState{uint64(12)}),
|
|
)
|
|
|
|
require.NoError(t, db.Open(context.Background(), mode.ReadWrite))
|
|
require.NoError(t, db.Init(context.Background()))
|
|
defer func() { require.NoError(t, db.Close(context.Background())) }()
|
|
|
|
cnr := cidtest.ID()
|
|
ecChunks := make([]oid.ID, chunksCount)
|
|
for idx := range ecChunks {
|
|
ecChunks[idx] = oidtest.ID()
|
|
}
|
|
ecParentID := oidtest.ID()
|
|
splitParentID := oidtest.ID()
|
|
tombstoneID := oidtest.ID()
|
|
splitID := objectSDK.NewSplitID()
|
|
linkingID := oidtest.ID()
|
|
|
|
ecChunkObjects := make([]*objectSDK.Object, chunksCount)
|
|
for idx := range ecChunkObjects {
|
|
ecChunkObjects[idx] = testutil.GenerateObjectWithCID(cnr)
|
|
ecChunkObjects[idx].SetContainerID(cnr)
|
|
ecChunkObjects[idx].SetID(ecChunks[idx])
|
|
ecChunkObjects[idx].SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
|
|
ecChunkObjects[idx].SetPayloadSize(uint64(10))
|
|
ecChunkObjects[idx].SetECHeader(objectSDK.NewECHeader(
|
|
objectSDK.ECParentInfo{
|
|
ID: ecParentID,
|
|
SplitParentID: &splitParentID, SplitID: splitID,
|
|
}, uint32(idx), uint32(chunksCount+1), []byte{}, 0))
|
|
}
|
|
|
|
splitParentObj := testutil.GenerateObjectWithCID(cnr)
|
|
splitParentObj.SetID(splitParentID)
|
|
|
|
var linkingAddress oid.Address
|
|
linkingAddress.SetContainer(cnr)
|
|
linkingAddress.SetObject(linkingID)
|
|
|
|
linkingObj := testutil.GenerateObjectWithCID(cnr)
|
|
linkingObj.SetID(linkingID)
|
|
linkingObj.SetParent(splitParentObj)
|
|
linkingObj.SetParentID(splitParentID)
|
|
linkingObj.SetChildren(ecParentID, oidtest.ID(), oidtest.ID())
|
|
linkingObj.SetSplitID(splitID)
|
|
|
|
// put object with EC and split info
|
|
|
|
var prm PutPrm
|
|
prm.SetStorageID([]byte("0/0"))
|
|
for _, obj := range ecChunkObjects {
|
|
prm.SetObject(obj)
|
|
_, err := db.Put(context.Background(), prm)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
if withLinking {
|
|
prm.SetObject(linkingObj)
|
|
_, err := db.Put(context.Background(), prm)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
var ecParentAddress oid.Address
|
|
ecParentAddress.SetContainer(cnr)
|
|
ecParentAddress.SetObject(ecParentID)
|
|
|
|
var getPrm GetPrm
|
|
var ecInfoError *objectSDK.ECInfoError
|
|
getPrm.SetAddress(ecParentAddress)
|
|
_, err := db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, &ecInfoError)
|
|
require.True(t, len(ecInfoError.ECInfo().Chunks) == chunksCount)
|
|
|
|
var splitParentAddress oid.Address
|
|
splitParentAddress.SetContainer(cnr)
|
|
splitParentAddress.SetObject(splitParentID)
|
|
|
|
var splitInfoError *objectSDK.SplitInfoError
|
|
getPrm.SetAddress(splitParentAddress)
|
|
getPrm.SetRaw(true)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, &splitInfoError)
|
|
require.True(t, splitInfoError.SplitInfo() != nil)
|
|
require.Equal(t, splitID, splitInfoError.SplitInfo().SplitID())
|
|
lastPart, set := splitInfoError.SplitInfo().LastPart()
|
|
require.True(t, set)
|
|
require.Equal(t, lastPart, ecParentID)
|
|
if withLinking {
|
|
l, ok := splitInfoError.SplitInfo().Link()
|
|
require.True(t, ok)
|
|
require.Equal(t, linkingID, l)
|
|
}
|
|
getPrm.SetRaw(false)
|
|
|
|
// inhume EC parent and split objects (like Delete does)
|
|
|
|
inhumeAddresses := []oid.Address{splitParentAddress, ecParentAddress}
|
|
if withLinking {
|
|
inhumeAddresses = append(inhumeAddresses, linkingAddress)
|
|
}
|
|
|
|
var inhumePrm InhumePrm
|
|
var tombAddress oid.Address
|
|
tombAddress.SetContainer(cnr)
|
|
tombAddress.SetObject(tombstoneID)
|
|
inhumePrm.SetAddresses(inhumeAddresses...)
|
|
inhumePrm.SetTombstoneAddress(tombAddress)
|
|
_, err = db.Inhume(context.Background(), inhumePrm)
|
|
require.NoError(t, err)
|
|
|
|
getPrm.SetAddress(ecParentAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
|
|
|
getPrm.SetAddress(splitParentAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
|
|
|
if withLinking {
|
|
getPrm.SetAddress(linkingAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
|
}
|
|
|
|
for _, id := range ecChunks {
|
|
var ecChunkAddress oid.Address
|
|
ecChunkAddress.SetContainer(cnr)
|
|
ecChunkAddress.SetObject(id)
|
|
getPrm.SetAddress(ecChunkAddress)
|
|
_, err = db.Get(context.Background(), getPrm)
|
|
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
|
}
|
|
|
|
// GC finds and deletes split, EC parent and EC chunks
|
|
|
|
parentCount := 2 // split + ec
|
|
if withLinking {
|
|
parentCount = 3
|
|
}
|
|
|
|
var garbageAddresses []oid.Address
|
|
var itPrm GarbageIterationPrm
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
garbageAddresses = append(garbageAddresses, g.Address())
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
require.Equal(t, parentCount+chunksCount, len(garbageAddresses))
|
|
require.True(t, slices.Contains(garbageAddresses, splitParentAddress))
|
|
require.True(t, slices.Contains(garbageAddresses, ecParentAddress))
|
|
if withLinking {
|
|
require.True(t, slices.Contains(garbageAddresses, linkingAddress))
|
|
}
|
|
for _, id := range ecChunks {
|
|
var ecChunkAddress oid.Address
|
|
ecChunkAddress.SetContainer(cnr)
|
|
ecChunkAddress.SetObject(id)
|
|
require.True(t, slices.Contains(garbageAddresses, ecChunkAddress))
|
|
}
|
|
|
|
var deletePrm DeletePrm
|
|
deletePrm.SetAddresses(garbageAddresses...)
|
|
_, err = db.Delete(context.Background(), deletePrm)
|
|
require.NoError(t, err)
|
|
|
|
var garbageStub []oid.Address
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
garbageStub = append(garbageStub, g.Address())
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
require.Equal(t, 0, len(garbageStub))
|
|
|
|
// after tombstone expired GC inhumes tombstone and drops graves
|
|
|
|
var tombstonedObjects []TombstonedObject
|
|
var graveyardIterationPrm GraveyardIterationPrm
|
|
graveyardIterationPrm.SetHandler(func(object TombstonedObject) error {
|
|
tombstonedObjects = append(tombstonedObjects, object)
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGraveyard(context.Background(), graveyardIterationPrm))
|
|
require.True(t, len(tombstonedObjects) == parentCount+chunksCount)
|
|
|
|
_, err = db.InhumeTombstones(context.Background(), tombstonedObjects)
|
|
require.NoError(t, err)
|
|
|
|
// GC finds tombstone as garbage and deletes it
|
|
|
|
garbageAddresses = nil
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
garbageAddresses = append(garbageAddresses, g.Address())
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
require.Equal(t, 1, len(garbageAddresses))
|
|
require.Equal(t, tombstoneID, garbageAddresses[0].Object())
|
|
|
|
deletePrm.SetAddresses(garbageAddresses...)
|
|
_, err = db.Delete(context.Background(), deletePrm)
|
|
require.NoError(t, err)
|
|
|
|
// no more objects should left as garbage
|
|
|
|
itPrm.SetHandler(func(g GarbageObject) error {
|
|
require.FailNow(t, "no garbage objects should left")
|
|
return nil
|
|
})
|
|
require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm))
|
|
|
|
require.NoError(t, db.boltDB.View(testVerifyNoObjectDataLeft))
|
|
|
|
require.NoError(t, testCountersAreZero(db, cnr))
|
|
}
|
|
|
|
func testVerifyNoObjectDataLeft(tx *bbolt.Tx) error {
|
|
return tx.ForEach(func(name []byte, b *bbolt.Bucket) error {
|
|
if bytes.Equal(name, shardInfoBucket) ||
|
|
bytes.Equal(name, containerCounterBucketName) ||
|
|
bytes.Equal(name, containerVolumeBucketName) ||
|
|
bytes.Equal(name, expEpochToObjectBucketName) {
|
|
return nil
|
|
}
|
|
return testBucketEmpty(name, b)
|
|
})
|
|
}
|
|
|
|
func testBucketEmpty(name []byte, b *bbolt.Bucket) error {
|
|
err := b.ForEach(func(k, v []byte) error {
|
|
if len(v) > 0 {
|
|
return fmt.Errorf("bucket %v is not empty", name)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return b.ForEachBucket(func(k []byte) error {
|
|
return testBucketEmpty(k, b.Bucket(k))
|
|
})
|
|
}
|
|
|
|
func testCountersAreZero(db *DB, cnr cid.ID) error {
|
|
c, err := db.ContainerCount(context.Background(), cnr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !c.IsZero() {
|
|
return fmt.Errorf("container %s has non zero counters", cnr.EncodeToString())
|
|
}
|
|
s, err := db.ContainerSize(cnr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if s != 0 {
|
|
return fmt.Errorf("container %s has non zero size", cnr.EncodeToString())
|
|
}
|
|
return nil
|
|
}
|