forked from TrueCloudLab/frostfs-node
[#378] metabase: Implement iterator over metabase graveyard
Implement `DB.IterateOverGraveyard` method that iterates over all graves and passes passes their descriptors (new type `Grave`) to handler (new type `GraveHandler`). `Grave` currently have buried object address and garbage flag. Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
parent
3d7d074af4
commit
d72d37a322
3 changed files with 142 additions and 0 deletions
70
pkg/local_object_storage/metabase/graveyard.go
Normal file
70
pkg/local_object_storage/metabase/graveyard.go
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
package meta
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"go.etcd.io/bbolt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Grave represents descriptor of DB's graveyard item.
|
||||||
|
type Grave struct {
|
||||||
|
gcMark bool
|
||||||
|
|
||||||
|
addr *object.Address
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithGCMark returns true if grave marked for GC to be removed.
|
||||||
|
func (g *Grave) WithGCMark() bool {
|
||||||
|
return g.gcMark
|
||||||
|
}
|
||||||
|
|
||||||
|
// Address returns buried object address.
|
||||||
|
func (g *Grave) Address() *object.Address {
|
||||||
|
return g.addr
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraveHandler is a Grave handling function.
|
||||||
|
type GraveHandler func(*Grave) error
|
||||||
|
|
||||||
|
// IterateOverGraveyard iterates over all graves in DB.
|
||||||
|
//
|
||||||
|
// Returns errors of h directly.
|
||||||
|
func (db *DB) IterateOverGraveyard(h GraveHandler) error {
|
||||||
|
return db.boltDB.View(func(tx *bbolt.Tx) error {
|
||||||
|
return db.iterateOverGraveyard(tx, h)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) iterateOverGraveyard(tx *bbolt.Tx, h GraveHandler) error {
|
||||||
|
// get graveyard bucket
|
||||||
|
bktGraveyard := tx.Bucket(graveyardBucketName)
|
||||||
|
if bktGraveyard == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// iterate over all graves
|
||||||
|
return bktGraveyard.ForEach(func(k, v []byte) error {
|
||||||
|
// parse Grave
|
||||||
|
g, err := graveFromKV(k, v)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "could not parse Grave")
|
||||||
|
}
|
||||||
|
|
||||||
|
// handler Grave
|
||||||
|
return h(g)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func graveFromKV(k, v []byte) (*Grave, error) {
|
||||||
|
addr, err := addressFromKey(k)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "could not parse address")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Grave{
|
||||||
|
gcMark: bytes.Equal(v, []byte(inhumeGCMarkValue)),
|
||||||
|
addr: addr,
|
||||||
|
}, nil
|
||||||
|
}
|
66
pkg/local_object_storage/metabase/graveyard_test.go
Normal file
66
pkg/local_object_storage/metabase/graveyard_test.go
Normal file
|
@ -0,0 +1,66 @@
|
||||||
|
package meta_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDB_IterateOverGraveyard(t *testing.T) {
|
||||||
|
db := newDB(t)
|
||||||
|
defer releaseDB(db)
|
||||||
|
|
||||||
|
// generate and put 2 objects
|
||||||
|
obj1 := generateRawObject(t)
|
||||||
|
obj2 := generateRawObject(t)
|
||||||
|
|
||||||
|
var err error
|
||||||
|
|
||||||
|
err = putBig(db, obj1.Object())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = putBig(db, obj2.Object())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
inhumePrm := new(meta.InhumePrm)
|
||||||
|
|
||||||
|
// inhume with tombstone
|
||||||
|
addrTombstone := generateAddress()
|
||||||
|
|
||||||
|
_, err = db.Inhume(inhumePrm.
|
||||||
|
WithAddresses(obj1.Object().Address()).
|
||||||
|
WithTombstoneAddress(addrTombstone),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// inhume with GC mark
|
||||||
|
_, err = db.Inhume(inhumePrm.
|
||||||
|
WithAddresses(obj2.Object().Address()).
|
||||||
|
WithGCMark(),
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
counterAll int
|
||||||
|
buriedTS, buriedGC []*object.Address
|
||||||
|
)
|
||||||
|
|
||||||
|
err = db.IterateOverGraveyard(func(g *meta.Grave) error {
|
||||||
|
if g.WithGCMark() {
|
||||||
|
buriedGC = append(buriedGC, g.Address())
|
||||||
|
} else {
|
||||||
|
buriedTS = append(buriedTS, g.Address())
|
||||||
|
}
|
||||||
|
|
||||||
|
counterAll++
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.Equal(t, 2, counterAll)
|
||||||
|
require.Equal(t, []*object.Address{obj1.Object().Address()}, buriedTS)
|
||||||
|
require.Equal(t, []*object.Address{obj2.Object().Address()}, buriedGC)
|
||||||
|
}
|
|
@ -97,6 +97,12 @@ func addressKey(addr *object.Address) []byte {
|
||||||
return []byte(addr.String())
|
return []byte(addr.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parses object address formed by addressKey.
|
||||||
|
func addressFromKey(k []byte) (*object.Address, error) {
|
||||||
|
a := object.NewAddress()
|
||||||
|
return a, a.Parse(string(k))
|
||||||
|
}
|
||||||
|
|
||||||
// objectKey returns key for K-V tables when key is an object id.
|
// objectKey returns key for K-V tables when key is an object id.
|
||||||
func objectKey(oid *object.ID) []byte {
|
func objectKey(oid *object.ID) []byte {
|
||||||
return []byte(oid.String())
|
return []byte(oid.String())
|
||||||
|
|
Loading…
Reference in a new issue