dkirillov/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#237] metabase: Structure parameters and results of all operations

Browse source 
All parameters and resulting values of all metabase operations are
structured in new types. The most popular scenarios for using operations are
moved to auxiliary functions.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2020-12-08 12:56:14 +03:00 committed by Alex Vanin
parent a875d80491
commit 590745204c
28 changed files with 482 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
pkg/local_object_storage/metabase/cleanup.go
View file

@ -6,8 +6,15 @@ import (
"go.etcd.io/bbolt"
)
func (db *DB) CleanUp() error {
return db.boltDB.Update(func(tx *bbolt.Tx) error {
// CleanUpPrm groups the parameters of CleanUp operation.
type CleanUpPrm struct{}
// CleanUpRes groups resulting values of CleanUp operation.
type CleanUpRes struct{}
// CleanUp removes empty buckets from metabase.
func (db *DB) CleanUp(prm *CleanUpPrm) (res *CleanUpRes, err error) {
err = db.boltDB.Update(func(tx *bbolt.Tx) error {
return tx.ForEach(func(name []byte, b *bbolt.Bucket) error {
switch {
case isFKBTBucket(name):
@ -21,6 +28,8 @@ func (db *DB) CleanUp() error {
return nil
})
})
return
}
func isFKBTBucket(name []byte) bool {

2
pkg/local_object_storage/metabase/containers_test.go
View file

@ -19,7 +19,7 @@ func TestDB_Containers(t *testing.T) {
cids[obj.ContainerID().String()] = 0
err := db.Put(obj.Object(), nil)
err := putBig(db, obj.Object())
require.NoError(t, err)
}

7
pkg/local_object_storage/metabase/db_test.go
View file

@ -17,8 +17,13 @@ import (
"github.com/stretchr/testify/require"
)
// saves "big" object in DB.
func putBig(db *meta.DB, obj *object.Object) error {
return meta.Put(db, obj, nil)
}
func testSelect(t *testing.T, db *meta.DB, fs objectSDK.SearchFilters, exp ...*objectSDK.Address) {
res, err := db.Select(fs)
res, err := meta.Select(db, fs)
require.NoError(t, err)
require.Len(t, res, len(exp))

33
pkg/local_object_storage/metabase/delete.go
View file

@ -10,13 +10,38 @@ import (
"go.etcd.io/bbolt"
)
// DeletePrm groups the parameters of Delete operation.
type DeletePrm struct {
addrs []*objectSDK.Address
}
// DeleteRes groups resulting values of Delete operation.
type DeleteRes struct{}
var ErrVirtualObject = errors.New("do not remove virtual object directly")
// WithAddresses is a Delete option to set the addresses of the objects to delete.
//
// Option is required.
func (p *DeletePrm) WithAddresses(addrs ...*objectSDK.Address) *DeletePrm {
if p != nil {
p.addrs = addrs
}
return p
}
// Delete removes objects from DB.
func Delete(db *DB, addrs ...*objectSDK.Address) error {
_, err := db.Delete(new(DeletePrm).WithAddresses(addrs...))
return err
}
// DeleteObjects marks list of objects as deleted.
func (db *DB) Delete(lst ...*objectSDK.Address) error {
return db.boltDB.Update(func(tx *bbolt.Tx) error {
for i := range lst {
err := db.delete(tx, lst[i], false)
func (db *DB) Delete(prm *DeletePrm) (*DeleteRes, error) {
return new(DeleteRes), db.boltDB.Update(func(tx *bbolt.Tx) error {
for i := range prm.addrs {
err := db.delete(tx, prm.addrs[i], false)
if err != nil {
return err // maybe log and continue?
}

19
pkg/local_object_storage/metabase/delete_test.go
View file

@ -3,6 +3,7 @@ package meta_test
import (
"testing"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -19,42 +20,42 @@ func TestDB_Delete(t *testing.T) {
child.SetParentID(parent.ID())
// put object with parent
err := db.Put(child.Object(), nil)
err := putBig(db, child.Object())
require.NoError(t, err)
// fill ToMoveIt index
err = db.ToMoveIt(child.Object().Address())
err = meta.ToMoveIt(db, child.Object().Address())
require.NoError(t, err)
// check if Movable list is not empty
l, err := db.Movable()
l, err := meta.Movable(db)
require.NoError(t, err)
require.Len(t, l, 1)
// inhume parent and child so they will be on graveyard
ts := generateRawObjectWithCID(t, cid)
err = db.Inhume(child.Object().Address(), ts.Object().Address())
err = meta.Inhume(db, child.Object().Address(), ts.Object().Address())
require.NoError(t, err)
err = db.Inhume(child.Object().Address(), ts.Object().Address())
err = meta.Inhume(db, child.Object().Address(), ts.Object().Address())
require.NoError(t, err)
// delete object
err = db.Delete(child.Object().Address())
err = meta.Delete(db, child.Object().Address())
require.NoError(t, err)
// check if there is no data in Movable index
l, err = db.Movable()
l, err = meta.Movable(db)
require.NoError(t, err)
require.Len(t, l, 0)
// check if they removed from graveyard
ok, err := db.Exists(child.Object().Address())
ok, err := meta.Exists(db, child.Object().Address())
require.NoError(t, err)
require.False(t, ok)
ok, err = db.Exists(parent.Object().Address())
ok, err = meta.Exists(db, parent.Object().Address())
require.NoError(t, err)
require.False(t, ok)
}

42
pkg/local_object_storage/metabase/exists.go
View file

@ -9,18 +9,54 @@ import (
"go.etcd.io/bbolt"
)
// ExistsPrm groups the parameters of Exists operation.
type ExistsPrm struct {
addr *objectSDK.Address
}
// ExistsRes groups resulting values of Exists operation.
type ExistsRes struct {
exists bool
}
var ErrLackSplitInfo = errors.New("no split info on parent object")
// WithAddress is an Exists option to set object checked for existence.
func (p *ExistsPrm) WithAddress(addr *objectSDK.Address) *ExistsPrm {
if p != nil {
p.addr = addr
}
return p
}
// Exists returns the fact that the object is in the metabase.
func (p *ExistsRes) Exists() bool {
return p.exists
}
// Exists checks if object is presented in DB.
func Exists(db *DB, addr *objectSDK.Address) (bool, error) {
r, err := db.Exists(new(ExistsPrm).WithAddress(addr))
if err != nil {
return false, err
}
return r.Exists(), nil
}
// Exists returns ErrAlreadyRemoved if addr was marked as removed. Otherwise it
// returns true if addr is in primary index or false if it is not.
func (db *DB) Exists(addr *objectSDK.Address) (exists bool, err error) {
func (db *DB) Exists(prm *ExistsPrm) (res *ExistsRes, err error) {
res = new(ExistsRes)
err = db.boltDB.View(func(tx *bbolt.Tx) error {
exists, err = db.exists(tx, addr)
res.exists, err = db.exists(tx, prm.addr)
return err
})
return exists, err
return
}
func (db *DB) exists(tx *bbolt.Tx, addr *objectSDK.Address) (exists bool, err error) {

31
pkg/local_object_storage/metabase/exists_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -14,17 +15,17 @@ func TestDB_Exists(t *testing.T) {
t.Run("no object", func(t *testing.T) {
nonExist := generateRawObject(t)
exists, err := db.Exists(nonExist.Object().Address())
exists, err := meta.Exists(db, nonExist.Object().Address())
require.NoError(t, err)
require.False(t, exists)
})
t.Run("regular object", func(t *testing.T) {
regular := generateRawObject(t)
err := db.Put(regular.Object(), nil)
err := putBig(db, regular.Object())
require.NoError(t, err)
exists, err := db.Exists(regular.Object().Address())
exists, err := meta.Exists(db, regular.Object().Address())
require.NoError(t, err)
require.True(t, exists)
})
@ -33,10 +34,10 @@ func TestDB_Exists(t *testing.T) {
ts := generateRawObject(t)
ts.SetType(objectSDK.TypeTombstone)
err := db.Put(ts.Object(), nil)
err := putBig(db, ts.Object())
require.NoError(t, err)
exists, err := db.Exists(ts.Object().Address())
exists, err := meta.Exists(db, ts.Object().Address())
require.NoError(t, err)
require.True(t, exists)
})
@ -45,10 +46,10 @@ func TestDB_Exists(t *testing.T) {
sg := generateRawObject(t)
sg.SetType(objectSDK.TypeStorageGroup)
err := db.Put(sg.Object(), nil)
err := putBig(db, sg.Object())
require.NoError(t, err)
exists, err := db.Exists(sg.Object().Address())
exists, err := meta.Exists(db, sg.Object().Address())
require.NoError(t, err)
require.True(t, exists)
})
@ -61,10 +62,10 @@ func TestDB_Exists(t *testing.T) {
child.SetParent(parent.Object().SDK())
child.SetParentID(parent.ID())
err := db.Put(child.Object(), nil)
err := putBig(db, child.Object())
require.NoError(t, err)
_, err = db.Exists(parent.Object().Address())
_, err = meta.Exists(db, parent.Object().Address())
var expectedErr *objectSDK.SplitInfoError
require.True(t, errors.As(err, &expectedErr))
@ -89,13 +90,13 @@ func TestDB_Exists(t *testing.T) {
link.SetSplitID(splitID)
t.Run("direct order", func(t *testing.T) {
err := db.Put(child.Object(), nil)
err := putBig(db, child.Object())
require.NoError(t, err)
err = db.Put(link.Object(), nil)
err = putBig(db, link.Object())
require.NoError(t, err)
_, err = db.Exists(parent.Object().Address())
_, err = meta.Exists(db, parent.Object().Address())
require.Error(t, err)
si, ok := err.(*objectSDK.SplitInfoError)
@ -107,13 +108,13 @@ func TestDB_Exists(t *testing.T) {
})
t.Run("reverse order", func(t *testing.T) {
err := db.Put(link.Object(), nil)
err := meta.Put(db, link.Object(), nil)
require.NoError(t, err)
err = db.Put(child.Object(), nil)
err = putBig(db, child.Object())
require.NoError(t, err)
_, err = db.Exists(parent.Object().Address())
_, err = meta.Exists(db, parent.Object().Address())
require.Error(t, err)
si, ok := err.(*objectSDK.SplitInfoError)

44
pkg/local_object_storage/metabase/get.go
View file

@ -9,15 +9,53 @@ import (
"go.etcd.io/bbolt"
)
// GetPrm groups the parameters of Get operation.
type GetPrm struct {
addr *objectSDK.Address
}
// GetRes groups resulting values of Get operation.
type GetRes struct {
hdr *object.Object
}
// WithAddress is a Get option to set the address of the requested object.
//
// Option is required.
func (p *GetPrm) WithAddress(addr *objectSDK.Address) *GetPrm {
if p != nil {
p.addr = addr
}
return p
}
// Header returns the requested object header.
func (r *GetRes) Header() *object.Object {
return r.hdr
}
// Get read the object from DB.
func Get(db *DB, addr *objectSDK.Address) (*object.Object, error) {
r, err := db.Get(new(GetPrm).WithAddress(addr))
if err != nil {
return nil, err
}
return r.Header(), nil
}
// Get returns object header for specified address.
func (db *DB) Get(addr *objectSDK.Address) (obj *object.Object, err error) {
func (db *DB) Get(prm *GetPrm) (res *GetRes, err error) {
res = new(GetRes)
err = db.boltDB.View(func(tx *bbolt.Tx) error {
obj, err = db.get(tx, addr, true)
res.hdr, err = db.get(tx, prm.addr, true)
return err
})
return obj, err
return
}
func (db *DB) get(tx *bbolt.Tx, addr *objectSDK.Address, checkGraveyard bool) (*object.Object, error) {

21
pkg/local_object_storage/metabase/get_test.go
View file

@ -6,6 +6,7 @@ import (
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -20,15 +21,15 @@ func TestDB_Get(t *testing.T) {
addAttribute(raw, "foo", "bar")
t.Run("object not found", func(t *testing.T) {
_, err := db.Get(raw.Object().Address())
_, err := meta.Get(db, raw.Object().Address())
require.Error(t, err)
})
t.Run("put regular object", func(t *testing.T) {
err := db.Put(raw.Object(), nil)
err := putBig(db, raw.Object())
require.NoError(t, err)
newObj, err := db.Get(raw.Object().Address())
newObj, err := meta.Get(db, raw.Object().Address())
require.NoError(t, err)
require.Equal(t, raw.Object(), newObj)
})
@ -37,10 +38,10 @@ func TestDB_Get(t *testing.T) {
raw.SetType(objectSDK.TypeTombstone)
raw.SetID(testOID())
err := db.Put(raw.Object(), nil)
err := putBig(db, raw.Object())
require.NoError(t, err)
newObj, err := db.Get(raw.Object().Address())
newObj, err := meta.Get(db, raw.Object().Address())
require.NoError(t, err)
require.Equal(t, raw.Object(), newObj)
})
@ -49,10 +50,10 @@ func TestDB_Get(t *testing.T) {
raw.SetType(objectSDK.TypeStorageGroup)
raw.SetID(testOID())
err := db.Put(raw.Object(), nil)
err := putBig(db, raw.Object())
require.NoError(t, err)
newObj, err := db.Get(raw.Object().Address())
newObj, err := meta.Get(db, raw.Object().Address())
require.NoError(t, err)
require.Equal(t, raw.Object(), newObj)
})
@ -66,14 +67,14 @@ func TestDB_Get(t *testing.T) {
child.SetParent(parent.Object().SDK())
child.SetParentID(parent.ID())
err := db.Put(child.Object(), nil)
err := putBig(db, child.Object())
require.NoError(t, err)
newParent, err := db.Get(parent.Object().Address())
newParent, err := meta.Get(db, parent.Object().Address())
require.NoError(t, err)
require.True(t, binaryEqual(parent.Object(), newParent))
newChild, err := db.Get(child.Object().Address())
newChild, err := meta.Get(db, child.Object().Address())
require.NoError(t, err)
require.True(t, binaryEqual(child.Object(), newChild))
})

44
pkg/local_object_storage/metabase/inhume.go
View file

@ -5,15 +5,53 @@ import (
"go.etcd.io/bbolt"
)
// InhumePrm encapsulates parameters for Inhume operation.
type InhumePrm struct {
target, tomb *objectSDK.Address
}
// InhumeRes encapsulates results of Inhume operation.
type InhumeRes struct{}
// WithAddress sets object address that should be inhumed.
func (p *InhumePrm) WithAddress(addr *objectSDK.Address) *InhumePrm {
if p != nil {
p.target = addr
}
return p
}
// WithTombstoneAddress sets tombstone address as the reason for inhume operation.
func (p *InhumePrm) WithTombstoneAddress(addr *objectSDK.Address) *InhumePrm {
if p != nil {
p.tomb = addr
}
return p
}
// Inhume inhumes the object by specified address.
func Inhume(db *DB, target, tomb *objectSDK.Address) error {
_, err := db.Inhume(new(InhumePrm).
WithAddress(target).
WithTombstoneAddress(tomb),
)
return err
}
// Inhume marks objects as removed but not removes it from metabase.
func (db *DB) Inhume(target, tombstone *objectSDK.Address) error {
return db.boltDB.Update(func(tx *bbolt.Tx) error {
func (db *DB) Inhume(prm *InhumePrm) (res *InhumeRes, err error) {
err = db.boltDB.Update(func(tx *bbolt.Tx) error {
graveyard, err := tx.CreateBucketIfNotExists(graveyardBucketName)
if err != nil {
return err
}
// consider checking if target is already in graveyard?
return graveyard.Put(addressKey(target), addressKey(tombstone))
return graveyard.Put(addressKey(prm.target), addressKey(prm.tomb))
})
return
}

9
pkg/local_object_storage/metabase/inhume_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -16,15 +17,15 @@ func TestDB_Inhume(t *testing.T) {
tombstoneID := generateAddress()
err := db.Put(raw.Object(), nil)
err := putBig(db, raw.Object())
require.NoError(t, err)
err = db.Inhume(raw.Object().Address(), tombstoneID)
err = meta.Inhume(db, raw.Object().Address(), tombstoneID)
require.NoError(t, err)
_, err = db.Exists(raw.Object().Address())
_, err = meta.Exists(db, raw.Object().Address())
require.EqualError(t, err, object.ErrAlreadyRemoved.Error())
_, err = db.Get(raw.Object().Address())
_, err = meta.Get(db, raw.Object().Address())
require.EqualError(t, err, object.ErrAlreadyRemoved.Error())
}

91
pkg/local_object_storage/metabase/movable.go
View file

@ -7,33 +7,106 @@ import (
"go.etcd.io/bbolt"
)
// ToMoveItPrm groups the parameters of ToMoveIt operation.
type ToMoveItPrm struct {
addr *objectSDK.Address
}
// ToMoveItRes groups resulting values of ToMoveIt operation.
type ToMoveItRes struct{}
// WithAddress sets address of the object to move into another shard.
func (p *ToMoveItPrm) WithAddress(addr *objectSDK.Address) *ToMoveItPrm {
if p != nil {
p.addr = addr
}
return p
}
// DoNotMovePrm groups the parameters of DoNotMove operation.
type DoNotMovePrm struct {
addr *objectSDK.Address
}
// DoNotMoveRes groups resulting values of DoNotMove operation.
type DoNotMoveRes struct{}
// WithAddress sets address of the object to prevent moving into another shard.
func (p *DoNotMovePrm) WithAddress(addr *objectSDK.Address) *DoNotMovePrm {
if p != nil {
p.addr = addr
}
return p
}
// MovablePrm groups the parameters of Movable operation.
type MovablePrm struct{}
// MovableRes groups resulting values of Movable operation.
type MovableRes struct {
addrList []*objectSDK.Address
}
// WithAddress sets address of the object to prevent moving into another shard.
func (p *MovableRes) AddressList() []*objectSDK.Address {
return p.addrList
}
// ToMoveIt marks object to move it into another shard.
func ToMoveIt(db *DB, addr *objectSDK.Address) error {
_, err := db.ToMoveIt(new(ToMoveItPrm).WithAddress(addr))
return err
}
// ToMoveIt marks objects to move it into another shard. This useful for
// faster HRW fetching.
func (db *DB) ToMoveIt(addr *objectSDK.Address) error {
return db.boltDB.Update(func(tx *bbolt.Tx) error {
func (db *DB) ToMoveIt(prm *ToMoveItPrm) (res *ToMoveItRes, err error) {
err = db.boltDB.Update(func(tx *bbolt.Tx) error {
toMoveIt, err := tx.CreateBucketIfNotExists(toMoveItBucketName)
if err != nil {
return err
}
return toMoveIt.Put(addressKey(addr), zeroValue)
return toMoveIt.Put(addressKey(prm.addr), zeroValue)
})
return
}
// DoNotMove prevents the object to be moved into another shard.
func DoNotMove(db *DB, addr *objectSDK.Address) error {
_, err := db.DoNotMove(new(DoNotMovePrm).WithAddress(addr))
return err
}
// DoNotMove removes `MoveIt` mark from the object.
func (db *DB) DoNotMove(addr *objectSDK.Address) error {
return db.boltDB.Update(func(tx *bbolt.Tx) error {
func (db *DB) DoNotMove(prm *DoNotMovePrm) (res *DoNotMoveRes, err error) {
err = db.boltDB.Update(func(tx *bbolt.Tx) error {
toMoveIt := tx.Bucket(toMoveItBucketName)
if toMoveIt == nil {
return nil
}
return toMoveIt.Delete(addressKey(addr))
return toMoveIt.Delete(addressKey(prm.addr))
})
return
}
// Movable returns all movable objects of DB.
func Movable(db *DB) ([]*objectSDK.Address, error) {
r, err := db.Movable(new(MovablePrm))
if err != nil {
return nil, err
}
return r.AddressList(), nil
}
// Movable returns list of marked objects to move into other shard.
func (db *DB) Movable() ([]*objectSDK.Address, error) {
func (db *DB) Movable(prm *MovablePrm) (*MovableRes, error) {
var strAddrs []string
err := db.boltDB.View(func(tx *bbolt.Tx) error {
@ -69,5 +142,7 @@ func (db *DB) Movable() ([]*objectSDK.Address, error) {
addrs = append(addrs, addr)
}
return addrs, nil
return &MovableRes{
addrList: addrs,
}, nil
}

19
pkg/local_object_storage/metabase/movable_test.go
View file

@ -3,6 +3,7 @@ package meta_test
import (
"testing"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -14,42 +15,42 @@ func TestDB_Movable(t *testing.T) {
raw2 := generateRawObject(t)
// put two objects in metabase
err := db.Put(raw1.Object(), nil)
err := putBig(db, raw1.Object())
require.NoError(t, err)
err = db.Put(raw2.Object(), nil)
err = putBig(db, raw2.Object())
require.NoError(t, err)
// check if toMoveIt index empty
toMoveList, err := db.Movable()
toMoveList, err := meta.Movable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 0)
// mark to move object2
err = db.ToMoveIt(raw2.Object().Address())
err = meta.ToMoveIt(db, raw2.Object().Address())
require.NoError(t, err)
// check if toMoveIt index contains address of object 2
toMoveList, err = db.Movable()
toMoveList, err = meta.Movable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 1)
require.Contains(t, toMoveList, raw2.Object().Address())
// remove from toMoveIt index non existing address
err = db.DoNotMove(raw1.Object().Address())
err = meta.DoNotMove(db, raw1.Object().Address())
require.NoError(t, err)
// check if toMoveIt index hasn't changed
toMoveList, err = db.Movable()
toMoveList, err = meta.Movable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 1)
// remove from toMoveIt index existing address
err = db.DoNotMove(raw2.Object().Address())
err = meta.DoNotMove(db, raw2.Object().Address())
require.NoError(t, err)
// check if toMoveIt index is empty now
toMoveList, err = db.Movable()
toMoveList, err = meta.Movable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 0)
}

46
pkg/local_object_storage/metabase/put.go
View file

@ -18,6 +18,34 @@ type (
}
)
// PutPrm groups the parameters of Put operation.
type PutPrm struct {
obj *object.Object
id *blobovnicza.ID
}
// PutRes groups resulting values of Put operation.
type PutRes struct{}
// WithObject is a Put option to set object to save.
func (p *PutPrm) WithObject(obj *object.Object) *PutPrm {
if p != nil {
p.obj = obj
}
return p
}
// WithBlobovniczaID is a Put option to set blobovnicza ID to save.
func (p *PutPrm) WithBlobovniczaID(id *blobovnicza.ID) *PutPrm {
if p != nil {
p.id = id
}
return p
}
var (
ErrUnknownObjectType = errors.New("unknown object type")
ErrIncorrectBlobovniczaUpdate = errors.New("updating blobovnicza id on object without it")
@ -25,12 +53,24 @@ var (
ErrIncorrectRootObject = errors.New("invalid root object")
)
// Put saves the object in DB.
func Put(db *DB, obj *object.Object, id *blobovnicza.ID) error {
_, err := db.Put(new(PutPrm).
WithObject(obj).
WithBlobovniczaID(id),
)
return err
}
// Put saves object header in metabase. Object payload expected to be cut.
// Big objects have nil blobovniczaID.
func (db *DB) Put(obj *object.Object, id *blobovnicza.ID) error {
return db.boltDB.Update(func(tx *bbolt.Tx) error {
return db.put(tx, obj, id, nil)
func (db *DB) Put(prm *PutPrm) (res *PutRes, err error) {
err = db.boltDB.Update(func(tx *bbolt.Tx) error {
return db.put(tx, prm.obj, prm.id, nil)
})
return
}
func (db *DB) put(tx *bbolt.Tx, obj *object.Object, id *blobovnicza.ID, si *objectSDK.SplitInfo) error {

15
pkg/local_object_storage/metabase/put_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -15,34 +16,34 @@ func TestDB_PutBlobovnicaUpdate(t *testing.T) {
blobovniczaID := blobovnicza.ID{1, 2, 3, 4}
// put one object with blobovniczaID
err := db.Put(raw1.Object(), &blobovniczaID)
err := meta.Put(db, raw1.Object(), &blobovniczaID)
require.NoError(t, err)
fetchedBlobovniczaID, err := db.IsSmall(raw1.Object().Address())
fetchedBlobovniczaID, err := meta.IsSmall(db, raw1.Object().Address())
require.NoError(t, err)
require.Equal(t, &blobovniczaID, fetchedBlobovniczaID)
t.Run("update blobovniczaID", func(t *testing.T) {
newID := blobovnicza.ID{5, 6, 7, 8}
err := db.Put(raw1.Object(), &newID)
err := meta.Put(db, raw1.Object(), &newID)
require.NoError(t, err)
fetchedBlobovniczaID, err := db.IsSmall(raw1.Object().Address())
fetchedBlobovniczaID, err := meta.IsSmall(db, raw1.Object().Address())
require.NoError(t, err)
require.Equal(t, &newID, fetchedBlobovniczaID)
})
t.Run("update blobovniczaID on bad object", func(t *testing.T) {
raw2 := generateRawObject(t)
err := db.Put(raw2.Object(), nil)
err := putBig(db, raw2.Object())
require.NoError(t, err)
fetchedBlobovniczaID, err := db.IsSmall(raw2.Object().Address())
fetchedBlobovniczaID, err := meta.IsSmall(db, raw2.Object().Address())
require.NoError(t, err)
require.Nil(t, fetchedBlobovniczaID)
err = db.Put(raw2.Object(), &blobovniczaID)
err = meta.Put(db, raw2.Object(), &blobovniczaID)
require.Error(t, err)
})
}

40
pkg/local_object_storage/metabase/select.go
View file

@ -24,12 +24,48 @@ type (
}
)
// SelectPrm groups the parameters of Select operation.
type SelectPrm struct {
filters object.SearchFilters
}
// SelectRes groups resulting values of Select operation.
type SelectRes struct {
addrList []*object.Address
}
// WithFilters is a Select option to set the object filters.
func (p *SelectPrm) WithFilters(fs object.SearchFilters) *SelectPrm {
if p != nil {
p.filters = fs
}
return p
}
// AddressList returns list of addresses of the selected objects.
func (r *SelectRes) AddressList() []*object.Address {
return r.addrList
}
var ErrContainerNotInQuery = errors.New("search query does not contain container id filter")
// Select selects the objects from DB with filtering.
func Select(db *DB, fs object.SearchFilters) ([]*object.Address, error) {
r, err := db.Select(new(SelectPrm).WithFilters(fs))
if err != nil {
return nil, err
}
return r.AddressList(), nil
}
// Select returns list of addresses of objects that match search filters.
func (db *DB) Select(fs object.SearchFilters) (res []*object.Address, err error) {
func (db *DB) Select(prm *SelectPrm) (res *SelectRes, err error) {
res = new(SelectRes)
err = db.boltDB.View(func(tx *bbolt.Tx) error {
res, err = db.selectObjects(tx, fs)
res.addrList, err = db.selectObjects(tx, prm.filters)
return err
})

45
pkg/local_object_storage/metabase/select_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/container"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
v2object "github.com/nspcc-dev/neofs-api-go/v2/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -21,20 +22,20 @@ func TestDB_SelectUserAttributes(t *testing.T) {
addAttribute(raw1, "foo", "bar")
addAttribute(raw1, "x", "y")
err := db.Put(raw1.Object(), nil)
err := putBig(db, raw1.Object())
require.NoError(t, err)
raw2 := generateRawObjectWithCID(t, cid)
addAttribute(raw2, "foo", "bar")
addAttribute(raw2, "x", "z")
err = db.Put(raw2.Object(), nil)
err = putBig(db, raw2.Object())
require.NoError(t, err)
raw3 := generateRawObjectWithCID(t, cid)
addAttribute(raw3, "a", "b")
err = db.Put(raw3.Object(), nil)
err = putBig(db, raw3.Object())
require.NoError(t, err)
fs := generateSearchFilter(cid)
@ -73,22 +74,22 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
// prepare
small := generateRawObjectWithCID(t, cid)
err := db.Put(small.Object(), nil)
err := putBig(db, small.Object())
require.NoError(t, err)
ts := generateRawObjectWithCID(t, cid)
ts.SetType(objectSDK.TypeTombstone)
err = db.Put(ts.Object(), nil)
err = putBig(db, ts.Object())
require.NoError(t, err)
sg := generateRawObjectWithCID(t, cid)
sg.SetType(objectSDK.TypeStorageGroup)
err = db.Put(sg.Object(), nil)
err = putBig(db, sg.Object())
require.NoError(t, err)
leftChild := generateRawObjectWithCID(t, cid)
leftChild.InitRelations()
err = db.Put(leftChild.Object(), nil)
err = putBig(db, leftChild.Object())
require.NoError(t, err)
parent := generateRawObjectWithCID(t, cid)
@ -96,7 +97,7 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
rightChild := generateRawObjectWithCID(t, cid)
rightChild.SetParent(parent.Object().SDK())
rightChild.SetParentID(parent.ID())
err = db.Put(rightChild.Object(), nil)
err = putBig(db, rightChild.Object())
require.NoError(t, err)
link := generateRawObjectWithCID(t, cid)
@ -104,7 +105,7 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
link.SetParentID(parent.ID())
link.SetChildren(leftChild.ID(), rightChild.ID())
err = db.Put(link.Object(), nil)
err = putBig(db, link.Object())
require.NoError(t, err)
t.Run("root objects", func(t *testing.T) {
@ -186,11 +187,11 @@ func TestDB_SelectInhume(t *testing.T) {
cid := testCID()
raw1 := generateRawObjectWithCID(t, cid)
err := db.Put(raw1.Object(), nil)
err := putBig(db, raw1.Object())
require.NoError(t, err)
raw2 := generateRawObjectWithCID(t, cid)
err = db.Put(raw2.Object(), nil)
err = putBig(db, raw2.Object())
require.NoError(t, err)
fs := generateSearchFilter(cid)
@ -203,7 +204,7 @@ func TestDB_SelectInhume(t *testing.T) {
tombstone.SetContainerID(cid)
tombstone.SetObjectID(testOID())
err = db.Inhume(raw2.Object().Address(), tombstone)
err = meta.Inhume(db, raw2.Object().Address(), tombstone)
require.NoError(t, err)
fs = generateSearchFilter(cid)
@ -219,11 +220,11 @@ func TestDB_SelectPayloadHash(t *testing.T) {
cid := testCID()
raw1 := generateRawObjectWithCID(t, cid)
err := db.Put(raw1.Object(), nil)
err := putBig(db, raw1.Object())
require.NoError(t, err)
raw2 := generateRawObjectWithCID(t, cid)
err = db.Put(raw2.Object(), nil)
err = putBig(db, raw2.Object())
require.NoError(t, err)
fs := generateSearchFilter(cid)
@ -251,14 +252,14 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
raw1.SetPayloadSize(10)
raw1.SetCreationEpoch(11)
raw1.SetVersion(v20)
err := db.Put(raw1.Object(), nil)
err := putBig(db, raw1.Object())
require.NoError(t, err)
raw2 := generateRawObjectWithCID(t, cid)
raw2.SetPayloadSize(20)
raw2.SetCreationEpoch(21)
raw2.SetVersion(v21)
err = db.Put(raw2.Object(), nil)
err = putBig(db, raw2.Object())
require.NoError(t, err)
t.Run("object with TZHash", func(t *testing.T) {
@ -312,17 +313,17 @@ func TestDB_SelectObjectID(t *testing.T) {
regular.SetParentID(parent.ID())
regular.SetParent(parent.Object().SDK())
err := db.Put(regular.Object(), nil)
err := putBig(db, regular.Object())
require.NoError(t, err)
ts := generateRawObjectWithCID(t, cid)
ts.SetType(objectSDK.TypeTombstone)
err = db.Put(ts.Object(), nil)
err = putBig(db, ts.Object())
require.NoError(t, err)
sg := generateRawObjectWithCID(t, cid)
sg.SetType(objectSDK.TypeStorageGroup)
err = db.Put(sg.Object(), nil)
err = putBig(db, sg.Object())
require.NoError(t, err)
t.Run("not found objects", func(t *testing.T) {
@ -376,9 +377,9 @@ func TestDB_SelectSplitID(t *testing.T) {
child2.SetSplitID(split1)
child3.SetSplitID(split2)
require.NoError(t, db.Put(child1.Object(), nil))
require.NoError(t, db.Put(child2.Object(), nil))
require.NoError(t, db.Put(child3.Object(), nil))
require.NoError(t, putBig(db, child1.Object()))
require.NoError(t, putBig(db, child2.Object()))
require.NoError(t, putBig(db, child3.Object()))
t.Run("split id", func(t *testing.T) {
fs := generateSearchFilter(cid)

44
pkg/local_object_storage/metabase/small.go
View file

@ -6,18 +6,56 @@ import (
"go.etcd.io/bbolt"
)
// IsSmallPrm groups the parameters of IsSmall operation.
type IsSmallPrm struct {
addr *objectSDK.Address
}
// IsSmallRes groups resulting values of IsSmall operation.
type IsSmallRes struct {
id *blobovnicza.ID
}
// WithAddress is a IsSmall option to set the object address to check.
func (p *IsSmallPrm) WithAddress(addr *objectSDK.Address) *IsSmallPrm {
if p != nil {
p.addr = addr
}
return p
}
// BlobovniczaID returns blobovnicza identifier.
func (r *IsSmallRes) BlobovniczaID() *blobovnicza.ID {
return r.id
}
// IsSmall wraps work with DB.IsSmall method with specified
// address and other parameters by default. Returns only
// the blobovnicza identifier.
func IsSmall(db *DB, addr *objectSDK.Address) (*blobovnicza.ID, error) {
r, err := db.IsSmall(new(IsSmallPrm).WithAddress(addr))
if err != nil {
return nil, err
}
return r.BlobovniczaID(), nil
}
// IsSmall returns blobovniczaID for small objects and nil for big objects.
// Small objects stored in blobovnicza instances. Big objects stored in FS by
// shallow path which is calculated from address and therefore it is not
// indexed in metabase.
func (db *DB) IsSmall(addr *objectSDK.Address) (id *blobovnicza.ID, err error) {
func (db *DB) IsSmall(prm *IsSmallPrm) (res *IsSmallRes, err error) {
res = new(IsSmallRes)
err = db.boltDB.View(func(tx *bbolt.Tx) error {
id, err = db.isSmall(tx, addr)
res.id, err = db.isSmall(tx, prm.addr)
return err
})
return id, err
return
}
func (db *DB) isSmall(tx *bbolt.Tx, addr *objectSDK.Address) (*blobovnicza.ID, error) {

11
pkg/local_object_storage/metabase/small_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/stretchr/testify/require"
)
@ -17,25 +18,25 @@ func TestDB_IsSmall(t *testing.T) {
blobovniczaID := blobovnicza.ID{1, 2, 3, 4}
// check IsSmall from empty database
fetchedBlobovniczaID, err := db.IsSmall(raw1.Object().Address())
fetchedBlobovniczaID, err := meta.IsSmall(db, raw1.Object().Address())
require.NoError(t, err)
require.Nil(t, fetchedBlobovniczaID)
// put one object with blobovniczaID
err = db.Put(raw1.Object(), &blobovniczaID)
err = meta.Put(db, raw1.Object(), &blobovniczaID)
require.NoError(t, err)
// put one object without blobovniczaID
err = db.Put(raw2.Object(), nil)
err = putBig(db, raw2.Object())
require.NoError(t, err)
// check IsSmall for object without blobovniczaID
fetchedBlobovniczaID, err = db.IsSmall(raw2.Object().Address())
fetchedBlobovniczaID, err = meta.IsSmall(db, raw2.Object().Address())
require.NoError(t, err)
require.Nil(t, fetchedBlobovniczaID)
// check IsSmall for object with blobovniczaID
fetchedBlobovniczaID, err = db.IsSmall(raw1.Object().Address())
fetchedBlobovniczaID, err = meta.IsSmall(db, raw1.Object().Address())
require.NoError(t, err)
require.Equal(t, &blobovniczaID, fetchedBlobovniczaID)
}

5
pkg/local_object_storage/shard/delete.go
View file

@ -4,6 +4,7 @@ import (
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"go.uber.org/zap"
)
@ -46,7 +47,7 @@ func (s *Shard) Delete(prm *DeletePrm) (*DeleteRes, error) {
}
}
blobovniczaID, err := s.metaBase.IsSmall(prm.addr[i])
blobovniczaID, err := meta.IsSmall(s.metaBase, prm.addr[i])
if err != nil {
s.log.Debug("can't get blobovniczaID from metabase",
zap.Stringer("object", prm.addr[i]),
@ -60,7 +61,7 @@ func (s *Shard) Delete(prm *DeletePrm) (*DeleteRes, error) {
}
}
err := s.metaBase.Delete(prm.addr...)
err := meta.Delete(s.metaBase, prm.addr...)
if err != nil {
return nil, err // stop on metabase error ?
}

3
pkg/local_object_storage/shard/exists.go
View file

@ -2,6 +2,7 @@ package shard
import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
)
// ExistsPrm groups the parameters of Exists operation.
@ -41,5 +42,5 @@ func (s *Shard) Exists(prm *ExistsPrm) (*ExistsRes, error) {
}
func (s *Shard) objectExists(addr *object.Address) (bool, error) {
return s.metaBase.Exists(addr)
return meta.Exists(s.metaBase, addr)
}

5
pkg/local_object_storage/shard/get.go
View file

@ -7,6 +7,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
)
// storFetcher is a type to unify object fetching mechanism in `fetchObjectData`
@ -103,7 +104,7 @@ func (s *Shard) fetchObjectData(addr *objectSDK.Address, big, small storFetcher)
s.log.Debug("miss in writeCache shallow dir")
}
exists, err := s.metaBase.Exists(addr)
exists, err := meta.Exists(s.metaBase, addr)
if err != nil {
return nil, err
}
@ -112,7 +113,7 @@ func (s *Shard) fetchObjectData(addr *objectSDK.Address, big, small storFetcher)
return nil, object.ErrNotFound
}
blobovniczaID, err := s.metaBase.IsSmall(addr)
blobovniczaID, err := meta.IsSmall(s.metaBase, addr)
if err != nil {
return nil, fmt.Errorf("can't fetch blobovnicza id from metabase: %w", err)
}

3
pkg/local_object_storage/shard/head.go
View file

@ -3,6 +3,7 @@ package shard
import (
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
)
// HeadPrm groups the parameters of Head operation.
@ -35,7 +36,7 @@ func (r *HeadRes) Object() *object.Object {
//
// Returns any error encountered.
func (s *Shard) Head(prm *HeadPrm) (*HeadRes, error) {
head, err := s.metaBase.Get(prm.addr)
head, err := meta.Get(s.metaBase, prm.addr)
return &HeadRes{
obj: head,

3
pkg/local_object_storage/shard/inhume.go
View file

@ -2,6 +2,7 @@ package shard
import (
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"go.uber.org/zap"
)
@ -28,7 +29,7 @@ func (p *InhumePrm) WithTarget(addr, tombstone *objectSDK.Address) *InhumePrm {
// Inhume calls metabase. Inhume method to mark object as removed. It won't be
// removed physically from blobStor and metabase until `Delete` operation.
func (s *Shard) Inhume(prm *InhumePrm) (*InhumeRes, error) {
err := s.metaBase.Inhume(prm.target, prm.tombstone)
err := meta.Inhume(s.metaBase, prm.target, prm.tombstone)
if err != nil {
s.log.Debug("could not mark object to delete in metabase",
zap.String("error", err.Error()),

3
pkg/local_object_storage/shard/list.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"go.uber.org/zap"
)
@ -20,7 +21,7 @@ func (s *Shard) List() (*SelectRes, error) {
filters = filters[:0]
filters.AddObjectContainerIDFilter(object.MatchStringEqual, lst[i])
ids, err := s.metaBase.Select(filters) // consider making List in metabase
ids, err := meta.Select(s.metaBase, filters) // consider making List in metabase
if err != nil {
s.log.Debug("can't select all objects",
zap.Stringer("cid", lst[i]),

3
pkg/local_object_storage/shard/move.go
View file

@ -2,6 +2,7 @@ package shard
import (
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"go.uber.org/zap"
)
@ -26,7 +27,7 @@ func (p *ToMoveItPrm) WithAddress(addr *objectSDK.Address) *ToMoveItPrm {
// ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to
// another shard.
func (s *Shard) ToMoveIt(prm *ToMoveItPrm) (*ToMoveItRes, error) {
err := s.metaBase.ToMoveIt(prm.addr)
err := meta.ToMoveIt(s.metaBase, prm.addr)
if err != nil {
s.log.Debug("could not mark object for shard relocation in metabase",
zap.String("error", err.Error()),

3
pkg/local_object_storage/shard/put.go
View file

@ -3,6 +3,7 @@ package shard
import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/pkg/errors"
)
@ -56,7 +57,7 @@ func (s *Shard) Put(prm *PutPrm) (*PutRes, error) {
}
// put to metabase
if err := s.metaBase.Put(prm.obj, res.BlobovniczaID()); err != nil {
if err := meta.Put(s.metaBase, prm.obj, res.BlobovniczaID()); err != nil {
// may we need to handle this case in a special way
// since the object has been successfully written to BlobStor
return nil, errors.Wrap(err, "could not put object to metabase")

3
pkg/local_object_storage/shard/select.go
View file

@ -2,6 +2,7 @@ package shard
import (
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/pkg/errors"
)
@ -34,7 +35,7 @@ func (r *SelectRes) AddressList() []*objectSDK.Address {
// Returns any error encountered that
// did not allow to completely select the objects.
func (s *Shard) Select(prm *SelectPrm) (*SelectRes, error) {
addrList, err := s.metaBase.Select(prm.filters)
addrList, err := meta.Select(s.metaBase, prm.filters)
if err != nil {
return nil, errors.Wrap(err, "could not select objects from metabase")
}