TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 5 Projects Releases 1 Activity Actions

[#1559] metabase: Remove public functions

Browse source 
Reduce public interface of this package. Later each result will contain
an additional status, so it makes more sense to use the same functions
and result processing everywhere.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-12 17:42:55 +03:00 committed by fyrchik
parent 30c7925b3c
commit f58234aa2f
31 changed files with 234 additions and 247 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
pkg/local_object_storage/metabase/containers_test.go
View file

@ -6,7 +6,6 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
@ -64,7 +63,7 @@ func TestDB_Containers(t *testing.T) {
assertContains(cnrs, cnr)
require.NoError(t, meta.Inhume(db, object.AddressOf(obj), oidtest.Address()))
require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.Address()))
cnrs, err = db.Containers()
require.NoError(t, err)
@ -81,7 +80,7 @@ func TestDB_Containers(t *testing.T) {
cnr, _ := obj.ContainerID()
assertContains(cnrs, cnr)
require.NoError(t, meta.ToMoveIt(db, object.AddressOf(obj)))
require.NoError(t, metaToMoveIt(db, object.AddressOf(obj)))
cnrs, err = db.Containers()
require.NoError(t, err)
@ -179,7 +178,7 @@ func TestDB_ContainerSize(t *testing.T) {
volume := cids[cnr]
for _, obj := range list {
require.NoError(t, meta.Inhume(
require.NoError(t, metaInhume(
db,
object.AddressOf(obj),
oidtest.Address(),

12
pkg/local_object_storage/metabase/control_test.go
View file

@ -22,7 +22,7 @@ func TestReset(t *testing.T) {
addrToInhume := oidtest.Address()
assertExists := func(addr oid.Address, expExists bool, assertErr func(error) bool) {
exists, err := meta.Exists(db, addr)
exists, err := metaExists(db, addr)
if assertErr != nil {
require.True(t, assertErr(err))
} else {
@ -37,7 +37,7 @@ func TestReset(t *testing.T) {
err = putBig(db, obj)
require.NoError(t, err)
err = meta.Inhume(db, addrToInhume, oidtest.Address())
err = metaInhume(db, addrToInhume, oidtest.Address())
require.NoError(t, err)
assertExists(addr, true, nil)
@ -49,3 +49,11 @@ func TestReset(t *testing.T) {
assertExists(addr, false, nil)
assertExists(addr, false, nil)
}
func metaExists(db *meta.DB, addr oid.Address) (bool, error) {
var existsPrm meta.ExistsPrm
existsPrm.WithAddress(addr)
res, err := db.Exists(existsPrm)
return res.Exists(), err
}

4
pkg/local_object_storage/metabase/db_test.go
View file

@ -20,11 +20,11 @@ import (
// saves "big" object in DB.
func putBig(db *meta.DB, obj *object.Object) error {
return meta.Put(db, obj, nil)
return metaPut(db, obj, nil)
}
func testSelect(t *testing.T, db *meta.DB, cnr cid.ID, fs object.SearchFilters, exp ...oid.Address) {
res, err := meta.Select(db, cnr, fs)
res, err := metaSelect(db, cnr, fs)
require.NoError(t, err)
require.Len(t, res, len(exp))

9
pkg/local_object_storage/metabase/delete.go
View file

@ -30,15 +30,6 @@ func (p *DeletePrm) WithAddresses(addrs ...oid.Address) {
}
}
// Delete removes objects from DB.
func Delete(db *DB, addrs ...oid.Address) error {
var deletePrm DeletePrm
deletePrm.WithAddresses(addrs...)
_, err := db.Delete(deletePrm)
return err
}
type referenceNumber struct {
all, cur int

35
pkg/local_object_storage/metabase/delete_test.go
View file

@ -9,6 +9,7 @@ import (
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
@ -30,40 +31,40 @@ func TestDB_Delete(t *testing.T) {
require.NoError(t, err)
// fill ToMoveIt index
err = meta.ToMoveIt(db, object.AddressOf(child))
err = metaToMoveIt(db, object.AddressOf(child))
require.NoError(t, err)
// check if Movable list is not empty
l, err := meta.Movable(db)
l, err := metaMovable(db)
require.NoError(t, err)
require.Len(t, l, 1)
// try to remove parent unsuccessfully
err = meta.Delete(db, object.AddressOf(parent))
err = metaDelete(db, object.AddressOf(parent))
require.Error(t, err)
// inhume parent and child so they will be on graveyard
ts := generateObjectWithCID(t, cnr)
err = meta.Inhume(db, object.AddressOf(child), object.AddressOf(ts))
err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts))
require.NoError(t, err)
// delete object
err = meta.Delete(db, object.AddressOf(child))
err = metaDelete(db, object.AddressOf(child))
require.NoError(t, err)
// check if there is no data in Movable index
l, err = meta.Movable(db)
l, err = metaMovable(db)
require.NoError(t, err)
require.Len(t, l, 0)
// check if they marked as already removed
ok, err := meta.Exists(db, object.AddressOf(child))
ok, err := metaExists(db, object.AddressOf(child))
require.Error(t, apistatus.ObjectAlreadyRemoved{})
require.False(t, ok)
ok, err = meta.Exists(db, object.AddressOf(parent))
ok, err = metaExists(db, object.AddressOf(parent))
require.Error(t, apistatus.ObjectAlreadyRemoved{})
require.False(t, ok)
}
@ -91,16 +92,16 @@ func TestDeleteAllChildren(t *testing.T) {
require.NoError(t, putBig(db, child2))
// Exists should return split info for parent
_, err := meta.Exists(db, object.AddressOf(parent))
_, err := metaExists(db, object.AddressOf(parent))
siErr := objectSDK.NewSplitInfoError(nil)
require.True(t, errors.As(err, &siErr))
// remove all children in single call
err = meta.Delete(db, object.AddressOf(child1), object.AddressOf(child2))
err = metaDelete(db, object.AddressOf(child1), object.AddressOf(child2))
require.NoError(t, err)
// parent should not be found now
ex, err := meta.Exists(db, object.AddressOf(parent))
ex, err := metaExists(db, object.AddressOf(parent))
require.NoError(t, err)
require.False(t, ex)
}
@ -111,8 +112,16 @@ func TestGraveOnlyDelete(t *testing.T) {
addr := oidtest.Address()
// inhume non-existent object by address
require.NoError(t, meta.Inhume(db, addr, oidtest.Address()))
require.NoError(t, metaInhume(db, addr, oidtest.Address()))
// delete the object data
require.NoError(t, meta.Delete(db, addr))
require.NoError(t, metaDelete(db, addr))
}
func metaDelete(db *meta.DB, addrs ...oid.Address) error {
var deletePrm meta.DeletePrm
deletePrm.WithAddresses(addrs...)
_, err := db.Delete(deletePrm)
return err
}

15
pkg/local_object_storage/metabase/exists.go
View file

@ -35,21 +35,6 @@ func (p ExistsRes) Exists() bool {
return p.exists
}
// Exists checks if object is presented in DB.
//
// See DB.Exists docs.
func Exists(db *DB, addr oid.Address) (bool, error) {
var existsPrm ExistsPrm
existsPrm.WithAddress(addr)
r, err := db.Exists(existsPrm)
if err != nil {
return false, err
}
return r.Exists(), nil
}
// Exists returns ErrAlreadyRemoved if addr was marked as removed. Otherwise it
// returns true if addr is in primary index or false if it is not.
//

19
pkg/local_object_storage/metabase/exists_test.go
View file

@ -5,7 +5,6 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/stretchr/testify/require"
@ -16,7 +15,7 @@ func TestDB_Exists(t *testing.T) {
t.Run("no object", func(t *testing.T) {
nonExist := generateObject(t)
exists, err := meta.Exists(db, object.AddressOf(nonExist))
exists, err := metaExists(db, object.AddressOf(nonExist))
require.NoError(t, err)
require.False(t, exists)
})
@ -26,7 +25,7 @@ func TestDB_Exists(t *testing.T) {
err := putBig(db, regular)
require.NoError(t, err)
exists, err := meta.Exists(db, object.AddressOf(regular))
exists, err := metaExists(db, object.AddressOf(regular))
require.NoError(t, err)
require.True(t, exists)
})
@ -38,7 +37,7 @@ func TestDB_Exists(t *testing.T) {
err := putBig(db, ts)
require.NoError(t, err)
exists, err := meta.Exists(db, object.AddressOf(ts))
exists, err := metaExists(db, object.AddressOf(ts))
require.NoError(t, err)
require.True(t, exists)
})
@ -50,7 +49,7 @@ func TestDB_Exists(t *testing.T) {
err := putBig(db, sg)
require.NoError(t, err)
exists, err := meta.Exists(db, object.AddressOf(sg))
exists, err := metaExists(db, object.AddressOf(sg))
require.NoError(t, err)
require.True(t, exists)
})
@ -62,7 +61,7 @@ func TestDB_Exists(t *testing.T) {
err := putBig(db, lock)
require.NoError(t, err)
exists, err := meta.Exists(db, object.AddressOf(lock))
exists, err := metaExists(db, object.AddressOf(lock))
require.NoError(t, err)
require.True(t, exists)
})
@ -79,7 +78,7 @@ func TestDB_Exists(t *testing.T) {
err := putBig(db, child)
require.NoError(t, err)
_, err = meta.Exists(db, object.AddressOf(parent))
_, err = metaExists(db, object.AddressOf(parent))
var expectedErr *objectSDK.SplitInfoError
require.True(t, errors.As(err, &expectedErr))
@ -112,7 +111,7 @@ func TestDB_Exists(t *testing.T) {
err = putBig(db, link)
require.NoError(t, err)
_, err = meta.Exists(db, object.AddressOf(parent))
_, err = metaExists(db, object.AddressOf(parent))
require.Error(t, err)
si, ok := err.(*objectSDK.SplitInfoError)
@ -130,13 +129,13 @@ func TestDB_Exists(t *testing.T) {
})
t.Run("reverse order", func(t *testing.T) {
err := meta.Put(db, link, nil)
err := metaPut(db, link, nil)
require.NoError(t, err)
err = putBig(db, child)
require.NoError(t, err)
_, err = meta.Exists(db, object.AddressOf(parent))
_, err = metaExists(db, object.AddressOf(parent))
require.Error(t, err)
si, ok := err.(*objectSDK.SplitInfoError)

27
pkg/local_object_storage/metabase/get.go
View file

@ -44,33 +44,6 @@ func (r GetRes) Header() *objectSDK.Object {
return r.hdr
}
// Get reads the object from DB.
func Get(db *DB, addr oid.Address) (*objectSDK.Object, error) {
var getPrm GetPrm
getPrm.WithAddress(addr)
r, err := db.Get(getPrm)
if err != nil {
return nil, err
}
return r.Header(), nil
}
// GetRaw reads physically stored object from DB.
func GetRaw(db *DB, addr oid.Address, raw bool) (*objectSDK.Object, error) {
var getPrm GetPrm
getPrm.WithAddress(addr)
getPrm.WithRaw(raw)
r, err := db.Get(getPrm)
if err != nil {
return nil, err
}
return r.Header(), nil
}
// Get returns object header for specified address.
//
// Returns an error of type apistatus.ObjectNotFound if object is missing in DB.

31
pkg/local_object_storage/metabase/get_test.go
View file

@ -26,7 +26,7 @@ func TestDB_Get(t *testing.T) {
addAttribute(raw, "foo", "bar")
t.Run("object not found", func(t *testing.T) {
_, err := meta.Get(db, object.AddressOf(raw))
_, err := metaGet(db, object.AddressOf(raw), false)
require.Error(t, err)
})
@ -34,7 +34,7 @@ func TestDB_Get(t *testing.T) {
err := putBig(db, raw)
require.NoError(t, err)
newObj, err := meta.Get(db, object.AddressOf(raw))
newObj, err := metaGet(db, object.AddressOf(raw), false)
require.NoError(t, err)
require.Equal(t, raw.CutPayload(), newObj)
})
@ -46,7 +46,7 @@ func TestDB_Get(t *testing.T) {
err := putBig(db, raw)
require.NoError(t, err)
newObj, err := meta.Get(db, object.AddressOf(raw))
newObj, err := metaGet(db, object.AddressOf(raw), false)
require.NoError(t, err)
require.Equal(t, raw.CutPayload(), newObj)
})
@ -58,7 +58,7 @@ func TestDB_Get(t *testing.T) {
err := putBig(db, raw)
require.NoError(t, err)
newObj, err := meta.Get(db, object.AddressOf(raw))
newObj, err := metaGet(db, object.AddressOf(raw), false)
require.NoError(t, err)
require.Equal(t, raw.CutPayload(), newObj)
})
@ -70,7 +70,7 @@ func TestDB_Get(t *testing.T) {
err := putBig(db, raw)
require.NoError(t, err)
newObj, err := meta.Get(db, object.AddressOf(raw))
newObj, err := metaGet(db, object.AddressOf(raw), false)
require.NoError(t, err)
require.Equal(t, raw.CutPayload(), newObj)
})
@ -92,7 +92,7 @@ func TestDB_Get(t *testing.T) {
require.NoError(t, err)
t.Run("raw is true", func(t *testing.T) {
_, err = meta.GetRaw(db, object.AddressOf(parent), true)
_, err = metaGet(db, object.AddressOf(parent), true)
require.Error(t, err)
siErr, ok := err.(*objectSDK.SplitInfoError)
@ -108,11 +108,11 @@ func TestDB_Get(t *testing.T) {
require.False(t, ok)
})
newParent, err := meta.GetRaw(db, object.AddressOf(parent), false)
newParent, err := metaGet(db, object.AddressOf(parent), false)
require.NoError(t, err)
require.True(t, binaryEqual(parent.CutPayload(), newParent))
newChild, err := meta.GetRaw(db, object.AddressOf(child), true)
newChild, err := metaGet(db, object.AddressOf(child), true)
require.NoError(t, err)
require.True(t, binaryEqual(child.CutPayload(), newChild))
})
@ -121,8 +121,8 @@ func TestDB_Get(t *testing.T) {
obj := oidtest.Address()
ts := oidtest.Address()
require.NoError(t, meta.Inhume(db, obj, ts))
_, err := meta.Get(db, obj)
require.NoError(t, metaInhume(db, obj, ts))
_, err := metaGet(db, obj, false)
require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved))
obj = oidtest.Address()
@ -132,7 +132,7 @@ func TestDB_Get(t *testing.T) {
_, err = db.Inhume(prm)
require.NoError(t, err)
_, err = meta.Get(db, obj)
_, err = metaGet(db, obj, false)
require.ErrorAs(t, err, new(apistatus.ObjectNotFound))
})
}
@ -203,3 +203,12 @@ func benchmarkGet(b *testing.B, numOfObj int) {
}
}
func metaGet(db *meta.DB, addr oid.Address, raw bool) (*objectSDK.Object, error) {
var prm meta.GetPrm
prm.WithAddress(addr)
prm.WithRaw(raw)
res, err := db.Get(prm)
return res.Header(), err
}

13
pkg/local_object_storage/metabase/inhume.go
View file

@ -78,19 +78,6 @@ func (p *InhumePrm) WithForceGCMark() {
}
}
// Inhume inhumes the object by specified address.
//
// tomb should not be nil.
func Inhume(db *DB, target, tomb oid.Address) error {
var inhumePrm InhumePrm
inhumePrm.WithAddresses(target)
inhumePrm.WithTombstoneAddress(tomb)
_, err := db.Inhume(inhumePrm)
return err
}
var errBreakBucketForEach = errors.New("bucket ForEach break")
// ErrLockObjectRemoval is returned when inhume operation is being

15
pkg/local_object_storage/metabase/inhume_test.go
View file

@ -22,13 +22,13 @@ func TestDB_Inhume(t *testing.T) {
err := putBig(db, raw)
require.NoError(t, err)
err = meta.Inhume(db, object.AddressOf(raw), tombstoneID)
err = metaInhume(db, object.AddressOf(raw), tombstoneID)
require.NoError(t, err)
_, err = meta.Exists(db, object.AddressOf(raw))
_, err = metaExists(db, object.AddressOf(raw))
require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved))
_, err = meta.Get(db, object.AddressOf(raw))
_, err = metaGet(db, object.AddressOf(raw), false)
require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved))
}
@ -110,3 +110,12 @@ func TestInhumeLocked(t *testing.T) {
var e apistatus.ObjectLocked
require.ErrorAs(t, err, &e)
}
func metaInhume(db *meta.DB, target, tomb oid.Address) error {
var inhumePrm meta.InhumePrm
inhumePrm.WithAddresses(target)
inhumePrm.WithTombstoneAddress(tomb)
_, err := db.Inhume(inhumePrm)
return err
}

19
pkg/local_object_storage/metabase/list.go
View file

@ -54,25 +54,6 @@ func (l ListRes) Cursor() *Cursor {
return l.cursor
}
// ListWithCursor lists physical objects available in metabase starting from
// cursor. Includes objects of all types. Does not include inhumed objects.
// Use cursor value from response for consecutive requests.
//
// Returns ErrEndOfListing if there are no more objects to return or count
// parameter set to zero.
func ListWithCursor(db *DB, count uint32, cursor *Cursor) ([]oid.Address, *Cursor, error) {
var listPrm ListPrm
listPrm.WithCount(count)
listPrm.WithCursor(cursor)
r, err := db.ListWithCursor(listPrm)
if err != nil {
return nil, nil, err
}
return r.AddressList(), r.Cursor(), nil
}
// ListWithCursor lists physical objects available in metabase starting from
// cursor. Includes objects of all types. Does not include inhumed objects.
// Use cursor value from response for consecutive requests.

23
pkg/local_object_storage/metabase/list_test.go
View file

@ -113,7 +113,7 @@ func TestLisObjectsWithCursor(t *testing.T) {
err = putBig(db, obj)
require.NoError(t, err)
ts := generateObjectWithCID(t, containerID)
err = meta.Inhume(db, object.AddressOf(obj), object.AddressOf(ts))
err = metaInhume(db, object.AddressOf(obj), object.AddressOf(ts))
require.NoError(t, err)
// add one child object (do not include parent into expected)
@ -136,7 +136,7 @@ func TestLisObjectsWithCursor(t *testing.T) {
for countPerReq := 1; countPerReq <= total; countPerReq++ {
got := make([]oid.Address, 0, total)
res, cursor, err := meta.ListWithCursor(db, uint32(countPerReq), nil)
res, cursor, err := metaListWithCursor(db, uint32(countPerReq), nil)
require.NoError(t, err, "count:%d", countPerReq)
got = append(got, res...)
@ -146,12 +146,12 @@ func TestLisObjectsWithCursor(t *testing.T) {
}
for i := 0; i < expectedIterations; i++ {
res, cursor, err = meta.ListWithCursor(db, uint32(countPerReq), cursor)
res, cursor, err = metaListWithCursor(db, uint32(countPerReq), cursor)
require.NoError(t, err, "count:%d", countPerReq)
got = append(got, res...)
}
_, _, err = meta.ListWithCursor(db, uint32(countPerReq), cursor)
_, _, err = metaListWithCursor(db, uint32(countPerReq), cursor)
require.ErrorIs(t, err, meta.ErrEndOfListing, "count:%d", countPerReq, cursor)
got = sortAddresses(got)
@ -160,7 +160,7 @@ func TestLisObjectsWithCursor(t *testing.T) {
})
t.Run("invalid count", func(t *testing.T) {
_, _, err := meta.ListWithCursor(db, 0, nil)
_, _, err := metaListWithCursor(db, 0, nil)
require.ErrorIs(t, err, meta.ErrEndOfListing)
})
}
@ -181,7 +181,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) {
}
// get half of the objects
got, cursor, err := meta.ListWithCursor(db, total/2, nil)
got, cursor, err := metaListWithCursor(db, total/2, nil)
require.NoError(t, err)
for _, obj := range got {
if _, ok := expected[obj.EncodeToString()]; ok {
@ -198,7 +198,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) {
// get remaining objects
for {
got, cursor, err = meta.ListWithCursor(db, total, cursor)
got, cursor, err = metaListWithCursor(db, total, cursor)
if errors.Is(err, meta.ErrEndOfListing) {
break
}
@ -222,3 +222,12 @@ func sortAddresses(addr []oid.Address) []oid.Address {
})
return addr
}
func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]oid.Address, *meta.Cursor, error) {
var listPrm meta.ListPrm
listPrm.WithCount(count)
listPrm.WithCursor(cursor)
r, err := db.ListWithCursor(listPrm)
return r.AddressList(), r.Cursor(), err
}

4
pkg/local_object_storage/metabase/lock_test.go
View file

@ -35,7 +35,7 @@ func TestDB_Lock(t *testing.T) {
obj.SetContainerID(cnr)
// save irregular object
err := meta.Put(db, obj, nil)
err := metaPut(db, obj, nil)
require.NoError(t, err, typ)
var e apistatus.LockNonRegularObject
@ -95,7 +95,7 @@ func TestDB_Lock(t *testing.T) {
lockAddr := objectcore.AddressOf(lockObj)
// try to inhume locked object using tombstone
err := meta.Inhume(db, objAddr, lockAddr)
err := metaInhume(db, objAddr, lockAddr)
require.ErrorAs(t, err, new(apistatus.ObjectLocked))
// free locked object

28
pkg/local_object_storage/metabase/movable.go
View file

@ -50,15 +50,6 @@ func (p MovableRes) AddressList() []oid.Address {
return p.addrList
}
// ToMoveIt marks object to move it into another shard.
func ToMoveIt(db *DB, addr oid.Address) error {
var toMovePrm ToMoveItPrm
toMovePrm.WithAddress(addr)
_, err := db.ToMoveIt(toMovePrm)
return err
}
// ToMoveIt marks objects to move it into another shard. This useful for
// faster HRW fetching.
func (db *DB) ToMoveIt(prm ToMoveItPrm) (res ToMoveItRes, err error) {
@ -74,15 +65,6 @@ func (db *DB) ToMoveIt(prm ToMoveItPrm) (res ToMoveItRes, err error) {
return
}
// DoNotMove prevents the object to be moved into another shard.
func DoNotMove(db *DB, addr oid.Address) error {
var doNotMovePrm DoNotMovePrm
doNotMovePrm.WithAddress(addr)
_, err := db.DoNotMove(doNotMovePrm)
return err
}
// DoNotMove removes `MoveIt` mark from the object.
func (db *DB) DoNotMove(prm DoNotMovePrm) (res DoNotMoveRes, err error) {
err = db.boltDB.Update(func(tx *bbolt.Tx) error {
@ -97,16 +79,6 @@ func (db *DB) DoNotMove(prm DoNotMovePrm) (res DoNotMoveRes, err error) {
return
}
// Movable returns all movable objects of DB.
func Movable(db *DB) ([]oid.Address, error) {
r, err := db.Movable(MovablePrm{})
if err != nil {
return nil, err
}
return r.AddressList(), nil
}
// Movable returns list of marked objects to move into other shard.
func (db *DB) Movable(_ MovablePrm) (MovableRes, error) {
var strAddrs []string

40
pkg/local_object_storage/metabase/movable_test.go
View file

@ -5,6 +5,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
@ -22,35 +23,60 @@ func TestDB_Movable(t *testing.T) {
require.NoError(t, err)
// check if toMoveIt index empty
toMoveList, err := meta.Movable(db)
toMoveList, err := metaMovable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 0)
// mark to move object2
err = meta.ToMoveIt(db, object.AddressOf(raw2))
err = metaToMoveIt(db, object.AddressOf(raw2))
require.NoError(t, err)
// check if toMoveIt index contains address of object 2
toMoveList, err = meta.Movable(db)
toMoveList, err = metaMovable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 1)
require.Contains(t, toMoveList, object.AddressOf(raw2))
// remove from toMoveIt index non existing address
err = meta.DoNotMove(db, object.AddressOf(raw1))
err = metaDoNotMove(db, object.AddressOf(raw1))
require.NoError(t, err)
// check if toMoveIt index hasn't changed
toMoveList, err = meta.Movable(db)
toMoveList, err = metaMovable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 1)
// remove from toMoveIt index existing address
err = meta.DoNotMove(db, object.AddressOf(raw2))
err = metaDoNotMove(db, object.AddressOf(raw2))
require.NoError(t, err)
// check if toMoveIt index is empty now
toMoveList, err = meta.Movable(db)
toMoveList, err = metaMovable(db)
require.NoError(t, err)
require.Len(t, toMoveList, 0)
}
func metaToMoveIt(db *meta.DB, addr oid.Address) error {
var toMovePrm meta.ToMoveItPrm
toMovePrm.WithAddress(addr)
_, err := db.ToMoveIt(toMovePrm)
return err
}
func metaMovable(db *meta.DB) ([]oid.Address, error) {
r, err := db.Movable(meta.MovablePrm{})
if err != nil {
return nil, err
}
return r.AddressList(), nil
}
func metaDoNotMove(db *meta.DB, addr oid.Address) error {
var doNotMovePrm meta.DoNotMovePrm
doNotMovePrm.WithAddress(addr)
_, err := db.DoNotMove(doNotMovePrm)
return err
}

13
pkg/local_object_storage/metabase/put.go
View file

@ -53,19 +53,6 @@ var (
ErrIncorrectRootObject = errors.New("invalid root object")
)
// Put saves the object in DB.
//
// See DB.Put docs.
func Put(db *DB, obj *objectSDK.Object, id *blobovnicza.ID) error {
var putPrm PutPrm
putPrm.WithObject(obj)
putPrm.WithBlobovniczaID(id)
_, err := db.Put(putPrm)
return err
}
// Put saves object header in metabase. Object payload expected to be cut.
// Big objects have nil blobovniczaID.
//

24
pkg/local_object_storage/metabase/put_test.go
View file

@ -54,7 +54,7 @@ func BenchmarkPut(b *testing.B) {
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if err := meta.Put(db, objs[index.Inc()], nil); err != nil {
if err := metaPut(db, objs[index.Inc()], nil); err != nil {
b.Fatal(err)
}
}
@ -69,7 +69,7 @@ func BenchmarkPut(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if err := meta.Put(db, objs[index.Inc()], nil); err != nil {
if err := metaPut(db, objs[index.Inc()], nil); err != nil {
b.Fatal(err)
}
}
@ -83,20 +83,20 @@ func TestDB_PutBlobovnicaUpdate(t *testing.T) {
blobovniczaID := blobovnicza.ID{1, 2, 3, 4}
// put one object with blobovniczaID
err := meta.Put(db, raw1, &blobovniczaID)
err := metaPut(db, raw1, &blobovniczaID)
require.NoError(t, err)
fetchedBlobovniczaID, err := meta.IsSmall(db, object.AddressOf(raw1))
fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw1))
require.NoError(t, err)
require.Equal(t, &blobovniczaID, fetchedBlobovniczaID)
t.Run("update blobovniczaID", func(t *testing.T) {
newID := blobovnicza.ID{5, 6, 7, 8}
err := meta.Put(db, raw1, &newID)
err := metaPut(db, raw1, &newID)
require.NoError(t, err)
fetchedBlobovniczaID, err := meta.IsSmall(db, object.AddressOf(raw1))
fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw1))
require.NoError(t, err)
require.Equal(t, &newID, fetchedBlobovniczaID)
})
@ -106,8 +106,18 @@ func TestDB_PutBlobovnicaUpdate(t *testing.T) {
err := putBig(db, raw2)
require.NoError(t, err)
fetchedBlobovniczaID, err := meta.IsSmall(db, object.AddressOf(raw2))
fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw2))
require.NoError(t, err)
require.Nil(t, fetchedBlobovniczaID)
})
}
func metaPut(db *meta.DB, obj *objectSDK.Object, id *blobovnicza.ID) error {
var putPrm meta.PutPrm
putPrm.WithObject(obj)
putPrm.WithBlobovniczaID(id)
_, err := db.Put(putPrm)
return err
}

14
pkg/local_object_storage/metabase/select.go
View file

@ -58,20 +58,6 @@ func (r SelectRes) AddressList() []oid.Address {
return r.addrList
}
// Select selects the objects from DB with filtering.
func Select(db *DB, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) {
var selectPrm SelectPrm
selectPrm.WithFilters(fs)
selectPrm.WithContainerID(cnr)
r, err := db.Select(selectPrm)
if err != nil {
return nil, err
}
return r.AddressList(), nil
}
// Select returns list of addresses of objects that match search filters.
func (db *DB) Select(prm SelectPrm) (res SelectRes, err error) {
if blindlyProcess(prm.filters) {

23
pkg/local_object_storage/metabase/select_test.go
View file

@ -344,7 +344,7 @@ func TestDB_SelectInhume(t *testing.T) {
tombstone.SetContainer(cnr)
tombstone.SetObject(oidtest.ID())
err = meta.Inhume(db, object.AddressOf(raw2), tombstone)
err = metaInhume(db, object.AddressOf(raw2), tombstone)
require.NoError(t, err)
fs = objectSDK.SearchFilters{}
@ -777,7 +777,7 @@ func BenchmarkSelect(b *testing.B) {
attr.SetValue(strconv.Itoa(i))
obj := generateObjectWithCID(b, cid)
obj.SetAttributes(attr)
require.NoError(b, meta.Put(db, obj, nil))
require.NoError(b, metaPut(db, obj, nil))
}
b.Run("string equal", func(b *testing.B) {
@ -806,13 +806,26 @@ func BenchmarkSelect(b *testing.B) {
}
func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.SearchFilters, expected int) {
var prm meta.SelectPrm
prm.WithContainerID(cid)
prm.WithFilters(fs)
for i := 0; i < b.N; i++ {
res, err := meta.Select(db, cid, fs)
res, err := db.Select(prm)
if err != nil {
b.Fatal(err)
}
if len(res) != expected {
b.Fatalf("expected %d items, got %d", expected, len(res))
if len(res.AddressList()) != expected {
b.Fatalf("expected %d items, got %d", expected, len(res.AddressList()))
}
}
}
func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) {
var prm meta.SelectPrm
prm.WithFilters(fs)
prm.WithContainerID(cnr)
res, err := db.Select(prm)
return res.AddressList(), err
}

15
pkg/local_object_storage/metabase/small.go
View file

@ -29,21 +29,6 @@ func (r IsSmallRes) BlobovniczaID() *blobovnicza.ID {
return r.id
}
// IsSmall wraps work with DB.IsSmall method with specified
// address and other parameters by default. Returns only
// the blobovnicza identifier.
func IsSmall(db *DB, addr oid.Address) (*blobovnicza.ID, error) {
var isSmallPrm IsSmallPrm
isSmallPrm.WithAddress(addr)
r, err := db.IsSmall(isSmallPrm)
if err != nil {
return nil, err
}
return r.BlobovniczaID(), nil
}
// IsSmall returns blobovniczaID for small objects and nil for big objects.
// Small objects stored in blobovnicza instances. Big objects stored in FS by
// shallow path which is calculated from address and therefore it is not

17
pkg/local_object_storage/metabase/small_test.go
View file

@ -6,6 +6,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
@ -18,12 +19,12 @@ func TestDB_IsSmall(t *testing.T) {
blobovniczaID := blobovnicza.ID{1, 2, 3, 4}
// check IsSmall from empty database
fetchedBlobovniczaID, err := meta.IsSmall(db, object.AddressOf(raw1))
fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw1))
require.NoError(t, err)
require.Nil(t, fetchedBlobovniczaID)
// put one object with blobovniczaID
err = meta.Put(db, raw1, &blobovniczaID)
err = metaPut(db, raw1, &blobovniczaID)
require.NoError(t, err)
// put one object without blobovniczaID
@ -31,12 +32,20 @@ func TestDB_IsSmall(t *testing.T) {
require.NoError(t, err)
// check IsSmall for object without blobovniczaID
fetchedBlobovniczaID, err = meta.IsSmall(db, object.AddressOf(raw2))
fetchedBlobovniczaID, err = metaIsSmall(db, object.AddressOf(raw2))
require.NoError(t, err)
require.Nil(t, fetchedBlobovniczaID)
// check IsSmall for object with blobovniczaID
fetchedBlobovniczaID, err = meta.IsSmall(db, object.AddressOf(raw1))
fetchedBlobovniczaID, err = metaIsSmall(db, object.AddressOf(raw1))
require.NoError(t, err)
require.Equal(t, &blobovniczaID, fetchedBlobovniczaID)
}
func metaIsSmall(db *meta.DB, addr oid.Address) (*blobovnicza.ID, error) {
var isSmallPrm meta.IsSmallPrm
isSmallPrm.WithAddress(addr)
r, err := db.IsSmall(isSmallPrm)
return r.BlobovniczaID(), err
}

6
pkg/local_object_storage/shard/control.go
View file

@ -145,7 +145,11 @@ func (s *Shard) refillMetabase() error {
}
}
err := meta.Put(s.metaBase, obj, blzID)
var mPrm meta.PutPrm
mPrm.WithObject(obj)
mPrm.WithBlobovniczaID(blzID)
_, err := s.metaBase.Put(mPrm)
if err != nil && !meta.IsErrRemoved(err) {
return err
}

14
pkg/local_object_storage/shard/delete.go
View file

@ -47,7 +47,10 @@ func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) {
}
}
blobovniczaID, err := meta.IsSmall(s.metaBase, prm.addr[i])
var sPrm meta.IsSmallPrm
sPrm.WithAddress(prm.addr[i])
res, err := s.metaBase.IsSmall(sPrm)
if err != nil {
s.log.Debug("can't get blobovniczaID from metabase",
zap.Stringer("object", prm.addr[i]),
@ -56,12 +59,15 @@ func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) {
continue
}
if blobovniczaID != nil {
smalls[prm.addr[i]] = blobovniczaID
if res.BlobovniczaID() != nil {
smalls[prm.addr[i]] = res.BlobovniczaID()
}
}
err := meta.Delete(s.metaBase, prm.addr...)
var delPrm meta.DeletePrm
delPrm.WithAddresses(prm.addr...)
_, err := s.metaBase.Delete(delPrm)
if err != nil {
return DeleteRes{}, err // stop on metabase error ?
}

6
pkg/local_object_storage/shard/exists.go
View file

@ -38,7 +38,11 @@ func (p ExistsRes) Exists() bool {
//
// Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed.
func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) {
exists, err := meta.Exists(s.metaBase, prm.addr)
var existsPrm meta.ExistsPrm
existsPrm.WithAddress(prm.addr)
res, err := s.metaBase.Exists(existsPrm)
exists := res.Exists()
if err != nil {
// If the shard is in degraded mode, try to consult blobstor directly.
// Otherwise, just return an error.

15
pkg/local_object_storage/shard/get.go
View file

@ -119,10 +119,14 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
var exists bool
if !skipMeta {
exists, err = meta.Exists(s.metaBase, addr)
var mPrm meta.ExistsPrm
mPrm.WithAddress(addr)
mRes, err := s.metaBase.Exists(mPrm)
if err != nil && s.GetMode() != ModeDegraded {
return res, false, err
}
exists = mRes.Exists()
}
if skipMeta || err != nil {
@ -140,13 +144,16 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, big, small stor
return nil, false, errNotFound
}
blobovniczaID, err := meta.IsSmall(s.metaBase, addr)
var mPrm meta.IsSmallPrm
mPrm.WithAddress(addr)
mRes, err := s.metaBase.IsSmall(mPrm)
if err != nil {
return nil, true, fmt.Errorf("can't fetch blobovnicza id from metabase: %w", err)
}
if blobovniczaID != nil {
res, err = small(s.blobStor, blobovniczaID)
if mRes.BlobovniczaID() != nil {
res, err = small(s.blobStor, mRes.BlobovniczaID())
} else {
res, err = big(s.blobStor, nil)
}

8
pkg/local_object_storage/shard/list.go
View file

@ -73,7 +73,11 @@ func (s *Shard) List() (res SelectRes, err error) {
filters.AddPhyFilter()
for i := range lst {
ids, err := meta.Select(s.metaBase, lst[i], filters) // consider making List in metabase
var sPrm meta.SelectPrm
sPrm.WithContainerID(lst[i])
sPrm.WithFilters(filters)
sRes, err := s.metaBase.Select(sPrm) // consider making List in metabase
if err != nil {
s.log.Debug("can't select all objects",
zap.Stringer("cid", lst[i]),
@ -82,7 +86,7 @@ func (s *Shard) List() (res SelectRes, err error) {
continue
}
res.addrList = append(res.addrList, ids...)
res.addrList = append(res.addrList, sRes.AddressList()...)
}
return res, nil

5
pkg/local_object_storage/shard/move.go
View file

@ -29,7 +29,10 @@ func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) {
return ToMoveItRes{}, ErrReadOnlyMode
}
err := meta.ToMoveIt(s.metaBase, prm.addr)
var toMovePrm meta.ToMoveItPrm
toMovePrm.WithAddress(prm.addr)
_, err := s.metaBase.ToMoveIt(toMovePrm)
if err != nil {
s.log.Debug("could not mark object for shard relocation in metabase",
zap.String("error", err.Error()),

5
pkg/local_object_storage/shard/put.go
View file

@ -60,7 +60,10 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
}
// put to metabase
if err := meta.Put(s.metaBase, prm.obj, res.BlobovniczaID()); err != nil {
var pPrm meta.PutPrm
pPrm.WithObject(prm.obj)
pPrm.WithBlobovniczaID(res.BlobovniczaID())
if _, err := s.metaBase.Put(pPrm); err != nil {
// may we need to handle this case in a special way
// since the object has been successfully written to BlobStor
return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err)

8
pkg/local_object_storage/shard/select.go
View file

@ -44,12 +44,16 @@ func (r SelectRes) AddressList() []oid.Address {
// Returns any error encountered that
// did not allow to completely select the objects.
func (s *Shard) Select(prm SelectPrm) (SelectRes, error) {
addrList, err := meta.Select(s.metaBase, prm.cnr, prm.filters)
var selectPrm meta.SelectPrm
selectPrm.WithFilters(prm.filters)
selectPrm.WithContainerID(prm.cnr)
mRes, err := s.metaBase.Select(selectPrm)
if err != nil {
return SelectRes{}, fmt.Errorf("could not select objects from metabase: %w", err)
}
return SelectRes{
addrList: addrList,
addrList: mRes.AddressList(),
}, nil
}

7
pkg/local_object_storage/writecache/flush.go
View file

@ -238,7 +238,12 @@ func (c *cache) writeObject(obj *object.Object, metaOnly bool) error {
id = res.BlobovniczaID()
}
return meta.Put(c.metabase, obj, id)
var pPrm meta.PutPrm
pPrm.WithObject(obj)
pPrm.WithBlobovniczaID(id)
_, err := c.metabase.Put(pPrm)
return err
}
func cloneBytes(a []byte) []byte {