From 73f8bb3e5f351b8300ea633aec7872fe8296b263 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Jul 2022 17:09:50 +0300 Subject: [PATCH] [#1523] shard: Store generic storage ID in metabase Allow to extend blobstor with more storage sub-systems. Currently objects stored in the FSTree have empty byte slice descriptor and object from blobovnicza tree have the same id as earlier. Each such change in the identifier formation should be accompanied with metabase version increase. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobovnicza/id.go | 4 ++ .../blobstor/blobovniczatree/blobovnicza.go | 19 +++--- .../blobovniczatree/blobovnicza_test.go | 8 +-- .../blobstor/common/delete.go | 5 +- .../blobstor/common/get.go | 5 +- .../blobstor/common/get_range.go | 7 +-- .../blobstor/common/put.go | 3 +- pkg/local_object_storage/blobstor/delete.go | 4 +- pkg/local_object_storage/blobstor/get.go | 4 +- .../blobstor/get_range.go | 4 +- pkg/local_object_storage/blobstor/iterate.go | 17 +++--- .../blobstor/iterate_test.go | 7 +-- pkg/local_object_storage/metabase/VERSION.md | 23 ++++--- pkg/local_object_storage/metabase/delete.go | 2 +- pkg/local_object_storage/metabase/put.go | 31 +++++----- pkg/local_object_storage/metabase/put_test.go | 29 +++++---- pkg/local_object_storage/metabase/small.go | 61 ------------------- .../metabase/small_test.go | 51 ---------------- .../metabase/storage_id.go | 53 ++++++++++++++++ .../metabase/storage_id_test.go | 50 +++++++++++++++ pkg/local_object_storage/metabase/version.go | 2 +- pkg/local_object_storage/shard/control.go | 5 +- pkg/local_object_storage/shard/delete.go | 42 ++++--------- pkg/local_object_storage/shard/get.go | 21 +++---- pkg/local_object_storage/shard/put.go | 2 +- pkg/local_object_storage/shard/range.go | 5 +- pkg/local_object_storage/writecache/flush.go | 7 +-- 27 files changed, 220 insertions(+), 251 deletions(-) delete mode 100644 pkg/local_object_storage/metabase/small.go delete mode 100644 pkg/local_object_storage/metabase/small_test.go create mode 100644 pkg/local_object_storage/metabase/storage_id.go create mode 100644 pkg/local_object_storage/metabase/storage_id_test.go diff --git a/pkg/local_object_storage/blobovnicza/id.go b/pkg/local_object_storage/blobovnicza/id.go index 79c30a210..3d3ccf8b9 100644 --- a/pkg/local_object_storage/blobovnicza/id.go +++ b/pkg/local_object_storage/blobovnicza/id.go @@ -11,3 +11,7 @@ func NewIDFromBytes(v []byte) *ID { func (id ID) String() string { return string(id) } + +func (id ID) Bytes() []byte { + return id +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 5f8ad0b94..551178c47 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -204,7 +204,7 @@ func (b *Blobovniczas) Put(prm common.PutPrm) (common.PutRes, error) { return common.PutRes{}, errPutFailed } - return common.PutRes{BlobovniczaID: id}, nil + return common.PutRes{StorageID: id.Bytes()}, nil } // Get reads object from blobovnicza tree. @@ -215,8 +215,9 @@ func (b *Blobovniczas) Get(prm common.GetPrm) (res common.GetRes, err error) { var bPrm blobovnicza.GetPrm bPrm.SetAddress(prm.Address) - if prm.BlobovniczaID != nil { - blz, err := b.openBlobovnicza(prm.BlobovniczaID.String()) + if prm.StorageID != nil { + id := blobovnicza.NewIDFromBytes(prm.StorageID) + blz, err := b.openBlobovnicza(id.String()) if err != nil { return res, err } @@ -265,8 +266,9 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e var bPrm blobovnicza.DeletePrm bPrm.SetAddress(prm.Address) - if prm.BlobovniczaID != nil { - blz, err := b.openBlobovnicza(prm.BlobovniczaID.String()) + if prm.StorageID != nil { + id := blobovnicza.NewIDFromBytes(prm.StorageID) + blz, err := b.openBlobovnicza(id.String()) if err != nil { return res, err } @@ -318,8 +320,9 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) GetRange(prm common.GetRangePrm) (res common.GetRangeRes, err error) { - if prm.BlobovniczaID != nil { - blz, err := b.openBlobovnicza(prm.BlobovniczaID.String()) + if prm.StorageID != nil { + id := blobovnicza.NewIDFromBytes(prm.StorageID) + blz, err := b.openBlobovnicza(id.String()) if err != nil { return common.GetRangeRes{}, err } @@ -573,7 +576,7 @@ func (b *Blobovniczas) deleteObject(blz *blobovnicza.Blobovnicza, prm blobovnicz storagelog.Write(b.log, storagelog.AddressField(dp.Address), storagelog.OpField("Blobovniczas DELETE"), - zap.Stringer("blobovnicza ID", dp.BlobovniczaID), + zap.Stringer("blobovnicza ID", blobovnicza.NewIDFromBytes(dp.StorageID)), ) return common.DeleteRes{}, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza_test.go index 1006a64f4..b1febdba5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza_test.go @@ -78,7 +78,7 @@ func TestBlobovniczas(t *testing.T) { // get w/ blobovnicza ID var prm common.GetPrm - prm.BlobovniczaID = pRes.BlobovniczaID + prm.StorageID = pRes.StorageID prm.Address = addr res, err := b.Get(prm) @@ -86,7 +86,7 @@ func TestBlobovniczas(t *testing.T) { require.Equal(t, obj, res.Object) // get w/o blobovnicza ID - prm.BlobovniczaID = nil + prm.StorageID = nil res, err = b.Get(prm) require.NoError(t, err) @@ -94,7 +94,7 @@ func TestBlobovniczas(t *testing.T) { // get range w/ blobovnicza ID var rngPrm common.GetRangePrm - rngPrm.BlobovniczaID = pRes.BlobovniczaID + rngPrm.StorageID = pRes.StorageID rngPrm.Address = addr payload := obj.Payload() @@ -109,7 +109,7 @@ func TestBlobovniczas(t *testing.T) { require.Equal(t, payload[off:off+ln], rngRes.Data) // get range w/o blobovnicza ID - rngPrm.BlobovniczaID = nil + rngPrm.StorageID = nil rngRes, err = b.GetRange(rngPrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/blobstor/common/delete.go b/pkg/local_object_storage/blobstor/common/delete.go index 0ed993450..03cae4fd2 100644 --- a/pkg/local_object_storage/blobstor/common/delete.go +++ b/pkg/local_object_storage/blobstor/common/delete.go @@ -1,14 +1,13 @@ package common import ( - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) // DeletePrm groups the parameters of Delete operation. type DeletePrm struct { - Address oid.Address - BlobovniczaID *blobovnicza.ID + Address oid.Address + StorageID []byte } // DeleteRes groups the resulting values of Delete operation. diff --git a/pkg/local_object_storage/blobstor/common/get.go b/pkg/local_object_storage/blobstor/common/get.go index 8c267ceeb..414c39ea1 100644 --- a/pkg/local_object_storage/blobstor/common/get.go +++ b/pkg/local_object_storage/blobstor/common/get.go @@ -1,14 +1,13 @@ package common import ( - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) type GetPrm struct { - Address oid.Address - BlobovniczaID *blobovnicza.ID + Address oid.Address + StorageID []byte } type GetRes struct { diff --git a/pkg/local_object_storage/blobstor/common/get_range.go b/pkg/local_object_storage/blobstor/common/get_range.go index 7730e1c35..d52fa408b 100644 --- a/pkg/local_object_storage/blobstor/common/get_range.go +++ b/pkg/local_object_storage/blobstor/common/get_range.go @@ -1,15 +1,14 @@ package common import ( - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) type GetRangePrm struct { - Address oid.Address - Range objectSDK.Range - BlobovniczaID *blobovnicza.ID + Address oid.Address + Range objectSDK.Range + StorageID []byte } type GetRangeRes struct { diff --git a/pkg/local_object_storage/blobstor/common/put.go b/pkg/local_object_storage/blobstor/common/put.go index 223f8b5e0..053036ba5 100644 --- a/pkg/local_object_storage/blobstor/common/put.go +++ b/pkg/local_object_storage/blobstor/common/put.go @@ -1,7 +1,6 @@ package common import ( - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) @@ -15,5 +14,5 @@ type PutPrm struct { // PutRes groups the resulting values of Put operation. type PutRes struct { - BlobovniczaID *blobovnicza.ID + StorageID []byte } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 9ab945087..8b675ea48 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -10,7 +10,7 @@ import ( ) func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { - if prm.BlobovniczaID == nil { + if prm.StorageID == nil { // Nothing specified, try everything. res, err := b.deleteBig(prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { @@ -18,7 +18,7 @@ func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { } return b.deleteSmall(prm) } - if *prm.BlobovniczaID == nil { + if len(prm.StorageID) == 0 { return b.deleteBig(prm) } return b.deleteSmall(prm) diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index b599d0883..fe9a8223b 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -14,7 +14,7 @@ import ( // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. func (b *BlobStor) Get(prm common.GetPrm) (common.GetRes, error) { - if prm.BlobovniczaID == nil { + if prm.StorageID == nil { // Nothing specified, try everything. res, err := b.getBig(prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { @@ -22,7 +22,7 @@ func (b *BlobStor) Get(prm common.GetPrm) (common.GetRes, error) { } return b.getSmall(prm) } - if *prm.BlobovniczaID == nil { + if len(prm.StorageID) == 0 { return b.getBig(prm) } return b.getSmall(prm) diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index a0c3a4dd8..f059b2b0d 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -14,7 +14,7 @@ import ( // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. func (b *BlobStor) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) { - if prm.BlobovniczaID == nil { + if prm.StorageID == nil { // Nothing specified, try everything. res, err := b.getRangeBig(prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { @@ -22,7 +22,7 @@ func (b *BlobStor) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) } return b.getRangeSmall(prm) } - if *prm.BlobovniczaID == nil { + if len(prm.StorageID) == 0 { return b.getRangeBig(prm) } return b.getRangeSmall(prm) diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index fd09b9605..1e534ef50 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -15,7 +15,7 @@ type IterationElement struct { addr oid.Address - blzID *blobovnicza.ID + descriptor []byte } // ObjectData returns the stored object in a binary representation. @@ -23,10 +23,9 @@ func (x IterationElement) ObjectData() []byte { return x.data } -// BlobovniczaID returns the identifier of Blobovnicza in which object is stored. -// Returns nil if the object isn't in Blobovnicza. -func (x IterationElement) BlobovniczaID() *blobovnicza.ID { - return x.blzID +// Descriptor returns the identifier of storage part where x is stored. +func (x IterationElement) Descriptor() []byte { + return x.descriptor } // Address returns the object address. @@ -89,7 +88,7 @@ func (b *BlobStor) Iterate(prm IteratePrm) (IterateRes, error) { } elem.addr = addr - elem.blzID = blobovnicza.NewIDFromBytes([]byte(p)) + elem.descriptor = []byte(p) return prm.handler(elem) }) @@ -103,7 +102,7 @@ func (b *BlobStor) Iterate(prm IteratePrm) (IterateRes, error) { return IterateRes{}, fmt.Errorf("blobovniczas iterator failure: %w", err) } - elem.blzID = nil + elem.descriptor = []byte{} var fsPrm fstree.IterationPrm fsPrm.WithIgnoreErrors(prm.ignoreErrors) @@ -136,11 +135,11 @@ func (b *BlobStor) Iterate(prm IteratePrm) (IterateRes, error) { // IterateBinaryObjects is a helper function which iterates over BlobStor and passes binary objects to f. // Errors related to object reading and unmarshaling are logged and skipped. -func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, blzID *blobovnicza.ID) error) error { +func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { var prm IteratePrm prm.SetIterationHandler(func(elem IterationElement) error { - return f(elem.Address(), elem.ObjectData(), elem.BlobovniczaID()) + return f(elem.Address(), elem.ObjectData(), elem.Descriptor()) }) prm.IgnoreErrors() prm.SetErrorHandler(func(addr oid.Address, err error) error { diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 06cc6684f..2aad4741c 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -5,7 +5,6 @@ import ( "os" "testing" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" @@ -71,16 +70,16 @@ func TestIterateObjects(t *testing.T) { require.NoError(t, err) } - err := IterateBinaryObjects(blobStor, func(_ oid.Address, data []byte, blzID *blobovnicza.ID) error { + err := IterateBinaryObjects(blobStor, func(_ oid.Address, data []byte, descriptor []byte) error { v, ok := mObjs[string(data)] require.True(t, ok) require.Equal(t, v.data, data) if v.big { - require.Nil(t, blzID) + require.True(t, descriptor != nil && len(descriptor) == 0) } else { - require.NotNil(t, blzID) + require.NotEmpty(t, descriptor) } delete(mObjs, string(data)) diff --git a/pkg/local_object_storage/metabase/VERSION.md b/pkg/local_object_storage/metabase/VERSION.md index d66c0a628..7ad674b6a 100644 --- a/pkg/local_object_storage/metabase/VERSION.md +++ b/pkg/local_object_storage/metabase/VERSION.md @@ -2,11 +2,7 @@ This file describes changes between the metabase versions. -## Version 0 - -- Container ID is encoded as base58 string -- Object ID is encoded as base58 string -- Address is encoded as container ID + "/" + object ID +## Current ### Primary buckets - Graveyard bucket @@ -53,10 +49,10 @@ This file describes changes between the metabase versions. - Name: container ID + `_TS` - Key: object ID - Value: marshaled object -- Buckets mapping objects to the blobovniczas they are stored in +- Buckets mapping objects to the storage ID they are stored in - Name: container ID + `_small` - Key: object ID - - Value: blobovnicza ID + - Value: storage ID - Buckets for mapping parent object to the split info - Name: container ID + `_root` - Key: object ID @@ -85,3 +81,16 @@ This file describes changes between the metabase versions. - Name: container ID + `_splitid` - Key: split ID - Value: list of object IDs + + +# History + +## Version 1 + +- Metabase now stores generic storage id instead of blobovnicza ID. + +## Version 0 + +- Container ID is encoded as base58 string +- Object ID is encoded as base58 string +- Address is encoded as container ID + "/" + object ID \ No newline at end of file diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 0c7218bf5..3cf50ff37 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -254,7 +254,7 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error }) } - delUniqueIndexItem(tx, namedBucketItem{ // remove from small blobovnicza id index + delUniqueIndexItem(tx, namedBucketItem{ // remove from storage id index name: smallBucketName(cnr), key: objKey, }) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index f04250a06..f47b5e0ec 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -9,7 +9,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neofs-node/pkg/core/object" objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/util" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" @@ -27,7 +26,7 @@ type ( type PutPrm struct { obj *objectSDK.Object - id *blobovnicza.ID + id []byte } // PutRes groups the resulting values of Put operation. @@ -38,8 +37,8 @@ func (p *PutPrm) SetObject(obj *objectSDK.Object) { p.obj = obj } -// SetBlobovniczaID is a Put option to set blobovnicza ID to save. -func (p *PutPrm) SetBlobovniczaID(id *blobovnicza.ID) { +// SetStorageID is a Put option to set storage ID to save. +func (p *PutPrm) SetStorageID(id []byte) { p.id = id } @@ -50,7 +49,6 @@ var ( ) // Put saves object header in metabase. Object payload expected to be cut. -// Big objects have nil blobovniczaID. // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. @@ -73,7 +71,7 @@ func (db *DB) Put(prm PutPrm) (res PutRes, err error) { } func (db *DB) put( - tx *bbolt.Tx, obj *objectSDK.Object, id *blobovnicza.ID, + tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, currEpoch uint64) error { cnr, ok := obj.ContainerID() if !ok { @@ -93,11 +91,10 @@ func (db *DB) put( // most right child and split header overlap parent so we have to // check if object exists to not overwrite it twice if exists { - // when storage engine moves small objects from one blobovniczaID - // to another, then it calls metabase.Put method with new blobovniczaID - // and this code should be triggered + // When storage engine moves objects between different sub-storages, + // it calls metabase.Put method with new storage ID, thus triggering this code. if !isParent && id != nil { - return updateBlobovniczaID(tx, object.AddressOf(obj), id) + return updateStorageID(tx, object.AddressOf(obj), id) } // when storage already has last object in split hierarchy and there is @@ -152,7 +149,7 @@ func putUniqueIndexes( tx *bbolt.Tx, obj *objectSDK.Object, si *objectSDK.SplitInfo, - id *blobovnicza.ID, + id []byte, ) error { isParent := si != nil addr := object.AddressOf(obj) @@ -190,12 +187,12 @@ func putUniqueIndexes( return err } - // index blobovniczaID if it is present + // index storageID if it is present if id != nil { err = putUniqueIndexItem(tx, namedBucketItem{ name: smallBucketName(cnr), key: objKey, - val: *id, + val: id, }) if err != nil { return err @@ -423,15 +420,15 @@ func getVarUint(data []byte) (uint64, int, error) { } } -// updateBlobovniczaID for existing objects if they were moved from from -// one blobovnicza to another. -func updateBlobovniczaID(tx *bbolt.Tx, addr oid.Address, id *blobovnicza.ID) error { +// updateStorageID for existing objects if they were moved from one +// storage location to another. +func updateStorageID(tx *bbolt.Tx, addr oid.Address, id []byte) error { bkt, err := tx.CreateBucketIfNotExists(smallBucketName(addr.Container())) if err != nil { return err } - return bkt.Put(objectKey(addr.Object()), *id) + return bkt.Put(objectKey(addr.Object()), id) } // updateSpliInfo for existing objects if storage filled with extra information diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index 82253d0b9..f6b020d1b 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/nspcc-dev/neofs-node/pkg/core/object" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/util/rand" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" @@ -80,42 +79,42 @@ func TestDB_PutBlobovnicaUpdate(t *testing.T) { db := newDB(t) raw1 := generateObject(t) - blobovniczaID := blobovnicza.ID{1, 2, 3, 4} + storageID := []byte{1, 2, 3, 4} - // put one object with blobovniczaID - err := metaPut(db, raw1, &blobovniczaID) + // put one object with storageID + err := metaPut(db, raw1, storageID) require.NoError(t, err) - fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw1)) + fetchedStorageID, err := metaStorageID(db, object.AddressOf(raw1)) require.NoError(t, err) - require.Equal(t, &blobovniczaID, fetchedBlobovniczaID) + require.Equal(t, storageID, fetchedStorageID) - t.Run("update blobovniczaID", func(t *testing.T) { - newID := blobovnicza.ID{5, 6, 7, 8} + t.Run("update storageID", func(t *testing.T) { + newID := []byte{5, 6, 7, 8} - err := metaPut(db, raw1, &newID) + err := metaPut(db, raw1, newID) require.NoError(t, err) - fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw1)) + fetchedBlobovniczaID, err := metaStorageID(db, object.AddressOf(raw1)) require.NoError(t, err) - require.Equal(t, &newID, fetchedBlobovniczaID) + require.Equal(t, newID, fetchedBlobovniczaID) }) - t.Run("update blobovniczaID on bad object", func(t *testing.T) { + t.Run("update storageID on bad object", func(t *testing.T) { raw2 := generateObject(t) err := putBig(db, raw2) require.NoError(t, err) - fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw2)) + fetchedBlobovniczaID, err := metaStorageID(db, object.AddressOf(raw2)) require.NoError(t, err) require.Nil(t, fetchedBlobovniczaID) }) } -func metaPut(db *meta.DB, obj *objectSDK.Object, id *blobovnicza.ID) error { +func metaPut(db *meta.DB, obj *objectSDK.Object, id []byte) error { var putPrm meta.PutPrm putPrm.SetObject(obj) - putPrm.SetBlobovniczaID(id) + putPrm.SetStorageID(id) _, err := db.Put(putPrm) diff --git a/pkg/local_object_storage/metabase/small.go b/pkg/local_object_storage/metabase/small.go deleted file mode 100644 index 6b2280fff..000000000 --- a/pkg/local_object_storage/metabase/small.go +++ /dev/null @@ -1,61 +0,0 @@ -package meta - -import ( - "github.com/nspcc-dev/neo-go/pkg/util/slice" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "go.etcd.io/bbolt" -) - -// IsSmallPrm groups the parameters of IsSmall operation. -type IsSmallPrm struct { - addr oid.Address -} - -// IsSmallRes groups the resulting values of IsSmall operation. -type IsSmallRes struct { - id *blobovnicza.ID -} - -// WithAddress is a IsSmall option to set the object address to check. -func (p *IsSmallPrm) WithAddress(addr oid.Address) { - if p != nil { - p.addr = addr - } -} - -// BlobovniczaID returns blobovnicza identifier. -func (r IsSmallRes) BlobovniczaID() *blobovnicza.ID { - return r.id -} - -// IsSmall returns blobovniczaID for small objects and nil for big objects. -// Small objects stored in blobovnicza instances. Big objects stored in FS by -// shallow path which is calculated from address and therefore it is not -// indexed in metabase. -func (db *DB) IsSmall(prm IsSmallPrm) (res IsSmallRes, err error) { - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() - - err = db.boltDB.View(func(tx *bbolt.Tx) error { - res.id, err = db.isSmall(tx, prm.addr) - - return err - }) - - return -} - -func (db *DB) isSmall(tx *bbolt.Tx, addr oid.Address) (*blobovnicza.ID, error) { - smallBucket := tx.Bucket(smallBucketName(addr.Container())) - if smallBucket == nil { - return nil, nil - } - - blobovniczaID := smallBucket.Get(objectKey(addr.Object())) - if len(blobovniczaID) == 0 { - return nil, nil - } - - return blobovnicza.NewIDFromBytes(slice.Copy(blobovniczaID)), nil -} diff --git a/pkg/local_object_storage/metabase/small_test.go b/pkg/local_object_storage/metabase/small_test.go deleted file mode 100644 index 534a6862b..000000000 --- a/pkg/local_object_storage/metabase/small_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package meta_test - -import ( - "testing" - - "github.com/nspcc-dev/neofs-node/pkg/core/object" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" - meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/stretchr/testify/require" -) - -func TestDB_IsSmall(t *testing.T) { - db := newDB(t) - - raw1 := generateObject(t) - raw2 := generateObject(t) - - blobovniczaID := blobovnicza.ID{1, 2, 3, 4} - - // check IsSmall from empty database - fetchedBlobovniczaID, err := metaIsSmall(db, object.AddressOf(raw1)) - require.NoError(t, err) - require.Nil(t, fetchedBlobovniczaID) - - // put one object with blobovniczaID - err = metaPut(db, raw1, &blobovniczaID) - require.NoError(t, err) - - // put one object without blobovniczaID - err = putBig(db, raw2) - require.NoError(t, err) - - // check IsSmall for object without blobovniczaID - fetchedBlobovniczaID, err = metaIsSmall(db, object.AddressOf(raw2)) - require.NoError(t, err) - require.Nil(t, fetchedBlobovniczaID) - - // check IsSmall for object with blobovniczaID - fetchedBlobovniczaID, err = metaIsSmall(db, object.AddressOf(raw1)) - require.NoError(t, err) - require.Equal(t, &blobovniczaID, fetchedBlobovniczaID) -} - -func metaIsSmall(db *meta.DB, addr oid.Address) (*blobovnicza.ID, error) { - var isSmallPrm meta.IsSmallPrm - isSmallPrm.WithAddress(addr) - - r, err := db.IsSmall(isSmallPrm) - return r.BlobovniczaID(), err -} diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go new file mode 100644 index 000000000..ce893cffd --- /dev/null +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -0,0 +1,53 @@ +package meta + +import ( + "github.com/nspcc-dev/neo-go/pkg/util/slice" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + "go.etcd.io/bbolt" +) + +// StorageIDPrm groups the parameters of StorageID operation. +type StorageIDPrm struct { + addr oid.Address +} + +// StorageIDRes groups the resulting values of StorageID operation. +type StorageIDRes struct { + id []byte +} + +// SetAddress is a StorageID option to set the object address to check. +func (p *StorageIDPrm) SetAddress(addr oid.Address) { + p.addr = addr +} + +// StorageID returns storage ID. +func (r StorageIDRes) StorageID() []byte { + return r.id +} + +// StorageID returns storage descriptor for objects from the blobstor. +// It is put together with the object can makes get/delete operation faster. +func (db *DB) StorageID(prm StorageIDPrm) (res StorageIDRes, err error) { + err = db.boltDB.View(func(tx *bbolt.Tx) error { + res.id, err = db.storageID(tx, prm.addr) + + return err + }) + + return +} + +func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) ([]byte, error) { + smallBucket := tx.Bucket(smallBucketName(addr.Container())) + if smallBucket == nil { + return nil, nil + } + + storageID := smallBucket.Get(objectKey(addr.Object())) + if storageID == nil { + return nil, nil + } + + return slice.Copy(storageID), nil +} diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go new file mode 100644 index 000000000..aa22c2eca --- /dev/null +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -0,0 +1,50 @@ +package meta_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/core/object" + meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + "github.com/stretchr/testify/require" +) + +func TestDB_IsSmall(t *testing.T) { + db := newDB(t) + + raw1 := generateObject(t) + raw2 := generateObject(t) + + storageID := []byte{1, 2, 3, 4} + + // check StorageID from empty database + fetchedStorageID, err := metaStorageID(db, object.AddressOf(raw1)) + require.NoError(t, err) + require.Nil(t, fetchedStorageID) + + // put one object with storageID + err = metaPut(db, raw1, storageID) + require.NoError(t, err) + + // put one object without storageID + err = putBig(db, raw2) + require.NoError(t, err) + + // check StorageID for object without storageID + fetchedStorageID, err = metaStorageID(db, object.AddressOf(raw2)) + require.NoError(t, err) + require.Nil(t, fetchedStorageID) + + // check StorageID for object with storageID + fetchedStorageID, err = metaStorageID(db, object.AddressOf(raw1)) + require.NoError(t, err) + require.Equal(t, storageID, fetchedStorageID) +} + +func metaStorageID(db *meta.DB, addr oid.Address) ([]byte, error) { + var sidPrm meta.StorageIDPrm + sidPrm.SetAddress(addr) + + r, err := db.StorageID(sidPrm) + return r.StorageID(), err +} diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index b9d3e86b7..78b4ed3f4 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -8,7 +8,7 @@ import ( ) // version contains current metabase version. -const version = 0 +const version = 1 var versionKey = []byte("version") diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 4e27b9b4a..7ca5f8a38 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/nspcc-dev/neofs-node/pkg/core/object" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" @@ -150,7 +149,7 @@ func (s *Shard) refillMetabase() error { obj := objectSDK.New() - return blobstor.IterateBinaryObjects(s.blobStor, func(addr oid.Address, data []byte, blzID *blobovnicza.ID) error { + return blobstor.IterateBinaryObjects(s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { if err := obj.Unmarshal(data); err != nil { s.log.Warn("could not unmarshal object", zap.Stringer("address", addr), @@ -206,7 +205,7 @@ func (s *Shard) refillMetabase() error { var mPrm meta.PutPrm mPrm.SetObject(obj) - mPrm.SetBlobovniczaID(blzID) + mPrm.SetStorageID(descriptor) _, err := s.metaBase.Put(mPrm) if err != nil && !meta.IsErrRemoved(err) && !errors.Is(err, object.ErrObjectIsExpired) { diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 140b54a54..e5e9ded75 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -1,7 +1,6 @@ package shard import ( - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" @@ -36,7 +35,7 @@ func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { ln := len(prm.addr) - smalls := make(map[oid.Address]*blobovnicza.ID, ln) + smalls := make(map[oid.Address][]byte, ln) for i := range prm.addr { if s.hasWriteCache() { @@ -46,10 +45,10 @@ func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { } } - var sPrm meta.IsSmallPrm - sPrm.WithAddress(prm.addr[i]) + var sPrm meta.StorageIDPrm + sPrm.SetAddress(prm.addr[i]) - res, err := s.metaBase.IsSmall(sPrm) + res, err := s.metaBase.StorageID(sPrm) if err != nil { s.log.Debug("can't get blobovniczaID from metabase", zap.Stringer("object", prm.addr[i]), @@ -58,8 +57,8 @@ func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { continue } - if res.BlobovniczaID() != nil { - smalls[prm.addr[i]] = res.BlobovniczaID() + if res.StorageID() != nil { + smalls[prm.addr[i]] = res.StorageID() } } @@ -72,31 +71,14 @@ func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { } for i := range prm.addr { // delete small object - if id, ok := smalls[prm.addr[i]]; ok { - var delSmallPrm common.DeletePrm - delSmallPrm.Address = prm.addr[i] - delSmallPrm.BlobovniczaID = id + var delPrm common.DeletePrm + delPrm.Address = prm.addr[i] + id := smalls[prm.addr[i]] + delPrm.StorageID = id - _, err = s.blobStor.Delete(delSmallPrm) - if err != nil { - s.log.Debug("can't remove small object from blobStor", - zap.Stringer("object_address", prm.addr[i]), - zap.String("error", err.Error())) - } - - continue - } - - var id blobovnicza.ID - - // delete big object - var delBigPrm common.DeletePrm - delBigPrm.Address = prm.addr[i] - delBigPrm.BlobovniczaID = &id - - _, err = s.blobStor.Delete(delBigPrm) + _, err = s.blobStor.Delete(delPrm) if err != nil { - s.log.Debug("can't remove big object from blobStor", + s.log.Debug("can't remove small object from blobStor", zap.Stringer("object_address", prm.addr[i]), zap.String("error", err.Error())) } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 457833b9d..7beb6bdac 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -3,7 +3,6 @@ package shard import ( "fmt" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" @@ -16,7 +15,7 @@ import ( // storFetcher is a type to unify object fetching mechanism in `fetchObjectData` // method. It represents generalization of `getSmall` and `getBig` methods. -type storFetcher = func(stor *blobstor.BlobStor, id *blobovnicza.ID) (*objectSDK.Object, error) +type storFetcher = func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) // GetPrm groups the parameters of Get operation. type GetPrm struct { @@ -62,10 +61,10 @@ func (r GetRes) HasMeta() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (s *Shard) Get(prm GetPrm) (GetRes, error) { - cb := func(stor *blobstor.BlobStor, id *blobovnicza.ID) (*objectSDK.Object, error) { + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getPrm common.GetPrm getPrm.Address = prm.addr - getPrm.BlobovniczaID = id + getPrm.StorageID = id res, err := stor.Get(getPrm) if err != nil { @@ -131,21 +130,15 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, return nil, false, errNotFound } - var mPrm meta.IsSmallPrm - mPrm.WithAddress(addr) + var mPrm meta.StorageIDPrm + mPrm.SetAddress(addr) - mRes, err := s.metaBase.IsSmall(mPrm) + mRes, err := s.metaBase.StorageID(mPrm) if err != nil { return nil, true, fmt.Errorf("can't fetch blobovnicza id from metabase: %w", err) } - blobovniczaID := mRes.BlobovniczaID() - if blobovniczaID == nil { - var id blobovnicza.ID - blobovniczaID = &id - } - - res, err = cb(s.blobStor, blobovniczaID) + res, err = cb(s.blobStor, mRes.StorageID()) return res, true, err } diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index c8834ae01..cee7a9d0c 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -61,7 +61,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { if !m.NoMetabase() { var pPrm meta.PutPrm pPrm.SetObject(prm.obj) - pPrm.SetBlobovniczaID(res.BlobovniczaID) + pPrm.SetStorageID(res.StorageID) if _, err := s.metaBase.Put(pPrm); err != nil { // may we need to handle this case in a special way // since the object has been successfully written to BlobStor diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index b3160d02b..8131767f2 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -1,7 +1,6 @@ package shard import ( - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" @@ -67,12 +66,12 @@ func (r RngRes) HasMeta() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (s *Shard) GetRange(prm RngPrm) (RngRes, error) { - cb := func(stor *blobstor.BlobStor, id *blobovnicza.ID) (*object.Object, error) { + cb := func(stor *blobstor.BlobStor, id []byte) (*object.Object, error) { var getRngPrm common.GetRangePrm getRngPrm.Address = prm.addr getRngPrm.Range.SetOffset(prm.off) getRngPrm.Range.SetLength(prm.ln) - getRngPrm.BlobovniczaID = id + getRngPrm.StorageID = id res, err := stor.GetRange(getRngPrm) if err != nil { diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index a626bf715..4951c2e03 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -5,7 +5,6 @@ import ( "time" "github.com/mr-tron/base58" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" @@ -228,7 +227,7 @@ func (c *cache) flushWorker(num int) { // writeObject is used to write object directly to the main storage. func (c *cache) writeObject(obj *object.Object, metaOnly bool) error { - var id *blobovnicza.ID + var descriptor []byte if !metaOnly { var prm common.PutPrm @@ -239,12 +238,12 @@ func (c *cache) writeObject(obj *object.Object, metaOnly bool) error { return err } - id = res.BlobovniczaID + descriptor = res.StorageID } var pPrm meta.PutPrm pPrm.SetObject(obj) - pPrm.SetBlobovniczaID(id) + pPrm.SetStorageID(descriptor) _, err := c.metabase.Put(pPrm) return err