From a9c4ba62c348ed3a47d91b92697012d4b442869b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:48:24 +0300 Subject: [PATCH] [#248] metabase: Remove storage group bucket Backwards compatible change, so no version increase. Signed-off-by: Evgenii Stratonikov --- .../metabase/containers_test.go | 1 - pkg/local_object_storage/metabase/delete.go | 2 - .../metabase/exists_test.go | 12 ---- pkg/local_object_storage/metabase/get.go | 6 -- pkg/local_object_storage/metabase/get_test.go | 12 ---- .../metabase/iterators.go | 1 - .../metabase/iterators_test.go | 1 - pkg/local_object_storage/metabase/list.go | 2 - .../metabase/list_test.go | 9 +-- .../metabase/lock_test.go | 1 - pkg/local_object_storage/metabase/put.go | 2 - pkg/local_object_storage/metabase/select.go | 9 +-- .../metabase/select_test.go | 58 ------------------- pkg/local_object_storage/metabase/util.go | 14 ++--- 14 files changed, 8 insertions(+), 122 deletions(-) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index ef2bba638..a0be2c743 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -100,7 +100,6 @@ func TestDB_ContainersCount(t *testing.T) { }{ {R, objectSDK.TypeRegular}, {T, objectSDK.TypeTombstone}, - {SG, objectSDK.TypeStorageGroup}, {L, objectSDK.TypeLock}, } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 990d3997b..5340f5d08 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -345,8 +345,6 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error bucketName = primaryBucketName(cnr, bucketName) case objectSDK.TypeTombstone: bucketName = tombstoneBucketName(cnr, bucketName) - case objectSDK.TypeStorageGroup: - bucketName = storageGroupBucketName(cnr, bucketName) case objectSDK.TypeLock: bucketName = bucketNameLockers(cnr, bucketName) default: diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index e344e9ee8..034bc0005 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -57,18 +57,6 @@ func TestDB_Exists(t *testing.T) { require.True(t, exists) }) - t.Run("storage group object", func(t *testing.T) { - sg := testutil.GenerateObject() - sg.SetType(objectSDK.TypeStorageGroup) - - err := putBig(db, sg) - require.NoError(t, err) - - exists, err := metaExists(db, object.AddressOf(sg)) - require.NoError(t, err) - require.True(t, exists) - }) - t.Run("lock object", func(t *testing.T) { lock := testutil.GenerateObject() lock.SetType(objectSDK.TypeLock) diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index fff32d6ad..e76b9d4a7 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -106,12 +106,6 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b return obj, obj.Unmarshal(data) } - // if not found then check in storage group index - data = getFromBucket(tx, storageGroupBucketName(cnr, bucketName), key) - if len(data) != 0 { - return obj, obj.Unmarshal(data) - } - // if not found then check in locker index data = getFromBucket(tx, bucketNameLockers(cnr, bucketName), key) if len(data) != 0 { diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index 4d2a7682d..c19fea8d8 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -55,18 +55,6 @@ func TestDB_Get(t *testing.T) { require.Equal(t, raw.CutPayload(), newObj) }) - t.Run("put storage group object", func(t *testing.T) { - raw.SetType(objectSDK.TypeStorageGroup) - raw.SetID(oidtest.ID()) - - err := putBig(db, raw) - require.NoError(t, err) - - newObj, err := metaGet(db, object.AddressOf(raw), false) - require.NoError(t, err) - require.Equal(t, raw.CutPayload(), newObj) - }) - t.Run("put lock object", func(t *testing.T) { raw.SetType(objectSDK.TypeLock) raw.SetID(oidtest.ID()) diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 3c5888e1c..4c9dc782c 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -186,7 +186,6 @@ func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID) error) error { switch postfix { case primaryPrefix, - storageGroupPrefix, lockersPrefix, tombstonePrefix: default: diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index e7d6ad04f..69bf2bee5 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -26,7 +26,6 @@ func TestDB_IterateExpired(t *testing.T) { for _, typ := range []object.Type{ object.TypeRegular, object.TypeTombstone, - object.TypeStorageGroup, object.TypeLock, } { mAlive[typ] = putWithExpiration(t, db, typ, epoch) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 93b7efb99..7fe1040bb 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -108,8 +108,6 @@ loop: switch prefix { case primaryPrefix: objType = object.TypeRegular - case storageGroupPrefix: - objType = object.TypeStorageGroup case lockersPrefix: objType = object.TypeLock case tombstonePrefix: diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index ab2d9d75d..07a0d80f8 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -70,7 +70,7 @@ func TestLisObjectsWithCursor(t *testing.T) { const ( containers = 5 - total = containers * 5 // regular + ts + sg + child + lock + total = containers * 4 // regular + ts + child + lock ) expected := make([]object.AddressWithType, 0, total) @@ -93,13 +93,6 @@ func TestLisObjectsWithCursor(t *testing.T) { require.NoError(t, err) expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeTombstone}) - // add one storage group - obj = testutil.GenerateObjectWithCID(containerID) - obj.SetType(objectSDK.TypeStorageGroup) - err = putBig(db, obj) - require.NoError(t, err) - expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeStorageGroup}) - // add one lock obj = testutil.GenerateObjectWithCID(containerID) obj.SetType(objectSDK.TypeLock) diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 1d6ea6ffb..7b62841dc 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -28,7 +28,6 @@ func TestDB_Lock(t *testing.T) { t.Run("(ir)regular", func(t *testing.T) { for _, typ := range [...]object.Type{ object.TypeTombstone, - object.TypeStorageGroup, object.TypeLock, object.TypeRegular, } { diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 2c78bda0f..bc6520a05 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -205,8 +205,6 @@ func putUniqueIndexes( bucketName = primaryBucketName(cnr, bucketName) case objectSDK.TypeTombstone: bucketName = tombstoneBucketName(cnr, bucketName) - case objectSDK.TypeStorageGroup: - bucketName = storageGroupBucketName(cnr, bucketName) case objectSDK.TypeLock: bucketName = bucketNameLockers(cnr, bucketName) default: diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index ecd83f863..a4b14f77c 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -151,7 +151,6 @@ func (db *DB) selectAll(tx *bbolt.Tx, cnr cid.ID, to map[string]int) { bucketName := make([]byte, bucketKeySize) selectAllFromBucket(tx, primaryBucketName(cnr, bucketName), to, 0) selectAllFromBucket(tx, tombstoneBucketName(cnr, bucketName), to, 0) - selectAllFromBucket(tx, storageGroupBucketName(cnr, bucketName), to, 0) selectAllFromBucket(tx, parentBucketName(cnr, bucketName), to, 0) selectAllFromBucket(tx, bucketNameLockers(cnr, bucketName), to, 0) } @@ -206,7 +205,6 @@ func (db *DB) selectFastFilter( case v2object.FilterPropertyPhy: selectAllFromBucket(tx, primaryBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, tombstoneBucketName(cnr, bucketName), to, fNum) - selectAllFromBucket(tx, storageGroupBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, bucketNameLockers(cnr, bucketName), to, fNum) default: // user attribute bucketName := attributeBucketName(cnr, f.Header(), bucketName) @@ -220,10 +218,9 @@ func (db *DB) selectFastFilter( } var mBucketNaming = map[string][]func(cid.ID, []byte) []byte{ - v2object.TypeRegular.String(): {primaryBucketName, parentBucketName}, - v2object.TypeTombstone.String(): {tombstoneBucketName}, - v2object.TypeStorageGroup.String(): {storageGroupBucketName}, - v2object.TypeLock.String(): {bucketNameLockers}, + v2object.TypeRegular.String(): {primaryBucketName, parentBucketName}, + v2object.TypeTombstone.String(): {tombstoneBucketName}, + v2object.TypeLock.String(): {bucketNameLockers}, } func allBucketNames(cnr cid.ID) (names [][]byte) { diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 386797529..dab4c028d 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -157,11 +157,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { err = putBig(db, ts) require.NoError(t, err) - sg := testutil.GenerateObjectWithCID(cnr) - sg.SetType(objectSDK.TypeStorageGroup) - err = putBig(db, sg) - require.NoError(t, err) - leftChild := testutil.GenerateObjectWithCID(cnr) leftChild.InitRelations() err = putBig(db, leftChild) @@ -210,7 +205,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(small), object.AddressOf(ts), - object.AddressOf(sg), object.AddressOf(leftChild), object.AddressOf(rightChild), object.AddressOf(link), @@ -237,7 +231,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeRegular.String(), objectSDK.MatchStringNotEqual) testSelect(t, db, cnr, fs, object.AddressOf(ts), - object.AddressOf(sg), object.AddressOf(lock), ) @@ -259,29 +252,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { object.AddressOf(rightChild), object.AddressOf(link), object.AddressOf(parent), - object.AddressOf(sg), - object.AddressOf(lock), - ) - - fs = objectSDK.SearchFilters{} - fs.AddFilter(v2object.FilterHeaderObjectType, "", objectSDK.MatchNotPresent) - testSelect(t, db, cnr, fs) - }) - - t.Run("storage group objects", func(t *testing.T) { - fs := objectSDK.SearchFilters{} - fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeStorageGroup.String(), objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs, object.AddressOf(sg)) - - fs = objectSDK.SearchFilters{} - fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeStorageGroup.String(), objectSDK.MatchStringNotEqual) - testSelect(t, db, cnr, fs, - object.AddressOf(small), - object.AddressOf(leftChild), - object.AddressOf(rightChild), - object.AddressOf(link), - object.AddressOf(parent), - object.AddressOf(ts), object.AddressOf(lock), ) @@ -313,7 +283,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(small), object.AddressOf(ts), - object.AddressOf(sg), object.AddressOf(leftChild), object.AddressOf(rightChild), object.AddressOf(link), @@ -550,11 +519,6 @@ func TestDB_SelectObjectID(t *testing.T) { err = putBig(db, ts) require.NoError(t, err) - sg := testutil.GenerateObjectWithCID(cnr) - sg.SetType(objectSDK.TypeStorageGroup) - err = putBig(db, sg) - require.NoError(t, err) - lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) err = putBig(db, lock) @@ -576,7 +540,6 @@ func TestDB_SelectObjectID(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(regular), object.AddressOf(parent), - object.AddressOf(sg), object.AddressOf(ts), object.AddressOf(lock), ) @@ -593,7 +556,6 @@ func TestDB_SelectObjectID(t *testing.T) { fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, id) testSelect(t, db, cnr, fs, object.AddressOf(parent), - object.AddressOf(sg), object.AddressOf(ts), object.AddressOf(lock), ) @@ -611,24 +573,6 @@ func TestDB_SelectObjectID(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(regular), object.AddressOf(parent), - object.AddressOf(sg), - object.AddressOf(lock), - ) - }) - - t.Run("storage group objects", func(t *testing.T) { - id, _ := sg.ID() - - fs := objectSDK.SearchFilters{} - fs.AddObjectIDFilter(objectSDK.MatchStringEqual, id) - testSelect(t, db, cnr, fs, object.AddressOf(sg)) - - fs = objectSDK.SearchFilters{} - fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, id) - testSelect(t, db, cnr, fs, - object.AddressOf(regular), - object.AddressOf(parent), - object.AddressOf(ts), object.AddressOf(lock), ) }) @@ -644,7 +588,6 @@ func TestDB_SelectObjectID(t *testing.T) { fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, id) testSelect(t, db, cnr, fs, object.AddressOf(regular), - object.AddressOf(sg), object.AddressOf(ts), object.AddressOf(lock), ) @@ -662,7 +605,6 @@ func TestDB_SelectObjectID(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(regular), object.AddressOf(parent), - object.AddressOf(sg), object.AddressOf(ts), ) }) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index b60c97fd7..c50fd051f 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -65,10 +65,10 @@ const ( // Key: object ID // Value: marshalled object lockersPrefix - // storageGroupPrefix is used for prefixing buckets containing objects of STORAGEGROUP type. + // _ is unused. Previous usage was for prefixing buckets containing objects of STORAGEGROUP type. // Key: object ID // Value: marshaled object - storageGroupPrefix + _ // tombstonePrefix is used for prefixing buckets containing objects of TOMBSTONE type. // Key: object ID // Value: marshaled object @@ -138,11 +138,6 @@ func tombstoneBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, tombstonePrefix, key) } -// storageGroupBucketName returns _SG. -func storageGroupBucketName(cnr cid.ID, key []byte) []byte { - return bucketName(cnr, storageGroupPrefix, key) -} - // smallBucketName returns _small. func smallBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, smallPrefix, key) @@ -231,15 +226,14 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object panic("empty object list in firstIrregularObjectType") } - var keys [3][1 + cidSize]byte + var keys [2][1 + cidSize]byte irregularTypeBuckets := [...]struct { typ object.Type name []byte }{ {object.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, - {object.TypeStorageGroup, storageGroupBucketName(idCnr, keys[1][:])}, - {object.TypeLock, bucketNameLockers(idCnr, keys[2][:])}, + {object.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, } for i := range objs {