From 362f24953a7b91a0593922b6d17da1011853db09 Mon Sep 17 00:00:00 2001 From: Artem Tataurov Date: Thu, 2 Feb 2023 23:28:42 +0300 Subject: [PATCH] [#47] shard: Switch container size metric from physical to logical capacity Signed-off-by: Artem Tataurov --- CHANGELOG.md | 1 + pkg/local_object_storage/metabase/delete.go | 17 ++++++++--- pkg/local_object_storage/metabase/inhume.go | 33 +++++++++++++++++++-- pkg/local_object_storage/shard/delete.go | 7 +++-- pkg/local_object_storage/shard/gc.go | 21 +++++++++++++ pkg/local_object_storage/shard/inhume.go | 7 +++++ 6 files changed, 78 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9821edbd0..41ed456c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Changelog for FrostFS Node - Add command `frostfs-adm morph netmap-candidates` (#1889) ### Changed +- Change `frostfs_node_engine_container_size` to counting sizes of logical objects - `common.PrintVerbose` prints via `cobra.Command.Printf` (#1962) - Env prefix in configuration changed to `FROSTFS_*` (#43) - Link object is broadcast throughout the whole container now (#57) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 0204e98f2..511300b19 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -23,6 +23,7 @@ type DeleteRes struct { rawRemoved uint64 availableRemoved uint64 sizes []uint64 + availableSizes []uint64 } // AvailableObjectsRemoved returns the number of removed available @@ -36,11 +37,16 @@ func (d DeleteRes) RawObjectsRemoved() uint64 { return d.rawRemoved } -// RemovedObjectSizes returns the sizes of removed objects. -func (d DeleteRes) RemovedObjectSizes() []uint64 { +// RemovedPhysicalObjectSizes returns the sizes of removed physical objects. +func (d DeleteRes) RemovedPhysicalObjectSizes() []uint64 { return d.sizes } +// RemovedLogicalObjectSizes returns the sizes of removed logical objects. +func (d DeleteRes) RemovedLogicalObjectSizes() []uint64 { + return d.availableSizes +} + // SetAddresses is a Delete option to set the addresses of the objects to delete. // // Option is required. @@ -73,10 +79,11 @@ func (db *DB) Delete(prm DeletePrm) (DeleteRes, error) { var availableRemoved uint64 var err error var sizes = make([]uint64, len(prm.addrs)) + var availableSizes = make([]uint64, len(prm.addrs)) err = db.boltDB.Update(func(tx *bbolt.Tx) error { // We need to clear slice because tx can try to execute multiple times. - rawRemoved, availableRemoved, err = db.deleteGroup(tx, prm.addrs, sizes) + rawRemoved, availableRemoved, err = db.deleteGroup(tx, prm.addrs, sizes, availableSizes) return err }) if err == nil { @@ -90,6 +97,7 @@ func (db *DB) Delete(prm DeletePrm) (DeleteRes, error) { rawRemoved: rawRemoved, availableRemoved: availableRemoved, sizes: sizes, + availableSizes: availableSizes, }, err } @@ -99,7 +107,7 @@ func (db *DB) Delete(prm DeletePrm) (DeleteRes, error) { // objects that were stored. The second return value is a logical objects // removed number: objects that were available (without Tombstones, GCMarks // non-expired, etc.) -func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address, sizes []uint64) (uint64, uint64, error) { +func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address, sizes []uint64, availableSizes []uint64) (uint64, uint64, error) { refCounter := make(referenceCounter, len(addrs)) currEpoch := db.epochState.CurrentEpoch() @@ -119,6 +127,7 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address, sizes []uint64) (ui if available { availableDeleted++ + availableSizes[i] = size } } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 22e8e52b0..2d2107b67 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -7,6 +7,7 @@ import ( "github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "github.com/TrueCloudLab/frostfs-sdk-go/client/status" + cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/TrueCloudLab/frostfs-sdk-go/object" oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -23,10 +24,17 @@ type InhumePrm struct { forceRemoval bool } +// DeletionInfo contains details on deleted object. +type DeletionInfo struct { + Size uint64 + CID cid.ID +} + // InhumeRes encapsulates results of Inhume operation. type InhumeRes struct { deletedLockObj []oid.Address availableImhumed uint64 + deletionDetails []DeletionInfo } // AvailableInhumed return number of available object @@ -42,6 +50,27 @@ func (i InhumeRes) DeletedLockObjects() []oid.Address { return i.deletedLockObj } +// GetDeletionInfoLength returns amount of stored elements +// in deleted sizes array. +func (i InhumeRes) GetDeletionInfoLength() int { + return len(i.deletionDetails) +} + +// GetDeletionInfoByIndex returns both deleted object sizes and +// associated container ID by index. +func (i InhumeRes) GetDeletionInfoByIndex(target int) DeletionInfo { + return i.deletionDetails[target] +} + +// StoreDeletionInfo stores size of deleted object and associated container ID +// in corresponding arrays. +func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64) { + i.deletionDetails = append(i.deletionDetails, DeletionInfo{ + Size: deletedSize, + CID: containerID, + }) +} + // SetAddresses sets a list of object addresses that should be inhumed. func (p *InhumePrm) SetAddresses(addrs ...oid.Address) { p.target = addrs @@ -164,10 +193,10 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { obj, err := db.get(tx, prm.target[i], buf, false, true, currEpoch) targetKey := addressKey(prm.target[i], buf) if err == nil { + containerID, _ := obj.ContainerID() if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { - // object is available, decrement the - // logical counter inhumed++ + res.storeDeletionInfo(containerID, obj.PayloadSize()) } // if object is stored, and it is regular object then update bucket diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 9cb9cfe8b..0c2cfa265 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -83,9 +83,12 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { s.decObjectCounterBy(physical, res.RawObjectsRemoved()) s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) for i := range prm.addr { - removedPayload := res.RemovedObjectSizes()[i] + removedPayload := res.RemovedPhysicalObjectSizes()[i] totalRemovedPayload += removedPayload - s.addToContainerSize(prm.addr[i].Container().EncodeToString(), -int64(removedPayload)) + logicalRemovedPayload := res.RemovedLogicalObjectSizes()[i] + if logicalRemovedPayload > 0 { + s.addToContainerSize(prm.addr[i].Container().EncodeToString(), -int64(logicalRemovedPayload)) + } } s.addToPayloadSize(-int64(totalRemovedPayload)) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index fdfb7818e..11a190e79 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -268,6 +268,13 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { } s.decObjectCounterBy(logical, res.AvailableInhumed()) + + i := 0 + for i < res.GetDeletionInfoLength() { + delInfo := res.GetDeletionInfoByIndex(i) + s.addToContainerSize(delInfo.CID.EncodeToString(), -int64(delInfo.Size)) + i++ + } } func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { @@ -408,6 +415,13 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { s.decObjectCounterBy(logical, res.AvailableInhumed()) + i := 0 + for i < res.GetDeletionInfoLength() { + delInfo := res.GetDeletionInfoByIndex(i) + s.addToContainerSize(delInfo.CID.EncodeToString(), -int64(delInfo.Size)) + i++ + } + // drop just processed expired tombstones // from graveyard err = s.metaBase.DropGraves(tss) @@ -446,6 +460,13 @@ func (s *Shard) HandleExpiredLocks(lockers []oid.Address) { } s.decObjectCounterBy(logical, res.AvailableInhumed()) + + i := 0 + for i < res.GetDeletionInfoLength() { + delInfo := res.GetDeletionInfoByIndex(i) + s.addToContainerSize(delInfo.CID.EncodeToString(), -int64(delInfo.Size)) + i++ + } } // HandleDeletedLocks unlocks all objects which were locked by lockers. diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 5a84c2c98..bdb2c1ec8 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -111,6 +111,13 @@ func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) { s.decObjectCounterBy(logical, res.AvailableInhumed()) + i := 0 + for i < res.GetDeletionInfoLength() { + delInfo := res.GetDeletionInfoByIndex(i) + s.addToContainerSize(delInfo.CID.EncodeToString(), -int64(delInfo.Size)) + i++ + } + if deletedLockObjs := res.DeletedLockObjects(); len(deletedLockObjs) != 0 { s.deletedLockCallBack(context.Background(), deletedLockObjs) }