From 1e6588e7612a04df0807b8d972833c668aa9f53d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Oct 2022 09:12:09 +0300 Subject: [PATCH] [#1974] shard: Do not panic in degraded mode Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/container.go | 8 +++++-- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/select.go | 6 +++++- .../metabase/containers.go | 4 ++++ pkg/local_object_storage/metabase/inhume.go | 4 ++++ pkg/local_object_storage/shard/container.go | 7 +++++++ pkg/local_object_storage/shard/gc.go | 21 +++++++++++++++++++ pkg/local_object_storage/shard/list.go | 8 +++++++ 8 files changed, 56 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 1a8dbf16a4..fcda01c298 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -1,6 +1,8 @@ package engine import ( + "errors" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "go.uber.org/zap" @@ -75,7 +77,7 @@ func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRe csPrm.SetContainerID(prm.cnr) csRes, err := sh.Shard.ContainerSize(csPrm) - if err != nil { + if err != nil && !errors.Is(err, shard.ErrDegradedMode) { e.reportShardError(sh, "can't get container size", err, zap.Stringer("container_id", prm.cnr), ) @@ -124,7 +126,9 @@ func (e *StorageEngine) listContainers() (ListContainersRes, error) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { res, err := sh.Shard.ListContainers(shard.ListContainersPrm{}) if err != nil { - e.reportShardError(sh, "can't get list of containers", err) + if !errors.Is(err, shard.ErrDegradedMode) { + e.reportShardError(sh, "can't get list of containers", err) + } return false } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index e3f32f9342..0a19e537f4 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -125,7 +125,7 @@ mainLoop: // because ListWithCursor works only with the metabase. listRes, err := sh.ListWithCursor(listPrm) if err != nil { - if errors.Is(err, meta.ErrEndOfListing) { + if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { continue mainLoop } return res, err diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 0e5a8ca227..f06e6a60f0 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -1,6 +1,8 @@ package engine import ( + "errors" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -109,7 +111,9 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { res, err := sh.List() // consider limit result of shard iterator if err != nil { - e.reportShardError(sh, "could not select objects from shard", err) + if !errors.Is(err, shard.ErrDegradedMode) { + e.reportShardError(sh, "could not select objects from shard", err) + } } else { for _, addr := range res.AddressList() { // save only unique values if _, ok := uniqueMap[addr.EncodeToString()]; !ok { diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index de3966d663..278f4adf38 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -11,6 +11,10 @@ func (db *DB) Containers() (list []cid.ID, err error) { db.modeMtx.RLock() defer db.modeMtx.RUnlock() + if db.mode.NoMetabase() { + return nil, ErrDegradedMode + } + err = db.boltDB.View(func(tx *bbolt.Tx) error { list, err = db.containers(tx) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 296e3bd6c9..3dcd2c2307 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -92,6 +92,10 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { db.modeMtx.RLock() defer db.modeMtx.RUnlock() + if db.mode.NoMetabase() { + return InhumeRes{}, ErrDegradedMode + } + currEpoch := db.epochState.CurrentEpoch() var inhumed uint64 diff --git a/pkg/local_object_storage/shard/container.go b/pkg/local_object_storage/shard/container.go index dad55ddc90..a8684b58a0 100644 --- a/pkg/local_object_storage/shard/container.go +++ b/pkg/local_object_storage/shard/container.go @@ -23,6 +23,13 @@ func (r ContainerSizeRes) Size() uint64 { } func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return ContainerSizeRes{}, ErrDegradedMode + } + size, err := s.metaBase.ContainerSize(prm.cnr) if err != nil { return ContainerSizeRes{}, fmt.Errorf("could not get container size: %w", err) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 95a7ddba2d..c19e89cd49 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -284,6 +284,11 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { for { log.Debug("iterating tombstones") + if s.GetMode().NoMetabase() { + s.log.Debug("shard is in a degraded mode, skip collecting expired tombstones") + return + } + err := s.metaBase.IterateOverGraveyard(iterPrm) if err != nil { log.Error("iterator over graveyard failed", zap.Error(err)) @@ -327,6 +332,10 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { } func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, typeCond func(object.Type) bool) ([]oid.Address, error) { + if s.GetMode().NoMetabase() { + return nil, ErrDegradedMode + } + var expired []oid.Address err := s.metaBase.IterateExpired(epoch, func(expiredObject *meta.ExpiredObject) error { @@ -351,6 +360,10 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, typeCond fu // // Does not modify tss. func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { + if s.GetMode().NoMetabase() { + return + } + // Mark tombstones as garbage. var pInhume meta.InhumePrm @@ -385,6 +398,10 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { // HandleExpiredLocks unlocks all objects which were locked by lockers. // If successful, marks lockers themselves as garbage. func (s *Shard) HandleExpiredLocks(lockers []oid.Address) { + if s.GetMode().NoMetabase() { + return + } + err := s.metaBase.FreeLockedBy(lockers) if err != nil { s.log.Warn("failure to unlock objects", @@ -412,6 +429,10 @@ func (s *Shard) HandleExpiredLocks(lockers []oid.Address) { // HandleDeletedLocks unlocks all objects which were locked by lockers. func (s *Shard) HandleDeletedLocks(lockers []oid.Address) { + if s.GetMode().NoMetabase() { + return + } + err := s.metaBase.FreeLockedBy(lockers) if err != nil { s.log.Warn("failure to unlock objects", diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 128847b4c0..5fc4a7d98d 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -64,6 +64,10 @@ func (r ListWithCursorRes) Cursor() *Cursor { // List returns all objects physically stored in the Shard. func (s *Shard) List() (res SelectRes, err error) { + if s.GetMode().NoMetabase() { + return SelectRes{}, ErrDegradedMode + } + lst, err := s.metaBase.Containers() if err != nil { return res, fmt.Errorf("can't list stored containers: %w", err) @@ -93,6 +97,10 @@ func (s *Shard) List() (res SelectRes, err error) { } func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { + if s.GetMode().NoMetabase() { + return ListContainersRes{}, ErrDegradedMode + } + containers, err := s.metaBase.Containers() if err != nil { return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err)