[#1974] shard: Do not panic in degraded mode

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-10-26 09:12:09 +03:00 committed by fyrchik
parent 7395ab8ef7
commit 1e6588e761
8 changed files with 56 additions and 4 deletions

View file

@ -1,6 +1,8 @@
package engine package engine
import ( import (
"errors"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.uber.org/zap" "go.uber.org/zap"
@ -75,7 +77,7 @@ func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRe
csPrm.SetContainerID(prm.cnr) csPrm.SetContainerID(prm.cnr)
csRes, err := sh.Shard.ContainerSize(csPrm) csRes, err := sh.Shard.ContainerSize(csPrm)
if err != nil { if err != nil && !errors.Is(err, shard.ErrDegradedMode) {
e.reportShardError(sh, "can't get container size", err, e.reportShardError(sh, "can't get container size", err,
zap.Stringer("container_id", prm.cnr), zap.Stringer("container_id", prm.cnr),
) )
@ -124,7 +126,9 @@ func (e *StorageEngine) listContainers() (ListContainersRes, error) {
e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
res, err := sh.Shard.ListContainers(shard.ListContainersPrm{}) res, err := sh.Shard.ListContainers(shard.ListContainersPrm{})
if err != nil { if err != nil {
if !errors.Is(err, shard.ErrDegradedMode) {
e.reportShardError(sh, "can't get list of containers", err) e.reportShardError(sh, "can't get list of containers", err)
}
return false return false
} }

View file

@ -125,7 +125,7 @@ mainLoop:
// because ListWithCursor works only with the metabase. // because ListWithCursor works only with the metabase.
listRes, err := sh.ListWithCursor(listPrm) listRes, err := sh.ListWithCursor(listPrm)
if err != nil { if err != nil {
if errors.Is(err, meta.ErrEndOfListing) { if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) {
continue mainLoop continue mainLoop
} }
return res, err return res, err

View file

@ -1,6 +1,8 @@
package engine package engine
import ( import (
"errors"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object"
@ -109,7 +111,9 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) {
e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
res, err := sh.List() // consider limit result of shard iterator res, err := sh.List() // consider limit result of shard iterator
if err != nil { if err != nil {
if !errors.Is(err, shard.ErrDegradedMode) {
e.reportShardError(sh, "could not select objects from shard", err) e.reportShardError(sh, "could not select objects from shard", err)
}
} else { } else {
for _, addr := range res.AddressList() { // save only unique values for _, addr := range res.AddressList() { // save only unique values
if _, ok := uniqueMap[addr.EncodeToString()]; !ok { if _, ok := uniqueMap[addr.EncodeToString()]; !ok {

View file

@ -11,6 +11,10 @@ func (db *DB) Containers() (list []cid.ID, err error) {
db.modeMtx.RLock() db.modeMtx.RLock()
defer db.modeMtx.RUnlock() defer db.modeMtx.RUnlock()
if db.mode.NoMetabase() {
return nil, ErrDegradedMode
}
err = db.boltDB.View(func(tx *bbolt.Tx) error { err = db.boltDB.View(func(tx *bbolt.Tx) error {
list, err = db.containers(tx) list, err = db.containers(tx)

View file

@ -92,6 +92,10 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) {
db.modeMtx.RLock() db.modeMtx.RLock()
defer db.modeMtx.RUnlock() defer db.modeMtx.RUnlock()
if db.mode.NoMetabase() {
return InhumeRes{}, ErrDegradedMode
}
currEpoch := db.epochState.CurrentEpoch() currEpoch := db.epochState.CurrentEpoch()
var inhumed uint64 var inhumed uint64

View file

@ -23,6 +23,13 @@ func (r ContainerSizeRes) Size() uint64 {
} }
func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) {
s.m.RLock()
defer s.m.RUnlock()
if s.info.Mode.NoMetabase() {
return ContainerSizeRes{}, ErrDegradedMode
}
size, err := s.metaBase.ContainerSize(prm.cnr) size, err := s.metaBase.ContainerSize(prm.cnr)
if err != nil { if err != nil {
return ContainerSizeRes{}, fmt.Errorf("could not get container size: %w", err) return ContainerSizeRes{}, fmt.Errorf("could not get container size: %w", err)

View file

@ -284,6 +284,11 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) {
for { for {
log.Debug("iterating tombstones") log.Debug("iterating tombstones")
if s.GetMode().NoMetabase() {
s.log.Debug("shard is in a degraded mode, skip collecting expired tombstones")
return
}
err := s.metaBase.IterateOverGraveyard(iterPrm) err := s.metaBase.IterateOverGraveyard(iterPrm)
if err != nil { if err != nil {
log.Error("iterator over graveyard failed", zap.Error(err)) log.Error("iterator over graveyard failed", zap.Error(err))
@ -327,6 +332,10 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) {
} }
func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, typeCond func(object.Type) bool) ([]oid.Address, error) { func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, typeCond func(object.Type) bool) ([]oid.Address, error) {
if s.GetMode().NoMetabase() {
return nil, ErrDegradedMode
}
var expired []oid.Address var expired []oid.Address
err := s.metaBase.IterateExpired(epoch, func(expiredObject *meta.ExpiredObject) error { err := s.metaBase.IterateExpired(epoch, func(expiredObject *meta.ExpiredObject) error {
@ -351,6 +360,10 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, typeCond fu
// //
// Does not modify tss. // Does not modify tss.
func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) {
if s.GetMode().NoMetabase() {
return
}
// Mark tombstones as garbage. // Mark tombstones as garbage.
var pInhume meta.InhumePrm var pInhume meta.InhumePrm
@ -385,6 +398,10 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) {
// HandleExpiredLocks unlocks all objects which were locked by lockers. // HandleExpiredLocks unlocks all objects which were locked by lockers.
// If successful, marks lockers themselves as garbage. // If successful, marks lockers themselves as garbage.
func (s *Shard) HandleExpiredLocks(lockers []oid.Address) { func (s *Shard) HandleExpiredLocks(lockers []oid.Address) {
if s.GetMode().NoMetabase() {
return
}
err := s.metaBase.FreeLockedBy(lockers) err := s.metaBase.FreeLockedBy(lockers)
if err != nil { if err != nil {
s.log.Warn("failure to unlock objects", s.log.Warn("failure to unlock objects",
@ -412,6 +429,10 @@ func (s *Shard) HandleExpiredLocks(lockers []oid.Address) {
// HandleDeletedLocks unlocks all objects which were locked by lockers. // HandleDeletedLocks unlocks all objects which were locked by lockers.
func (s *Shard) HandleDeletedLocks(lockers []oid.Address) { func (s *Shard) HandleDeletedLocks(lockers []oid.Address) {
if s.GetMode().NoMetabase() {
return
}
err := s.metaBase.FreeLockedBy(lockers) err := s.metaBase.FreeLockedBy(lockers)
if err != nil { if err != nil {
s.log.Warn("failure to unlock objects", s.log.Warn("failure to unlock objects",

View file

@ -64,6 +64,10 @@ func (r ListWithCursorRes) Cursor() *Cursor {
// List returns all objects physically stored in the Shard. // List returns all objects physically stored in the Shard.
func (s *Shard) List() (res SelectRes, err error) { func (s *Shard) List() (res SelectRes, err error) {
if s.GetMode().NoMetabase() {
return SelectRes{}, ErrDegradedMode
}
lst, err := s.metaBase.Containers() lst, err := s.metaBase.Containers()
if err != nil { if err != nil {
return res, fmt.Errorf("can't list stored containers: %w", err) return res, fmt.Errorf("can't list stored containers: %w", err)
@ -93,6 +97,10 @@ func (s *Shard) List() (res SelectRes, err error) {
} }
func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) {
if s.GetMode().NoMetabase() {
return ListContainersRes{}, ErrDegradedMode
}
containers, err := s.metaBase.Containers() containers, err := s.metaBase.Containers()
if err != nil { if err != nil {
return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err) return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err)