[#1549] engine: Disable shard on blobovnicza init failure

There is a need to support working w/o shard if it has problems with
blobovnicza tree.

Make `BlobStor.Init` to return new `ErrInitBlobovniczas` error. Remove
shard from storage engine's shard set if it returned this error from
`Init` call. So if some of the shards (but not all) return this error,
the node will be able to continue working without them.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-06-27 11:36:32 +03:00 committed by fyrchik
parent 40a56c6b42
commit e38b0aa4ba
2 changed files with 45 additions and 8 deletions

View file

@ -1,5 +1,10 @@
package blobstor package blobstor
import (
"errors"
"fmt"
)
// Open opens BlobStor. // Open opens BlobStor.
func (b *BlobStor) Open() error { func (b *BlobStor) Open() error {
b.log.Debug("opening...") b.log.Debug("opening...")
@ -7,13 +12,23 @@ func (b *BlobStor) Open() error {
return nil return nil
} }
// ErrInitBlobovniczas is returned when blobovnicza initialization fails.
var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stage")
// Init initializes internal data structures and system resources. // Init initializes internal data structures and system resources.
// //
// If BlobStor is already initialized, no action is taken. // If BlobStor is already initialized, no action is taken.
//
// Returns wrapped ErrInitBlobovniczas on blobovnicza tree's initializaiton failure.
func (b *BlobStor) Init() error { func (b *BlobStor) Init() error {
b.log.Debug("initializing...") b.log.Debug("initializing...")
return b.blobovniczas.init() err := b.blobovniczas.init()
if err != nil {
return fmt.Errorf("%w: %v", ErrInitBlobovniczas, err)
}
return nil
} }
// Close releases all internal resources of BlobStor. // Close releases all internal resources of BlobStor.

View file

@ -5,10 +5,16 @@ import (
"fmt" "fmt"
"sync" "sync"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"go.uber.org/zap" "go.uber.org/zap"
) )
type shardInitError struct {
err error
id string
}
// Open opens all StorageEngine's components. // Open opens all StorageEngine's components.
func (e *StorageEngine) Open() error { func (e *StorageEngine) Open() error {
return e.open() return e.open()
@ -44,28 +50,44 @@ func (e *StorageEngine) open() error {
// Init initializes all StorageEngine's components. // Init initializes all StorageEngine's components.
func (e *StorageEngine) Init() error { func (e *StorageEngine) Init() error {
e.mtx.RLock() e.mtx.Lock()
defer e.mtx.RUnlock() defer e.mtx.Unlock()
var wg sync.WaitGroup var wg sync.WaitGroup
var errCh = make(chan error, len(e.shards)) var errCh = make(chan shardInitError, len(e.shards))
for id, sh := range e.shards { for id, sh := range e.shards {
wg.Add(1) wg.Add(1)
go func(id string, sh *shard.Shard) { go func(id string, sh *shard.Shard) {
defer wg.Done() defer wg.Done()
if err := sh.Init(); err != nil { if err := sh.Init(); err != nil {
errCh <- fmt.Errorf("could not initialize shard %s: %w", id, err) errCh <- shardInitError{
err: err,
id: id,
}
} }
}(id, sh.Shard) }(id, sh.Shard)
} }
wg.Wait() wg.Wait()
close(errCh) close(errCh)
for err := range errCh { for res := range errCh {
if err != nil { if res.err != nil {
return err if errors.Is(res.err, blobstor.ErrInitBlobovniczas) {
delete(e.shards, res.id)
e.log.Error("shard initialization failure, skipping",
zap.String("id", res.id),
zap.Error(res.err))
continue
} }
return fmt.Errorf("could not initialize shard %s: %w", res.id, res.err)
}
}
if len(e.shards) == 0 {
return errors.New("failed initialization on all shards")
} }
return nil return nil