[#1549] engine: Disable shard on blobovnicza init failure

There is a need to support working w/o shard if it has problems with
blobovnicza tree.

Make `BlobStor.Init` to return new `ErrInitBlobovniczas` error. Remove
shard from storage engine's shard set if it returned this error from
`Init` call. So if some of the shards (but not all) return this error,
the node will be able to continue working without them.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

pkg/local_object_storage/blobstor/control.go

@@ -1,5 +1,10 @@
 package blobstor
 
+import (
+	"errors"
+	"fmt"
+)
+
 // Open opens BlobStor.
 func (b *BlobStor) Open() error {
 	b.log.Debug("opening...")
@@ -7,13 +12,23 @@ func (b *BlobStor) Open() error {
 	return nil
 }
 
+// ErrInitBlobovniczas is returned when blobovnicza initialization fails.
+var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stage")
+
 // Init initializes internal data structures and system resources.
 //
 // If BlobStor is already initialized, no action is taken.
+//
+// Returns wrapped ErrInitBlobovniczas on blobovnicza tree's initializaiton failure.
 func (b *BlobStor) Init() error {
 	b.log.Debug("initializing...")
 
-	return b.blobovniczas.init()
+	err := b.blobovniczas.init()
+	if err != nil {
+		return fmt.Errorf("%w: %v", ErrInitBlobovniczas, err)
+	}
+
+	return nil
 }
 
 // Close releases all internal resources of BlobStor.

pkg/local_object_storage/engine/control.go

@@ -5,10 +5,16 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	"go.uber.org/zap"
 )
 
+type shardInitError struct {
+	err error
+	id  string
+}
+
 // Open opens all StorageEngine's components.
 func (e *StorageEngine) Open() error {
 	return e.open()
@@ -44,28 +50,44 @@ func (e *StorageEngine) open() error {
 
 // Init initializes all StorageEngine's components.
 func (e *StorageEngine) Init() error {
-	e.mtx.RLock()
+	e.mtx.Lock()
-	defer e.mtx.RUnlock()
+	defer e.mtx.Unlock()
 
 	var wg sync.WaitGroup
-	var errCh = make(chan error, len(e.shards))
+	var errCh = make(chan shardInitError, len(e.shards))
 
 	for id, sh := range e.shards {
 		wg.Add(1)
 		go func(id string, sh *shard.Shard) {
 			defer wg.Done()
 			if err := sh.Init(); err != nil {
-				errCh <- fmt.Errorf("could not initialize shard %s: %w", id, err)
+				errCh <- shardInitError{
+					err: err,
+					id:  id,
+				}
 			}
 		}(id, sh.Shard)
 	}
 	wg.Wait()
 	close(errCh)
 
-	for err := range errCh {
+	for res := range errCh {
-		if err != nil {
+		if res.err != nil {
-			return err
+			if errors.Is(res.err, blobstor.ErrInitBlobovniczas) {
+				delete(e.shards, res.id)
+
+				e.log.Error("shard initialization failure, skipping",
+					zap.String("id", res.id),
+					zap.Error(res.err))
+
+				continue
 			}
+			return fmt.Errorf("could not initialize shard %s: %w", res.id, res.err)
+		}
+	}
+
+	if len(e.shards) == 0 {
+		return errors.New("failed initialization on all shards")
 	}
 
 	return nil