[#1549] shard: Turn to read-only mode on metabase failure

If metabase can't be opened in the default mode, try opening shard
first in `ReadOnly` mode and then in `DegradedReadOnly`.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

pkg/local_object_storage/metabase/control.go

@@ -123,5 +123,8 @@ func (db *DB) init(reset bool) error {
 
 // Close closes boltDB instance.
 func (db *DB) Close() error {
-	return db.boltDB.Close()
+	if db.boltDB != nil {
+		return db.boltDB.Close()
+	}
+	return nil
 }

pkg/local_object_storage/shard/control.go

@@ -7,11 +7,35 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
 )
 
+func (s *Shard) handleMetabaseFailure(stage string, err error) error {
+	s.log.Error("metabase failure, switching mode",
+		zap.String("stage", stage),
+		zap.Stringer("mode", mode.ReadOnly),
+		zap.Error(err))
+
+	err = s.SetMode(mode.ReadOnly)
+	if err == nil {
+		return nil
+	}
+
+	s.log.Error("can't move shard to readonly, switch mode",
+		zap.String("stage", stage),
+		zap.Stringer("mode", mode.DegradedReadOnly),
+		zap.Error(err))
+
+	err = s.SetMode(mode.DegradedReadOnly)
+	if err != nil {
+		return fmt.Errorf("could not switch to mode %s", mode.DegradedReadOnly)
+	}
+	return nil
+}
+
 // Open opens all Shard's components.
 func (s *Shard) Open() error {
 	components := []interface{ Open(bool) error }{
@@ -28,36 +52,70 @@ func (s *Shard) Open() error {
 
 	for _, component := range components {
 		if err := component.Open(false); err != nil {
+			if component == s.metaBase {
+				err = s.handleMetabaseFailure("open", err)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+
 			return fmt.Errorf("could not open %T: %w", component, err)
 		}
 	}
 	return nil
 }
 
+type metabaseSynchronizer Shard
+
+func (x *metabaseSynchronizer) Init() error {
+	return (*Shard)(x).refillMetabase()
+}
+
 // Init initializes all Shard's components.
 func (s *Shard) Init() error {
-	var fMetabase func() error
-
-	if s.needRefillMetabase() {
-		fMetabase = s.refillMetabase
-	} else {
-		fMetabase = s.metaBase.Init
+	type initializer interface {
+		Init() error
 	}
 
-	components := []func() error{
-		s.blobStor.Init, fMetabase,
+	var components []initializer
+
+	if !s.GetMode().NoMetabase() {
+		var initMetabase initializer
+
+		if s.needRefillMetabase() {
+			initMetabase = (*metabaseSynchronizer)(s)
+		} else {
+			initMetabase = s.metaBase
+		}
+
+		components = []initializer{
+			s.blobStor, initMetabase,
+		}
+	} else {
+		components = []initializer{s.blobStor}
 	}
 
 	if s.hasWriteCache() {
-		components = append(components, s.writeCache.Init)
+		components = append(components, s.writeCache)
 	}
 
 	if s.pilorama != nil {
-		components = append(components, s.pilorama.Init)
+		components = append(components, s.pilorama)
 	}
 
 	for _, component := range components {
-		if err := component(); err != nil {
+		if err := component.Init(); err != nil {
+			if component == s.metaBase {
+				err = s.handleMetabaseFailure("init", err)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+
 			return fmt.Errorf("could not initialize %T: %w", component, err)
 		}
 	}

pkg/local_object_storage/shard/control_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@@ -18,8 +20,55 @@ import (
 	oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
 	objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
 )
 
+func TestShardOpen(t *testing.T) {
+	dir := t.TempDir()
+	metaPath := filepath.Join(dir, "meta")
+
+	newShard := func() *Shard {
+		return New(
+			WithLogger(zaptest.NewLogger(t)),
+			WithBlobStorOptions(
+				blobstor.WithRootPath(filepath.Join(dir, "blob")),
+				blobstor.WithShallowDepth(1),
+				blobstor.WithSmallSizeLimit(1),
+				blobstor.WithBlobovniczaShallowWidth(1),
+				blobstor.WithBlobovniczaShallowDepth(1)),
+			WithMetaBaseOptions(meta.WithPath(metaPath)),
+			WithPiloramaOptions(
+				pilorama.WithPath(filepath.Join(dir, "pilorama"))),
+			WithWriteCache(true),
+			WithWriteCacheOptions(
+				writecache.WithPath(filepath.Join(dir, "wc"))))
+	}
+
+	sh := newShard()
+	require.NoError(t, sh.Open())
+	require.NoError(t, sh.Init())
+	require.Equal(t, mode.ReadWrite, sh.GetMode())
+	require.NoError(t, sh.Close())
+
+	// Metabase can be opened in read-only => start in ReadOnly mode.
+	require.NoError(t, os.Chmod(metaPath, 0444))
+	sh = newShard()
+	require.NoError(t, sh.Open())
+	require.NoError(t, sh.Init())
+	require.Equal(t, mode.ReadOnly, sh.GetMode())
+	require.Error(t, sh.SetMode(mode.ReadWrite))
+	require.Equal(t, mode.ReadOnly, sh.GetMode())
+	require.NoError(t, sh.Close())
+
+	// Metabase is corrupted => start in DegradedReadOnly mode.
+	require.NoError(t, os.Chmod(metaPath, 0000))
+	sh = newShard()
+	require.NoError(t, sh.Open())
+	require.NoError(t, sh.Init())
+	require.Equal(t, mode.DegradedReadOnly, sh.GetMode())
+	require.NoError(t, sh.Close())
+}
+
 func TestRefillMetabaseCorrupted(t *testing.T) {
 	dir := t.TempDir()