From fabe717d32f3c3f8ee80bb45ef73b45a8cf05002 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Mon, 27 Jun 2022 11:06:01 +0300 Subject: [PATCH] [#1549] shard: Turn to read-only mode on metabase failure If metabase can't be opened in the default mode, try opening shard first in `ReadOnly` mode and then in `DegradedReadOnly`. Signed-off-by: Leonard Lyubich Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/control.go | 5 +- pkg/local_object_storage/shard/control.go | 80 ++++++++++++++++--- .../shard/control_test.go | 49 ++++++++++++ 3 files changed, 122 insertions(+), 12 deletions(-) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 647f39321..b1e072e3a 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -123,5 +123,8 @@ func (db *DB) init(reset bool) error { // Close closes boltDB instance. func (db *DB) Close() error { - return db.boltDB.Close() + if db.boltDB != nil { + return db.boltDB.Close() + } + return nil } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index e645128f0..fe5c115f4 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -7,11 +7,35 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" ) +func (s *Shard) handleMetabaseFailure(stage string, err error) error { + s.log.Error("metabase failure, switching mode", + zap.String("stage", stage), + zap.Stringer("mode", mode.ReadOnly), + zap.Error(err)) + + err = s.SetMode(mode.ReadOnly) + if err == nil { + return nil + } + + s.log.Error("can't move shard to readonly, switch mode", + zap.String("stage", stage), + zap.Stringer("mode", mode.DegradedReadOnly), + zap.Error(err)) + + err = s.SetMode(mode.DegradedReadOnly) + if err != nil { + return fmt.Errorf("could not switch to mode %s", mode.DegradedReadOnly) + } + return nil +} + // Open opens all Shard's components. func (s *Shard) Open() error { components := []interface{ Open(bool) error }{ @@ -28,36 +52,70 @@ func (s *Shard) Open() error { for _, component := range components { if err := component.Open(false); err != nil { + if component == s.metaBase { + err = s.handleMetabaseFailure("open", err) + if err != nil { + return err + } + + break + } + return fmt.Errorf("could not open %T: %w", component, err) } } return nil } +type metabaseSynchronizer Shard + +func (x *metabaseSynchronizer) Init() error { + return (*Shard)(x).refillMetabase() +} + // Init initializes all Shard's components. func (s *Shard) Init() error { - var fMetabase func() error - - if s.needRefillMetabase() { - fMetabase = s.refillMetabase - } else { - fMetabase = s.metaBase.Init + type initializer interface { + Init() error } - components := []func() error{ - s.blobStor.Init, fMetabase, + var components []initializer + + if !s.GetMode().NoMetabase() { + var initMetabase initializer + + if s.needRefillMetabase() { + initMetabase = (*metabaseSynchronizer)(s) + } else { + initMetabase = s.metaBase + } + + components = []initializer{ + s.blobStor, initMetabase, + } + } else { + components = []initializer{s.blobStor} } if s.hasWriteCache() { - components = append(components, s.writeCache.Init) + components = append(components, s.writeCache) } if s.pilorama != nil { - components = append(components, s.pilorama.Init) + components = append(components, s.pilorama) } for _, component := range components { - if err := component(); err != nil { + if err := component.Init(); err != nil { + if component == s.metaBase { + err = s.handleMetabaseFailure("init", err) + if err != nil { + return err + } + + break + } + return fmt.Errorf("could not initialize %T: %w", component, err) } } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 22d11bf4b..a7918c283 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -10,6 +10,8 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" @@ -18,8 +20,55 @@ import ( oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" ) +func TestShardOpen(t *testing.T) { + dir := t.TempDir() + metaPath := filepath.Join(dir, "meta") + + newShard := func() *Shard { + return New( + WithLogger(zaptest.NewLogger(t)), + WithBlobStorOptions( + blobstor.WithRootPath(filepath.Join(dir, "blob")), + blobstor.WithShallowDepth(1), + blobstor.WithSmallSizeLimit(1), + blobstor.WithBlobovniczaShallowWidth(1), + blobstor.WithBlobovniczaShallowDepth(1)), + WithMetaBaseOptions(meta.WithPath(metaPath)), + WithPiloramaOptions( + pilorama.WithPath(filepath.Join(dir, "pilorama"))), + WithWriteCache(true), + WithWriteCacheOptions( + writecache.WithPath(filepath.Join(dir, "wc")))) + } + + sh := newShard() + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init()) + require.Equal(t, mode.ReadWrite, sh.GetMode()) + require.NoError(t, sh.Close()) + + // Metabase can be opened in read-only => start in ReadOnly mode. + require.NoError(t, os.Chmod(metaPath, 0444)) + sh = newShard() + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init()) + require.Equal(t, mode.ReadOnly, sh.GetMode()) + require.Error(t, sh.SetMode(mode.ReadWrite)) + require.Equal(t, mode.ReadOnly, sh.GetMode()) + require.NoError(t, sh.Close()) + + // Metabase is corrupted => start in DegradedReadOnly mode. + require.NoError(t, os.Chmod(metaPath, 0000)) + sh = newShard() + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init()) + require.Equal(t, mode.DegradedReadOnly, sh.GetMode()) + require.NoError(t, sh.Close()) +} + func TestRefillMetabaseCorrupted(t *testing.T) { dir := t.TempDir()