s.makhov/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#1549] shard: Turn to read-only mode on metabase failure

Browse source 
If metabase can't be opened in the default mode, try opening shard
first in `ReadOnly` mode and then in `DegradedReadOnly`.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-06-27 11:06:01 +03:00 committed by fyrchik
parent 4944490ffb
commit fabe717d32
3 changed files with 122 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
pkg/local_object_storage/metabase/control.go
View file

@ -123,5 +123,8 @@ func (db *DB) init(reset bool) error {
// Close closes boltDB instance.
func (db *DB) Close() error {
if db.boltDB != nil {
return db.boltDB.Close()
}
return nil
}

82
pkg/local_object_storage/shard/control.go
View file

@ -7,11 +7,35 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
func (s *Shard) handleMetabaseFailure(stage string, err error) error {
s.log.Error("metabase failure, switching mode",
zap.String("stage", stage),
zap.Stringer("mode", mode.ReadOnly),
zap.Error(err))
err = s.SetMode(mode.ReadOnly)
if err == nil {
return nil
}
s.log.Error("can't move shard to readonly, switch mode",
zap.String("stage", stage),
zap.Stringer("mode", mode.DegradedReadOnly),
zap.Error(err))
err = s.SetMode(mode.DegradedReadOnly)
if err != nil {
return fmt.Errorf("could not switch to mode %s", mode.DegradedReadOnly)
}
return nil
}
// Open opens all Shard's components.
func (s *Shard) Open() error {
components := []interface{ Open(bool) error }{
@ -28,36 +52,70 @@ func (s *Shard) Open() error {
for _, component := range components {
if err := component.Open(false); err != nil {
if component == s.metaBase {
err = s.handleMetabaseFailure("open", err)
if err != nil {
return err
}
break
}
return fmt.Errorf("could not open %T: %w", component, err)
}
}
return nil
}
// Init initializes all Shard's components.
func (s *Shard) Init() error {
var fMetabase func() error
type metabaseSynchronizer Shard
if s.needRefillMetabase() {
fMetabase = s.refillMetabase
} else {
fMetabase = s.metaBase.Init
func (x *metabaseSynchronizer) Init() error {
return (*Shard)(x).refillMetabase()
}
components := []func() error{
s.blobStor.Init, fMetabase,
// Init initializes all Shard's components.
func (s *Shard) Init() error {
type initializer interface {
Init() error
}
var components []initializer
if !s.GetMode().NoMetabase() {
var initMetabase initializer
if s.needRefillMetabase() {
initMetabase = (*metabaseSynchronizer)(s)
} else {
initMetabase = s.metaBase
}
components = []initializer{
s.blobStor, initMetabase,
}
} else {
components = []initializer{s.blobStor}
}
if s.hasWriteCache() {
components = append(components, s.writeCache.Init)
components = append(components, s.writeCache)
}
if s.pilorama != nil {
components = append(components, s.pilorama.Init)
components = append(components, s.pilorama)
}
for _, component := range components {
if err := component(); err != nil {
if err := component.Init(); err != nil {
if component == s.metaBase {
err = s.handleMetabaseFailure("init", err)
if err != nil {
return err
}
break
}
return fmt.Errorf("could not initialize %T: %w", component, err)
}
}

49
pkg/local_object_storage/shard/control_test.go
View file

@ -10,6 +10,8 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
@ -18,8 +20,55 @@ import (
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
func TestShardOpen(t *testing.T) {
dir := t.TempDir()
metaPath := filepath.Join(dir, "meta")
newShard := func() *Shard {
return New(
WithLogger(zaptest.NewLogger(t)),
WithBlobStorOptions(
blobstor.WithRootPath(filepath.Join(dir, "blob")),
blobstor.WithShallowDepth(1),
blobstor.WithSmallSizeLimit(1),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1)),
WithMetaBaseOptions(meta.WithPath(metaPath)),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithWriteCache(true),
WithWriteCacheOptions(
writecache.WithPath(filepath.Join(dir, "wc"))))
}
sh := newShard()
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
require.Equal(t, mode.ReadWrite, sh.GetMode())
require.NoError(t, sh.Close())
// Metabase can be opened in read-only => start in ReadOnly mode.
require.NoError(t, os.Chmod(metaPath, 0444))
sh = newShard()
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
require.Equal(t, mode.ReadOnly, sh.GetMode())
require.Error(t, sh.SetMode(mode.ReadWrite))
require.Equal(t, mode.ReadOnly, sh.GetMode())
require.NoError(t, sh.Close())
// Metabase is corrupted => start in DegradedReadOnly mode.
require.NoError(t, os.Chmod(metaPath, 0000))
sh = newShard()
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
require.Equal(t, mode.DegradedReadOnly, sh.GetMode())
require.NoError(t, sh.Close())
}
func TestRefillMetabaseCorrupted(t *testing.T) {
dir := t.TempDir()