From 29812f6683a09e38c3890333ed4f6d973fbffe41 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 9 Feb 2024 09:17:17 +0300 Subject: [PATCH] [#959] node: Set mode to shard's components when open it Avoid opening database for `metabase` and `cache` in `Degraded` mode. Signed-off-by: Anton Nikiforov --- cmd/frostfs-lens/internal/meta/root.go | 3 +- .../blobstor/blobstor_test.go | 9 +++-- pkg/local_object_storage/blobstor/control.go | 19 +++++++++- .../blobstor/exists_test.go | 3 +- .../blobstor/iterate_test.go | 3 +- pkg/local_object_storage/blobstor/mode.go | 2 +- .../internal/storagetest/storage.go | 18 ++++----- pkg/local_object_storage/metabase/control.go | 15 +++++++- pkg/local_object_storage/metabase/db_test.go | 3 +- pkg/local_object_storage/metabase/mode.go | 24 +++++------- .../metabase/mode_test.go | 37 +++++++++++++++++++ .../metabase/version_test.go | 15 ++++---- .../pilorama/bench_test.go | 3 +- pkg/local_object_storage/pilorama/boltdb.go | 21 +++++++---- pkg/local_object_storage/pilorama/forest.go | 2 +- .../pilorama/forest_test.go | 5 ++- .../pilorama/interface.go | 2 +- .../pilorama/mode_test.go | 31 ++++++++++++++++ pkg/local_object_storage/shard/control.go | 20 ++++++---- pkg/local_object_storage/shard/id.go | 7 ++-- .../writecache/benchmark/writecache_test.go | 3 +- .../writecache/cachebbolt.go | 10 ++++- .../writecache/flush_test.go | 6 +-- .../writecache/mode_test.go | 30 +++++++++++++++ .../writecache/writecache.go | 2 +- 25 files changed, 219 insertions(+), 74 deletions(-) create mode 100644 pkg/local_object_storage/metabase/mode_test.go create mode 100644 pkg/local_object_storage/pilorama/mode_test.go create mode 100644 pkg/local_object_storage/writecache/mode_test.go diff --git a/cmd/frostfs-lens/internal/meta/root.go b/cmd/frostfs-lens/internal/meta/root.go index a59574b6c..6741abd0c 100644 --- a/cmd/frostfs-lens/internal/meta/root.go +++ b/cmd/frostfs-lens/internal/meta/root.go @@ -5,6 +5,7 @@ import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/spf13/cobra" "go.etcd.io/bbolt" ) @@ -43,7 +44,7 @@ func openMeta(cmd *cobra.Command) *meta.DB { }), meta.WithEpochState(epochState{}), ) - common.ExitOnErr(cmd, common.Errf("could not open metabase: %w", db.Open(cmd.Context(), true))) + common.ExitOnErr(cmd, common.Errf("could not open metabase: %w", db.Open(cmd.Context(), mode.ReadOnly))) return db } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index f1d567da7..3dac3bfa4 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" ) @@ -51,7 +52,7 @@ func TestCompression(t *testing.T) { bs := New( WithCompressObjects(compress), WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, bs.Open(context.Background(), false)) + require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) return bs } @@ -126,7 +127,7 @@ func TestBlobstor_needsCompression(t *testing.T) { Storage: fstree.New(fstree.WithPath(dir)), }, })) - require.NoError(t, bs.Open(context.Background(), false)) + require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) return bs } @@ -188,7 +189,7 @@ func TestConcurrentPut(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, blobStor.Open(context.Background(), false)) + require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) require.NoError(t, blobStor.Init()) testGet := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { @@ -268,7 +269,7 @@ func TestConcurrentDelete(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, blobStor.Open(context.Background(), false)) + require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) require.NoError(t, blobStor.Init()) testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 4b8a36de8..45b0c3f10 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -6,13 +6,29 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "go.uber.org/zap" ) // Open opens BlobStor. -func (b *BlobStor) Open(ctx context.Context, readOnly bool) error { +func (b *BlobStor) Open(ctx context.Context, mode mode.Mode) error { b.log.Debug(logs.BlobstorOpening) + b.modeMtx.Lock() + defer b.modeMtx.Unlock() + b.mode = mode + + err := b.openBlobStor(ctx, mode) + if err != nil { + return err + } + b.metrics.SetMode(mode.ReadOnly()) + + return nil +} + +func (b *BlobStor) openBlobStor(ctx context.Context, mode mode.Mode) error { + readOnly := mode.ReadOnly() for i := range b.storage { select { case <-ctx.Done(): @@ -24,7 +40,6 @@ func (b *BlobStor) Open(ctx context.Context, readOnly bool) error { return err } } - b.metrics.SetMode(readOnly) return nil } diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 367b63af1..783c198b2 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -7,6 +7,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -20,7 +21,7 @@ func TestExists(t *testing.T) { b := New(WithStorages(storages)) - require.NoError(t, b.Open(context.Background(), false)) + require.NoError(t, b.Open(context.Background(), mode.ReadWrite)) require.NoError(t, b.Init()) objects := []*objectSDK.Object{ diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index ef3fda991..079728380 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -7,6 +7,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -26,7 +27,7 @@ func TestIterateObjects(t *testing.T) { defer os.RemoveAll(p) // open Blobstor - require.NoError(t, blobStor.Open(context.Background(), false)) + require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) // initialize Blobstor require.NoError(t, blobStor.Init()) diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index 2f4473bd8..dc7dad687 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -22,7 +22,7 @@ func (b *BlobStor) SetMode(m mode.Mode) error { err := b.Close() if err == nil { - if err = b.Open(context.TODO(), m.ReadOnly()); err == nil { + if err = b.openBlobStor(context.TODO(), m); err == nil { err = b.Init() } } diff --git a/pkg/local_object_storage/internal/storagetest/storage.go b/pkg/local_object_storage/internal/storagetest/storage.go index ec60a2d0e..586b3dcc6 100644 --- a/pkg/local_object_storage/internal/storagetest/storage.go +++ b/pkg/local_object_storage/internal/storagetest/storage.go @@ -10,7 +10,7 @@ import ( // Component represents single storage component. type Component interface { - Open(context.Context, bool) error + Open(context.Context, mode.Mode) error SetMode(mode.Mode) error Init() error Close() error @@ -58,18 +58,18 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { t.Run("RW", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Close()) }) t.Run("RO", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. // Open in read-only must be done after the db is here. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.Close()) - require.NoError(t, s.Open(context.Background(), true)) + require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) require.NoError(t, s.Close()) }) } @@ -78,7 +78,7 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { func TestCloseTwice(t *testing.T, cons Constructor) { // Use-case: move to maintenance mode twice, first time failed. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.Close()) require.NoError(t, s.Close()) // already closed, no-op @@ -90,12 +90,12 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { // Use-case: metabase `Init` failed, // call `SetMode` on all not-yet-initialized components. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.SetMode(m)) t.Run("after open in RO", func(t *testing.T) { require.NoError(t, s.Close()) - require.NoError(t, s.Open(context.Background(), true)) + require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) require.NoError(t, s.SetMode(m)) }) @@ -104,7 +104,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { t.Run("after init", func(t *testing.T) { s := cons(t) // Use-case: notmal node operation. - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.SetMode(m)) require.NoError(t, s.Close()) @@ -114,7 +114,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { func TestModeTransition(t *testing.T, cons Constructor, from, to mode.Mode) { // Use-case: normal node operation. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.SetMode(from)) require.NoError(t, s.SetMode(to)) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 00fe1e02b..c1a86819d 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -22,7 +22,18 @@ var ErrDegradedMode = logicerr.New("metabase is in a degraded mode") var ErrReadOnlyMode = logicerr.New("metabase is in a read-only mode") // Open boltDB instance for metabase. -func (db *DB) Open(_ context.Context, readOnly bool) error { +func (db *DB) Open(_ context.Context, mode mode.Mode) error { + db.modeMtx.Lock() + defer db.modeMtx.Unlock() + db.mode = mode + + if mode.NoMetabase() { + return nil + } + return db.openDB(mode) +} + +func (db *DB) openDB(mode mode.Mode) error { err := util.MkdirAllX(filepath.Dir(db.info.Path), db.info.Permission) if err != nil { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) @@ -34,7 +45,7 @@ func (db *DB) Open(_ context.Context, readOnly bool) error { opts := *bbolt.DefaultOptions db.boltOptions = &opts } - db.boltOptions.ReadOnly = readOnly + db.boltOptions.ReadOnly = mode.ReadOnly() return metaerr.Wrap(db.openBolt()) } diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index fc20e2aad..01e1ed2bc 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -9,6 +9,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -49,7 +50,7 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB { }, opts...)..., ) - require.NoError(t, bdb.Open(context.Background(), false)) + require.NoError(t, bdb.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bdb.Init()) return bdb diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index a18095f3e..b382e99c2 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -1,7 +1,6 @@ package meta import ( - "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -23,21 +22,16 @@ func (db *DB) SetMode(m mode.Mode) error { } } - var err error - switch { - case m.NoMetabase(): + if m.NoMetabase() { db.boltDB = nil - case m.ReadOnly(): - err = db.Open(context.TODO(), true) - default: - err = db.Open(context.TODO(), false) - } - if err == nil && !m.NoMetabase() && !m.ReadOnly() { - err = db.Init() - } - - if err != nil { - return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + } else { + err := db.openDB(m) + if err == nil && !m.ReadOnly() { + err = db.Init() + } + if err != nil { + return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + } } db.mode = m diff --git a/pkg/local_object_storage/metabase/mode_test.go b/pkg/local_object_storage/metabase/mode_test.go new file mode 100644 index 000000000..1b9f60055 --- /dev/null +++ b/pkg/local_object_storage/metabase/mode_test.go @@ -0,0 +1,37 @@ +package meta + +import ( + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" +) + +type epochStateTest struct{} + +func (s epochStateTest) CurrentEpoch() uint64 { + return 0 +} + +func Test_Mode(t *testing.T) { + t.Parallel() + bdb := New([]Option{ + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochStateTest{}), + }...) + + require.NoError(t, bdb.Open(context.Background(), mode.DegradedReadOnly)) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Init()) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Close()) + + require.NoError(t, bdb.Open(context.Background(), mode.Degraded)) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Init()) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Close()) +} diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index ddeddd189..cea42da0f 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -43,13 +44,13 @@ func TestVersion(t *testing.T) { } t.Run("simple", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) t.Run("reopen", func(t *testing.T) { - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) @@ -57,29 +58,29 @@ func TestVersion(t *testing.T) { }) t.Run("old data", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.WriteShardID([]byte{1, 2, 3, 4})) require.NoError(t, db.Close()) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) }) t.Run("invalid version", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { return updateVersion(tx, version+1) })) require.NoError(t, db.Close()) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.Error(t, db.Init()) require.NoError(t, db.Close()) t.Run("reset", func(t *testing.T) { - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Reset()) check(t, db) require.NoError(t, db.Close()) diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go index e3f3afd99..22b951a41 100644 --- a/pkg/local_object_storage/pilorama/bench_test.go +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" ) @@ -26,7 +27,7 @@ func BenchmarkCreate(b *testing.B) { f := NewBoltForest( WithPath(filepath.Join(tmpDir, "test.db")), WithMaxBatchSize(runtime.GOMAXPROCS(0))) - require.NoError(b, f.Open(context.Background(), false)) + require.NoError(b, f.Open(context.Background(), mode.ReadWrite)) require.NoError(b, f.Init()) defer func() { require.NoError(b, f.Close()) }() diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 52cdec586..dfdb8a468 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -99,7 +99,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { err := t.Close() if err == nil && !m.NoMetabase() { - if err = t.Open(context.TODO(), m.ReadOnly()); err == nil { + if err = t.openBolt(m); err == nil { err = t.Init() } } @@ -112,7 +112,18 @@ func (t *boltForest) SetMode(m mode.Mode) error { return nil } -func (t *boltForest) Open(_ context.Context, readOnly bool) error { +func (t *boltForest) Open(_ context.Context, mode mode.Mode) error { + t.modeMtx.Lock() + defer t.modeMtx.Unlock() + t.mode = mode + if mode.NoMetabase() { + return nil + } + return t.openBolt(mode) +} + +func (t *boltForest) openBolt(mode mode.Mode) error { + readOnly := mode.ReadOnly() err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) @@ -131,11 +142,7 @@ func (t *boltForest) Open(_ context.Context, readOnly bool) error { t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - m := mode.ReadWrite - if readOnly { - m = mode.ReadOnly - } - t.metrics.SetMode(m) + t.metrics.SetMode(mode) return nil } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index b82e5b3bd..90b4cc237 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -111,7 +111,7 @@ func (f *memoryForest) Init() error { return nil } -func (f *memoryForest) Open(context.Context, bool) error { +func (f *memoryForest) Open(context.Context, mode.Mode) error { return nil } diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 5813395f0..4e1e8601d 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -24,7 +25,7 @@ var providers = []struct { }{ {"inmemory", func(t testing.TB, _ ...Option) ForestStorage { f := NewMemoryForest() - require.NoError(t, f.Open(context.Background(), false)) + require.NoError(t, f.Open(context.Background(), mode.ReadWrite)) require.NoError(t, f.Init()) return f }}, @@ -34,7 +35,7 @@ var providers = []struct { WithPath(filepath.Join(t.TempDir(), "test.db")), WithMaxBatchSize(1), }, opts...)...) - require.NoError(t, f.Open(context.Background(), false)) + require.NoError(t, f.Open(context.Background(), mode.ReadWrite)) require.NoError(t, f.Init()) return f }}, diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index e7f7eb512..e474c8b68 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -58,7 +58,7 @@ type ForestStorage interface { // DumpInfo returns information about the pilorama. DumpInfo() Info Init() error - Open(context.Context, bool) error + Open(context.Context, mode.Mode) error Close() error SetMode(m mode.Mode) error SetParentID(id string) diff --git a/pkg/local_object_storage/pilorama/mode_test.go b/pkg/local_object_storage/pilorama/mode_test.go new file mode 100644 index 000000000..01d3da9f0 --- /dev/null +++ b/pkg/local_object_storage/pilorama/mode_test.go @@ -0,0 +1,31 @@ +package pilorama + +import ( + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" +) + +func Test_Mode(t *testing.T) { + t.Parallel() + f := NewBoltForest( + []Option{ + WithPath(filepath.Join(t.TempDir(), "test.db")), + WithMaxBatchSize(1), + }...) + + require.NoError(t, f.Open(context.Background(), mode.DegradedReadOnly)) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Init()) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Close()) + + require.NoError(t, f.Open(context.Background(), mode.Degraded)) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Init()) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Close()) +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 574bc1430..8d1007ffd 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -43,12 +43,17 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { // Open opens all Shard's components. func (s *Shard) Open(ctx context.Context) error { components := []interface { - Open(context.Context, bool) error + Open(context.Context, mode.Mode) error }{ - s.blobStor, s.metaBase, + s.blobStor, + } + m := s.GetMode() + + if !m.NoMetabase() { + components = append(components, s.metaBase) } - if s.hasWriteCache() { + if s.hasWriteCache() && !m.NoMetabase() { components = append(components, s.writeCache) } @@ -57,12 +62,12 @@ func (s *Shard) Open(ctx context.Context) error { } for i, component := range components { - if err := component.Open(ctx, false); err != nil { + if err := component.Open(ctx, m); err != nil { if component == s.metaBase { // We must first open all other components to avoid // opening non-existent DB in read-only mode. for j := i + 1; j < len(components); j++ { - if err := components[j].Open(ctx, false); err != nil { + if err := components[j].Open(ctx, m); err != nil { // Other components must be opened, fail. return fmt.Errorf("could not open %T: %w", components[j], err) } @@ -97,8 +102,9 @@ func (s *Shard) Init(ctx context.Context) error { } var components []initializer + m := s.GetMode() - if !s.GetMode().NoMetabase() { + if !m.NoMetabase() { var initMetabase initializer if s.NeedRefillMetabase() { @@ -114,7 +120,7 @@ func (s *Shard) Init(ctx context.Context) error { components = []initializer{s.blobStor} } - if s.hasWriteCache() { + if s.hasWriteCache() && !m.NoMetabase() { components = append(components, s.writeCache) } diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index e3c209907..b67fd59cd 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -31,12 +31,11 @@ func (s *Shard) ID() *ID { // UpdateID reads shard ID saved in the metabase and updates it if it is missing. func (s *Shard) UpdateID(ctx context.Context) (err error) { - var metabaseOpened bool var idFromMetabase []byte - if err = s.metaBase.Open(ctx, false); err != nil { + metabaseOpened := !s.GetMode().NoMetabase() + if err = s.metaBase.Open(ctx, s.GetMode()); err != nil { err = fmt.Errorf("failed to open metabase: %w", err) - } else { - metabaseOpened = true + metabaseOpened = false } if metabaseOpened { defer func() { diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index f7895a935..c1c0e88b3 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "github.com/stretchr/testify/require" ) @@ -75,7 +76,7 @@ func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { } func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { - require.NoError(b, cache.Open(context.Background(), false), "opening") + require.NoError(b, cache.Open(context.Background(), mode.ReadWrite), "opening") require.NoError(b, cache.Init(), "initializing") } diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index 5eeda33a6..2ae62c461 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -95,8 +95,14 @@ func (c *cache) DumpInfo() Info { } // Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(_ context.Context, readOnly bool) error { - err := c.openStore(readOnly) +func (c *cache) Open(_ context.Context, mode mode.Mode) error { + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + c.mode = mode + if mode.NoMetabase() { + return nil + } + err := c.openStore(mode.ReadOnly()) if err != nil { return metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index bf68765d8..3c951bebe 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -202,7 +202,7 @@ func newCache[Option any]( mb := meta.New( meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(dummyEpoch{})) - require.NoError(t, mb.Open(context.Background(), false)) + require.NoError(t, mb.Open(context.Background(), mode.ReadWrite)) require.NoError(t, mb.Init()) bs := blobstor.New(blobstor.WithStorages([]blobstor.SubStorage{ @@ -213,11 +213,11 @@ func newCache[Option any]( fstree.WithDirNameLen(1)), }, })) - require.NoError(t, bs.Open(context.Background(), false)) + require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) wc := createCacheFn(t, smallSize, mb, bs, opts...) - require.NoError(t, wc.Open(context.Background(), false)) + require.NoError(t, wc.Open(context.Background(), mode.ReadWrite)) require.NoError(t, wc.Init()) // First set mode for metabase and blobstor to prevent background flushes. diff --git a/pkg/local_object_storage/writecache/mode_test.go b/pkg/local_object_storage/writecache/mode_test.go new file mode 100644 index 000000000..f684c15bc --- /dev/null +++ b/pkg/local_object_storage/writecache/mode_test.go @@ -0,0 +1,30 @@ +package writecache + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/stretchr/testify/require" +) + +func TestMode(t *testing.T) { + t.Parallel() + wc := New( + WithLogger(test.NewLogger(t)), + WithFlushWorkersCount(2), + WithPath(t.TempDir())) + + require.NoError(t, wc.Open(context.Background(), mode.DegradedReadOnly)) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Init()) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Close()) + + require.NoError(t, wc.Open(context.Background(), mode.Degraded)) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Init()) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Close()) +} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 99fc5390a..69dfee953 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -39,7 +39,7 @@ type Cache interface { Seal(context.Context, bool) error Init() error - Open(ctx context.Context, readOnly bool) error + Open(ctx context.Context, mode mode.Mode) error Close() error } -- 2.45.2