From 0b0d829346663dd32899544e98bad19a7aec0d25 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 14 May 2024 14:44:55 +0300 Subject: [PATCH 1/3] [#1121] node: Refactor mods of shard Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/shard/mode/mode.go | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 49c888d63..461611465 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -8,27 +8,25 @@ type Mode uint32 const ( // ReadWrite is a Mode value for shard that is available // for read and write operations. Default shard mode. - ReadWrite Mode = 0 + ReadWrite Mode = 0b000 - // DegradedReadOnly is a Mode value for shard that is set automatically - // after a certain number of errors is encountered. It is the same as - // `mode.Degraded` but also is read-only. - DegradedReadOnly = Degraded | ReadOnly + // ReadOnly is a Mode value for shard that does not + // accept write operation but is readable. + ReadOnly Mode = 0b001 + + // Degraded is a Mode value for shard when the metabase is unavailable. + // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator. + Degraded Mode = 0b010 // Disabled mode is a mode where a shard is disabled. // An existing shard can't have this mode, but it can be used in // the configuration or control service commands. - Disabled = math.MaxUint32 -) + Disabled Mode = math.MaxUint32 -const ( - // ReadOnly is a Mode value for shard that does not - // accept write operation but is readable. - ReadOnly Mode = 1 << iota - - // Degraded is a Mode value for shard when the metabase is unavailable. - // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator. - Degraded + // DegradedReadOnly is a Mode value for shard that is set automatically + // after a certain number of errors is encountered. It is the same as + // `mode.Degraded` but also is read-only. + DegradedReadOnly Mode = Degraded | ReadOnly ) func (m Mode) String() string { -- 2.45.3 From d61f78ec1f62a47da72990c1539ad375faaf794e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 4 Jun 2024 16:28:47 +0300 Subject: [PATCH 2/3] [#1121] node: Change mode of shard components Signed-off-by: Alexander Chuprov --- .../blobovniczatree/concurrency_test.go | 3 +- .../blobstor/blobovniczatree/control.go | 7 ++- .../blobstor/blobovniczatree/control_test.go | 9 +-- .../blobstor/blobovniczatree/exists_test.go | 3 +- .../blobstor/blobovniczatree/iterate_test.go | 3 +- .../blobstor/blobovniczatree/metrics.go | 5 +- .../blobovniczatree/rebuild_failover_test.go | 3 +- .../blobstor/blobovniczatree/rebuild_test.go | 9 +-- .../blobstor/common/storage.go | 3 +- pkg/local_object_storage/blobstor/control.go | 5 +- .../blobstor/fstree/control.go | 7 ++- .../blobstor/fstree/fstree_test.go | 3 +- .../blobstor/fstree/metrics.go | 10 ++- .../blobstor/internal/blobstortest/control.go | 5 +- .../blobstor/internal/blobstortest/delete.go | 3 +- .../blobstor/internal/blobstortest/exists.go | 3 +- .../blobstor/internal/blobstortest/get.go | 3 +- .../internal/blobstortest/get_range.go | 3 +- .../blobstor/internal/blobstortest/iterate.go | 3 +- .../blobstor/memstore/control.go | 9 ++- .../blobstor/memstore/memstore_test.go | 3 +- .../blobstor/perf_test.go | 3 +- .../blobstor/teststore/option.go | 9 +-- .../blobstor/teststore/teststore.go | 9 +-- .../engine/control_test.go | 2 +- pkg/local_object_storage/engine/writecache.go | 4 +- pkg/local_object_storage/metabase/control.go | 16 ++--- pkg/local_object_storage/metabase/metrics.go | 4 +- pkg/local_object_storage/metabase/mode.go | 2 +- pkg/local_object_storage/metabase/shard_id.go | 2 +- .../metrics/blobovnicza.go | 5 +- pkg/local_object_storage/metrics/fstree.go | 5 +- pkg/local_object_storage/metrics/metabase.go | 2 +- pkg/local_object_storage/metrics/pilorama.go | 4 +- pkg/local_object_storage/pilorama/boltdb.go | 8 +-- pkg/local_object_storage/pilorama/metrics.go | 4 +- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/mode/mode.go | 62 +++++++++++++++++++ .../writecache/cachebbolt.go | 10 +-- pkg/local_object_storage/writecache/flush.go | 2 +- .../writecache/metrics.go | 4 +- pkg/local_object_storage/writecache/mode.go | 4 +- pkg/local_object_storage/writecache/seal.go | 2 +- .../writecache/storage.go | 9 +-- pkg/metrics/blobovnicza.go | 7 ++- pkg/metrics/fstree.go | 7 ++- pkg/metrics/pilorama.go | 4 +- 47 files changed, 194 insertions(+), 100 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index 97606df02..5bed86142 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" ) @@ -23,7 +24,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { WithBlobovniczaShallowWidth(10), WithBlobovniczaShallowDepth(1), WithRootPath(t.TempDir())) - require.NoError(t, st.Open(false)) + require.NoError(t, st.Open(mode.ComponentReadWrite)) require.NoError(t, st.Init()) defer func() { require.NoError(t, st.Close()) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index bd98967cb..67e8c8f18 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -8,6 +8,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -16,9 +17,9 @@ import ( var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode") // Open opens blobovnicza tree. -func (b *Blobovniczas) Open(readOnly bool) error { - b.readOnly = readOnly - b.metrics.SetMode(readOnly) +func (b *Blobovniczas) Open(mode mode.ComponentMode) error { + b.readOnly = mode.ReadOnly() + b.metrics.SetMode(mode) b.metrics.SetRebuildStatus(rebuildStatusNotStarted) b.openManagers() return nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index 0356fbaae..7218cd6bb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -54,7 +55,7 @@ func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) { WithBlobovniczaShallowWidth(width), WithRootPath(path), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) require.NoError(t, blz.Close()) } @@ -85,7 +86,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { WithRootPath(rootDir), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) obj35 := blobstortest.NewObject(10 * 1024) @@ -123,7 +124,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { WithRootPath(rootDir), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) gRes, err = blz.Get(context.Background(), common.GetPrm{ @@ -160,7 +161,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { WithBlobovniczaShallowWidth(5), WithRootPath(rootDir), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) gRes, err = blz.Get(context.Background(), common.GetPrm{ diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 625e620b8..d6ffd8bce 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" ) @@ -24,7 +25,7 @@ func TestExistsInvalidStorageID(t *testing.T) { WithBlobovniczaShallowDepth(2), WithRootPath(dir), WithBlobovniczaSize(1<<20)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) defer func() { require.NoError(t, b.Close()) }() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go index 506dd9a4b..b067a164b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -18,7 +19,7 @@ func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) { WithRootPath(t.TempDir()), ) blz.createDBInAdvance = true - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) defer func() { require.NoError(t, blz.Close()) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 28289c19e..68dc7ff38 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -4,6 +4,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) const ( @@ -18,7 +19,7 @@ type Metrics interface { SetParentID(parentID string) - SetMode(readOnly bool) + SetMode(mode.ComponentMode) Close() SetRebuildStatus(status string) @@ -37,7 +38,7 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) SetRebuildStatus(string) {} func (m *noopMetrics) SetRebuildPercent(uint32) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 31dadb533..a6afed60c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -138,7 +139,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object WithBlobovniczaSize(100*1024*1024), WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) var dPrm common.DeletePrm diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 6b8ae4b5c..7a1de4c13 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" @@ -53,7 +54,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) obj := blobstortest.NewObject(64 * 1024) // 64KB object @@ -81,7 +82,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) metaStub := &storageIDUpdateStub{ @@ -120,7 +121,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) eg, egCtx := errgroup.WithContext(context.Background()) @@ -161,7 +162,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(50)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) for addr, storageID := range storageIDs { diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 8f629b1de..4f3a20993 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -4,12 +4,13 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) // Storage represents key-value object storage. // It is used as a building block for a blobstor of a shard. type Storage interface { - Open(readOnly bool) error + Open(mode mode.ComponentMode) error Init() error Close() error diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 45b0c3f10..9b414a9be 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -27,15 +27,14 @@ func (b *BlobStor) Open(ctx context.Context, mode mode.Mode) error { return nil } -func (b *BlobStor) openBlobStor(ctx context.Context, mode mode.Mode) error { - readOnly := mode.ReadOnly() +func (b *BlobStor) openBlobStor(ctx context.Context, mod mode.Mode) error { for i := range b.storage { select { case <-ctx.Done(): return ctx.Err() default: } - err := b.storage[i].Storage.Open(readOnly) + err := b.storage[i].Storage.Open(mode.ConvertToComponentMode(mod)) if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index cec531f2e..c21d79f09 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -1,13 +1,14 @@ package fstree import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" ) // Open implements common.Storage. -func (t *FSTree) Open(ro bool) error { - t.readOnly = ro - t.metrics.SetMode(ro) +func (t *FSTree) Open(mode mode.ComponentMode) error { + t.readOnly = mode.ReadOnly() + t.metrics.SetMode(mode) return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index b81ce43f1..d633cbac3 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -6,6 +6,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -43,7 +44,7 @@ func TestObjectCounter(t *testing.T) { WithDepth(2), WithDirNameLen(2), WithFileCounter(counter)) - require.NoError(t, fst.Open(false)) + require.NoError(t, fst.Open(mode.ComponentReadWrite)) require.NoError(t, fst.Init()) counterValue := counter.Value() diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go index eeb686672..10de935eb 100644 --- a/pkg/local_object_storage/blobstor/fstree/metrics.go +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -1,11 +1,15 @@ package fstree -import "time" +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) type Metrics interface { SetParentID(parentID string) - SetMode(readOnly bool) + SetMode(mode mode.ComponentMode) Close() Iterate(d time.Duration, success bool) @@ -20,7 +24,7 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Delete(time.Duration, bool) {} diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index 96d54dec3..a3bbc021d 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -7,6 +7,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -14,13 +15,13 @@ import ( // cons must return a storage which is NOT opened. func TestControl(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) objects := prepare(t, 10, s, min, max) require.NoError(t, s.Close()) - require.NoError(t, s.Open(true)) + require.NoError(t, s.Open(mode.ComponentReadOnly)) for i := range objects { var prm common.GetPrm prm.Address = objects[i].addr diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index f551cbf5f..750619a30 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -5,6 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ import ( func TestDelete(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go index 29d8bafee..33b50b12f 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go @@ -5,13 +5,14 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) func TestExists(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index 65c0c3133..12f73c3e9 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -5,6 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ import ( func TestGet(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index 9b99c12d5..93de683c2 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -6,6 +6,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -14,7 +15,7 @@ import ( func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 45c68cebc..e66fe87b6 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -6,12 +6,13 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) func TestIterate(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 7f6e85a16..449d4352a 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -1,9 +1,12 @@ package memstore -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) -func (s *memstoreImpl) Open(readOnly bool) error { - s.readOnly = readOnly +func (s *memstoreImpl) Open(mod mode.ComponentMode) error { + s.readOnly = mod.ReadOnly() return nil } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index bc9436350..8d1480dff 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -18,7 +19,7 @@ func TestSimpleLifecycle(t *testing.T) { WithLogger(test.NewLogger(t)), ) defer func() { require.NoError(t, s.Close()) }() - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) obj := blobstortest.NewObject(1024) diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index fb5afb88b..501c95a1d 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -22,7 +23,7 @@ type storage struct { func (s storage) open(b *testing.B) common.Storage { st := s.create(b.TempDir()) - require.NoError(b, st.Open(false)) + require.NoError(b, st.Open(mode.ComponentReadWrite)) require.NoError(b, st.Init()) return st diff --git a/pkg/local_object_storage/blobstor/teststore/option.go b/pkg/local_object_storage/blobstor/teststore/option.go index b897832cf..bc0bed49d 100644 --- a/pkg/local_object_storage/blobstor/teststore/option.go +++ b/pkg/local_object_storage/blobstor/teststore/option.go @@ -3,12 +3,13 @@ package teststore import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) type cfg struct { st common.Storage overrides struct { - Open func(readOnly bool) error + Open func(mode mode.ComponentMode) error Init func() error Close func() error @@ -35,9 +36,9 @@ func WithSubstorage(st common.Storage) Option { } } -func WithOpen(f func(bool) error) Option { return func(c *cfg) { c.overrides.Open = f } } -func WithInit(f func() error) Option { return func(c *cfg) { c.overrides.Init = f } } -func WithClose(f func() error) Option { return func(c *cfg) { c.overrides.Close = f } } +func WithOpen(f func(mode.ComponentMode) error) Option { return func(c *cfg) { c.overrides.Open = f } } +func WithInit(f func() error) Option { return func(c *cfg) { c.overrides.Init = f } } +func WithClose(f func() error) Option { return func(c *cfg) { c.overrides.Close = f } } func WithType(f func() string) Option { return func(c *cfg) { c.overrides.Type = f } } func WithPath(f func() string) Option { return func(c *cfg) { c.overrides.Path = f } } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 016fd520f..fea4a2d49 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -20,6 +20,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) // TestStore is a common.Storage implementation for testing/mocking purposes. @@ -50,16 +51,16 @@ func (s *TestStore) SetOption(opt Option) { opt(s.cfg) } -func (s *TestStore) Open(readOnly bool) error { +func (s *TestStore) Open(mod mode.ComponentMode) error { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Open != nil: - return s.overrides.Open(readOnly) + return s.overrides.Open(mod) case s.st != nil: - return s.st.Open(readOnly) + return s.st.Open(mod) default: - panic(fmt.Sprintf("unexpected storage call: Open(%v)", readOnly)) + panic(fmt.Sprintf("unexpected storage call: Open(%v)", mod.String())) } } diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 6bf6beac6..f0809883c 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -74,7 +74,7 @@ func TestInitializationFailure(t *testing.T) { openFileMetabase: os.OpenFile, openFilePilorama: os.OpenFile, }) - largeFileStorage.SetOption(teststore.WithOpen(func(ro bool) error { + largeFileStorage.SetOption(teststore.WithOpen(func(primitiveMode mode.ComponentMode) error { return teststore.ErrDiskExploded })) beforeReload := func() { diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 670cfcbf5..023576716 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -171,8 +171,8 @@ func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree) } -func (m *writeCacheMetrics) SetMode(mode mode.Mode) { - m.metrics.SetMode(m.shardID, mode.String()) +func (m *writeCacheMetrics) SetMode(mod mode.ComponentMode) { + m.metrics.SetMode(m.shardID, mod.String()) } func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index cd53f0cd2..891a1e9b2 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -38,16 +38,16 @@ var ( ) // Open boltDB instance for metabase. -func (db *DB) Open(_ context.Context, mode mode.Mode) error { +func (db *DB) Open(_ context.Context, m mode.Mode) error { db.modeMtx.Lock() defer db.modeMtx.Unlock() - db.mode = mode - db.metrics.SetMode(mode) + db.mode = m + db.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) - if mode.NoMetabase() { + if m.NoMetabase() { return nil } - return db.openDB(mode) + return db.openDB(m) } func (db *DB) openDB(mode mode.Mode) error { @@ -239,15 +239,15 @@ func (db *DB) Reload(opts ...Option) (bool, error) { return false, err } - db.mode = mode.Degraded - db.metrics.SetMode(mode.Degraded) + db.mode = mode.Disabled + db.metrics.SetMode(mode.ComponentDisabled) db.info.Path = c.info.Path if err := db.openBolt(); err != nil { return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite - db.metrics.SetMode(mode.ReadWrite) + db.metrics.SetMode(mode.ComponentReadWrite) return true, nil } diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go index fc971bd81..d673560c7 100644 --- a/pkg/local_object_storage/metabase/metrics.go +++ b/pkg/local_object_storage/metabase/metrics.go @@ -9,7 +9,7 @@ import ( type Metrics interface { SetParentID(parentID string) - SetMode(m mode.Mode) + SetMode(m mode.ComponentMode) Close() AddMethodDuration(method string, d time.Duration, success bool) @@ -18,6 +18,6 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index b382e99c2..2032ed6b2 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -35,6 +35,6 @@ func (db *DB) SetMode(m mode.Mode) error { } db.mode = m - db.metrics.SetMode(m) + db.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) return nil } diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index 7ae336a6c..9b24e98d7 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -69,7 +69,7 @@ func (db *DB) SetShardID(id []byte, mode metamode.Mode) error { err := db.writeShardID(id) if err == nil { - db.metrics.SetMode(mode) + db.metrics.SetMode(metamode.ConvertToComponentModeDegraded(mode)) } if cErr := db.close(); cErr != nil { diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 1e294efa5..6886bbc1d 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) @@ -34,8 +35,8 @@ func (m *blobovniczaTreeMetrics) SetParentID(parentID string) { m.shardID = parentID } -func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) { - m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly) +func (m *blobovniczaTreeMetrics) SetMode(mod mode.ComponentMode) { + m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, mod) } func (m *blobovniczaTreeMetrics) Close() { diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index e035b3a46..e6b3175be 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -4,6 +4,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) @@ -25,8 +26,8 @@ func (m *fstreeMetrics) SetParentID(parentID string) { m.shardID = parentID } -func (m *fstreeMetrics) SetMode(readOnly bool) { - m.m.SetMode(m.shardID, m.path, readOnly) +func (m *fstreeMetrics) SetMode(mod mode.ComponentMode) { + m.m.SetMode(m.shardID, m.path, mod) } func (m *fstreeMetrics) Close() { diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go index d0fb31936..1a02f2705 100644 --- a/pkg/local_object_storage/metrics/metabase.go +++ b/pkg/local_object_storage/metrics/metabase.go @@ -26,7 +26,7 @@ func (m *metabaseMetrics) SetParentID(parentID string) { m.shardID = parentID } -func (m *metabaseMetrics) SetMode(mode mode.Mode) { +func (m *metabaseMetrics) SetMode(mode mode.ComponentMode) { m.m.SetMode(m.shardID, m.path, mode.String()) } diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go index 21f027a6e..1b715c2bd 100644 --- a/pkg/local_object_storage/metrics/pilorama.go +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -24,8 +24,8 @@ func (m *piloramaMetrics) SetParentID(id string) { m.shardID = id } -func (m *piloramaMetrics) SetMode(mode mode.Mode) { - m.m.SetMode(m.shardID, mode) +func (m *piloramaMetrics) SetMode(mod mode.ComponentMode) { + m.m.SetMode(m.shardID, mod) } func (m *piloramaMetrics) Close() { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 29a9306b6..fe6bf4fed 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -109,7 +109,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { } t.mode = m - t.metrics.SetMode(m) + t.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) return nil } @@ -123,8 +123,8 @@ func (t *boltForest) Open(_ context.Context, mode mode.Mode) error { return t.openBolt(mode) } -func (t *boltForest) openBolt(mode mode.Mode) error { - readOnly := mode.ReadOnly() +func (t *boltForest) openBolt(m mode.Mode) error { + readOnly := m.ReadOnly() err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) @@ -143,7 +143,7 @@ func (t *boltForest) openBolt(mode mode.Mode) error { t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - t.metrics.SetMode(mode) + t.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) return nil } diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go index 543ad3e31..6ffc479e4 100644 --- a/pkg/local_object_storage/pilorama/metrics.go +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -9,7 +9,7 @@ import ( type Metrics interface { SetParentID(id string) - SetMode(m mode.Mode) + SetMode(m mode.ComponentMode) Close() AddMethodDuration(method string, d time.Duration, success bool) @@ -18,6 +18,6 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 6712822a0..38a29cb12 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -37,7 +37,7 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { err = s.SetMode(mode.DegradedReadOnly) if err != nil { - return fmt.Errorf("could not switch to mode %s", mode.DegradedReadOnly) + return fmt.Errorf("could not switch to mode %s", mode.Mode(mode.DegradedReadOnly)) } return nil } diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 461611465..dc4d52b0e 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -29,6 +29,22 @@ const ( DegradedReadOnly Mode = Degraded | ReadOnly ) +// ComponentMode represents basic operation modes for shared components, including READ, READ_WRITE, and DISABLED. +type ComponentMode uint32 + +const ( + // ComponentReadWrite is a Mode value for component that is available + // for read and write operations. Default component mode. + ComponentReadWrite ComponentMode = 0 + + // ComponentReadOnly is a Mode value for component that does not + // accept write operation but is readable. + ComponentReadOnly ComponentMode = 0b001 + + // ComponentDisabled mode is a mode where a component is disabled. + ComponentDisabled ComponentMode = math.MaxUint32 +) + func (m Mode) String() string { switch m { default: @@ -46,6 +62,19 @@ func (m Mode) String() string { } } +func (m ComponentMode) String() string { + switch m { + default: + return "UNDEFINED" + case ComponentReadWrite: + return "READ_WRITE" + case ComponentReadOnly: + return "READ_ONLY" + case ComponentDisabled: + return "CLOSED" + } +} + // NoMetabase returns true iff m is operating without the metabase. func (m Mode) NoMetabase() bool { return m&Degraded != 0 @@ -56,6 +85,39 @@ func (m Mode) ReadOnly() bool { return m&ReadOnly != 0 } +// ReadOnly returns true iff m prohibits modifying operations with shard. +func (m ComponentMode) ReadOnly() bool { + return m&ComponentReadOnly != 0 +} + func (m Mode) Disabled() bool { return m == Disabled } + +func (m ComponentMode) Disabled() bool { + return m == ComponentDisabled +} + +// ConvertToComponentModeDegraded converts a ShardMode to a corresponding ComponentMode. +// Disables the component if the node is in degraded mode. Used in Metabase, Writecache, Pilorama. +func ConvertToComponentModeDegraded(m Mode) ComponentMode { + if m.NoMetabase() || m.Disabled() { + return ComponentDisabled + } + if m.ReadOnly() { + return ComponentReadOnly + } + return ComponentReadWrite +} + +// ConvertToComponentMode converts a ShardMode to a corresponding ComponentMode. +// Ignores the degraded mode of the node. Used in Blobstore. +func ConvertToComponentMode(m Mode) ComponentMode { + if m.Disabled() { + return ComponentDisabled + } + if m.ReadOnly() { + return ComponentReadOnly + } + return ComponentReadWrite +} diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index fdba8d409..f2b48d746 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -95,14 +95,14 @@ func (c *cache) DumpInfo() Info { } // Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(_ context.Context, mode mode.Mode) error { +func (c *cache) Open(_ context.Context, mod mode.Mode) error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - c.mode = mode - if mode.NoMetabase() { + c.mode = mod + if mod.NoMetabase() { return nil } - err := c.openStore(mode.ReadOnly()) + err := c.openStore(mode.ConvertToComponentModeDegraded(mod)) if err != nil { return metaerr.Wrap(err) } @@ -112,7 +112,7 @@ func (c *cache) Open(_ context.Context, mode mode.Mode) error { // Init runs necessary services. func (c *cache) Init() error { - c.metrics.SetMode(c.mode) + c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode)) ctx, cancel := context.WithCancel(context.Background()) c.cancel = cancel c.runFlushLoop(ctx) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index c3e19c4d2..da7feda9a 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -294,7 +294,7 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { if err := c.setMode(ctx, m, ignoreErrors); err != nil { return err } - c.metrics.SetMode(m) + c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) } return nil } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index 962e22eea..e68b6d8be 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -27,7 +27,7 @@ type Metrics interface { Evict(st StorageType) SetEstimateSize(db, fstree uint64) - SetMode(m mode.Mode) + SetMode(m mode.ComponentMode) SetActualCounters(db, fstree uint64) SetPath(path string) Close() @@ -49,7 +49,7 @@ func (metricsStub) Put(time.Duration, bool, StorageType) {} func (metricsStub) SetEstimateSize(uint64, uint64) {} -func (metricsStub) SetMode(mode.Mode) {} +func (metricsStub) SetMode(mode.ComponentMode) {} func (metricsStub) SetActualCounters(uint64, uint64) {} diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index a10e593aa..4172cfbc8 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -27,7 +27,7 @@ func (c *cache) SetMode(m mode.Mode) error { err := c.setMode(ctx, m, true) if err == nil { - c.metrics.SetMode(m) + c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) } return err } @@ -63,7 +63,7 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) err return nil } - if err = c.openStore(m.ReadOnly()); err != nil { + if err = c.openStore(mode.ConvertToComponentModeDegraded(m)); err != nil { return err } diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go index be4fec367..48107a75f 100644 --- a/pkg/local_object_storage/writecache/seal.go +++ b/pkg/local_object_storage/writecache/seal.go @@ -22,7 +22,7 @@ func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error { // flush will be done by setMode err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors) if err == nil { - c.metrics.SetMode(mode.DegradedReadOnly) + c.metrics.SetMode(mode.ComponentDisabled) } return err } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 6cc3b06d0..caf997af8 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -25,13 +26,13 @@ type store struct { const dbName = "small.bolt" -func (c *cache) openStore(readOnly bool) error { +func (c *cache) openStore(mod mode.ComponentMode) error { err := util.MkdirAllX(c.path, os.ModePerm) if err != nil { return err } - c.db, err = OpenDB(c.path, readOnly, c.openFile) + c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile) if err != nil { return fmt.Errorf("could not open database: %w", err) } @@ -39,7 +40,7 @@ func (c *cache) openStore(readOnly bool) error { c.db.MaxBatchSize = c.maxBatchSize c.db.MaxBatchDelay = c.maxBatchDelay - if !readOnly { + if !mod.ReadOnly() { err = c.db.Update(func(tx *bbolt.Tx) error { _, err := tx.CreateBucketIfNotExists(defaultBucket) return err @@ -57,7 +58,7 @@ func (c *cache) openStore(readOnly bool) error { fstree.WithNoSync(c.noSync), fstree.WithFileCounter(&c.objCounters), ) - if err := c.fsTree.Open(readOnly); err != nil { + if err := c.fsTree.Open(mod); err != nil { return fmt.Errorf("could not open FSTree: %w", err) } if err := c.fsTree.Init(); err != nil { diff --git a/pkg/metrics/blobovnicza.go b/pkg/metrics/blobovnicza.go index f3a14a96e..948272c88 100644 --- a/pkg/metrics/blobovnicza.go +++ b/pkg/metrics/blobovnicza.go @@ -4,12 +4,13 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) type BlobobvnizcaMetrics interface { - SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) + SetBlobobvnizcaTreeMode(shardID, path string, mode mode.ComponentMode) CloseBlobobvnizcaTree(shardID, path string) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) AddBlobobvnizcaTreePut(shardID, path string, size int) @@ -98,8 +99,8 @@ func newBlobovnicza() *blobovnicza { } } -func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { - b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) +func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, mod mode.ComponentMode) { + b.treeMode.SetMode(shardID, path, mod.String()) } func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) { diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go index 4d4f0693b..ecd4352bb 100644 --- a/pkg/metrics/fstree.go +++ b/pkg/metrics/fstree.go @@ -4,12 +4,13 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) type FSTreeMetrics interface { - SetMode(shardID, path string, readOnly bool) + SetMode(shardID, path string, mode mode.ComponentMode) Close(shardID, path string) MethodDuration(shardID, path string, method string, d time.Duration, success bool) @@ -48,8 +49,8 @@ func newFSTreeMetrics() *fstreeMetrics { } } -func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { - m.mode.SetMode(shardID, path, modeFromBool(readOnly)) +func (m *fstreeMetrics) SetMode(shardID, path string, mod mode.ComponentMode) { + m.mode.SetMode(shardID, path, mod.String()) } func (m *fstreeMetrics) Close(shardID, path string) { diff --git a/pkg/metrics/pilorama.go b/pkg/metrics/pilorama.go index 41672a4b5..c669275fe 100644 --- a/pkg/metrics/pilorama.go +++ b/pkg/metrics/pilorama.go @@ -10,7 +10,7 @@ import ( ) type PiloramaMetrics interface { - SetMode(shardID string, m mode.Mode) + SetMode(shardID string, m mode.ComponentMode) Close(shardID string) AddMethodDuration(shardID string, method string, d time.Duration, success bool) @@ -33,7 +33,7 @@ type piloramaMetrics struct { reqDuration *prometheus.HistogramVec } -func (m *piloramaMetrics) SetMode(shardID string, mode mode.Mode) { +func (m *piloramaMetrics) SetMode(shardID string, mode mode.ComponentMode) { m.mode.SetMode(shardID, mode.String()) } -- 2.45.3 From ce7500f00313a4156c591be8caa01db4d7064185 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 4 Jun 2024 16:29:27 +0300 Subject: [PATCH 3/3] [#1121] docs: Change mode of shard components Signed-off-by: Alexander Chuprov --- docs/shard-modes.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/shard-modes.md b/docs/shard-modes.md index 789a1c241..3b459335b 100644 --- a/docs/shard-modes.md +++ b/docs/shard-modes.md @@ -14,7 +14,16 @@ Each mode is characterized by two important properties: | `read-only` | Read-only mode, only read operations are allowed, metabase is available. | | `degraded` | Degraded mode in which metabase and write-cache is disabled. It shouldn't be used at all, because metabase can contain important indices, such as LOCK objects info and modifying operation in this mode can lead to unexpected behaviour. The purpose of this mode is to allow PUT/DELETE operations without the metabase if really necessary. | | `degraded-read-only` | Same as `degraded`, but with only read operations allowed. This mode is used during SSD replacement and/or when the metabase error counter exceeds threshold. | -| `disabled` | Currently used only in config file to temporarily disable a shard. | +| `disabled` | Currently used only in config file to temporarily disable a shard. + +## Shard and Component Status + +| Shard Mode | Metabase Mode | Blobstore Mode | Writecache Mode | Pilorama Mode | Blobovnicza Tree Mode | FSTree Mode | +|-----------------------|---------------|----------------|-----------------|---------------|-----------------------|-------------| +| `Read-Write` | READ_WRITE | READ_WRITE | READ_WRITE | READ_WRITE | READ_WRITE | READ_WRITE | +| `Read-Only` | READ_ONLY | READ_ONLY | READ_ONLY | READ_ONLY | READ_ONLY | READ_ONLY | +| `Degraded-Read-Write` | CLOSED | READ_WRITE | CLOSED | CLOSED | READ_WRITE | READ_WRITE | +| `Degraded-Read-Only` | CLOSED | READ_ONLY | CLOSED | CLOSED | READ_ONLY | READ_ONLY | ## Transition order -- 2.45.3