TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 5 Projects Releases 1 Activity Actions

[#1121] node: Change mode of shard components

Browse source 
Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
This commit is contained in:
Alexander Chuprov 2024-06-04 16:28:47 +03:00
parent 0b0d829346
commit d61f78ec1f
47 changed files with 194 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go
View file

@ -8,6 +8,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
"github.com/stretchr/testify/require"
)
@ -23,7 +24,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) {
WithBlobovniczaShallowWidth(10),
WithBlobovniczaShallowDepth(1),
WithRootPath(t.TempDir()))
require.NoError(t, st.Open(false))
require.NoError(t, st.Open(mode.ComponentReadWrite))
require.NoError(t, st.Init())
defer func() {
require.NoError(t, st.Close())

7
pkg/local_object_storage/blobstor/blobovniczatree/control.go
View file

@ -8,6 +8,7 @@ import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
@ -16,9 +17,9 @@ import (
var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode")
// Open opens blobovnicza tree.
func (b *Blobovniczas) Open(readOnly bool) error {
b.readOnly = readOnly
b.metrics.SetMode(readOnly)
func (b *Blobovniczas) Open(mode mode.ComponentMode) error {
b.readOnly = mode.ReadOnly()
b.metrics.SetMode(mode)
b.metrics.SetRebuildStatus(rebuildStatusNotStarted)
b.openManagers()
return nil

9
pkg/local_object_storage/blobstor/blobovniczatree/control_test.go
View file

@ -11,6 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require"
)
@ -54,7 +55,7 @@ func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) {
WithBlobovniczaShallowWidth(width),
WithRootPath(path),
)
require.NoError(t, blz.Open(false))
require.NoError(t, blz.Open(mode.ComponentReadWrite))
require.NoError(t, blz.Init())
require.NoError(t, blz.Close())
}
@ -85,7 +86,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) {
WithRootPath(rootDir),
)
require.NoError(t, blz.Open(false))
require.NoError(t, blz.Open(mode.ComponentReadWrite))
require.NoError(t, blz.Init())
obj35 := blobstortest.NewObject(10 * 1024)
@ -123,7 +124,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) {
WithRootPath(rootDir),
)
require.NoError(t, blz.Open(false))
require.NoError(t, blz.Open(mode.ComponentReadWrite))
require.NoError(t, blz.Init())
gRes, err = blz.Get(context.Background(), common.GetPrm{
@ -160,7 +161,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) {
WithBlobovniczaShallowWidth(5),
WithRootPath(rootDir),
)
require.NoError(t, blz.Open(false))
require.NoError(t, blz.Open(mode.ComponentReadWrite))
require.NoError(t, blz.Init())
gRes, err = blz.Get(context.Background(), common.GetPrm{

3
pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go
View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
"github.com/stretchr/testify/require"
)
@ -24,7 +25,7 @@ func TestExistsInvalidStorageID(t *testing.T) {
WithBlobovniczaShallowDepth(2),
WithRootPath(dir),
WithBlobovniczaSize(1<<20))
require.NoError(t, b.Open(false))
require.NoError(t, b.Open(mode.ComponentReadWrite))
require.NoError(t, b.Init())
defer func() { require.NoError(t, b.Close()) }()

3
pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go
View file

@ -4,6 +4,7 @@ import (
"context"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
@ -18,7 +19,7 @@ func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) {
WithRootPath(t.TempDir()),
)
blz.createDBInAdvance = true
require.NoError(t, blz.Open(false))
require.NoError(t, blz.Open(mode.ComponentReadWrite))
require.NoError(t, blz.Init())
defer func() {
require.NoError(t, blz.Close())

5
pkg/local_object_storage/blobstor/blobovniczatree/metrics.go
View file

@ -4,6 +4,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
const (
@ -18,7 +19,7 @@ type Metrics interface {
SetParentID(parentID string)
SetMode(readOnly bool)
SetMode(mode.ComponentMode)
Close()
SetRebuildStatus(status string)
@ -37,7 +38,7 @@ type Metrics interface {
type noopMetrics struct{}
func (m *noopMetrics) SetParentID(string) {}
func (m *noopMetrics) SetMode(bool) {}
func (m *noopMetrics) SetMode(mode.ComponentMode) {}
func (m *noopMetrics) Close() {}
func (m *noopMetrics) SetRebuildStatus(string) {}
func (m *noopMetrics) SetRebuildPercent(uint32) {}

3
pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go
View file

@ -11,6 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -138,7 +139,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object
WithBlobovniczaSize(100*1024*1024),
WithWaitBeforeDropDB(0),
WithOpenedCacheSize(1000))
require.NoError(t, b.Open(false))
require.NoError(t, b.Open(mode.ComponentReadWrite))
require.NoError(t, b.Init())
var dPrm common.DeletePrm

9
pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go
View file

@ -8,6 +8,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/stretchr/testify/require"
@ -53,7 +54,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) {
WithWaitBeforeDropDB(0),
WithOpenedCacheSize(1000),
WithMoveBatchSize(3))
require.NoError(t, b.Open(false))
require.NoError(t, b.Open(mode.ComponentReadWrite))
require.NoError(t, b.Init())
obj := blobstortest.NewObject(64 * 1024) // 64KB object
@ -81,7 +82,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) {
WithWaitBeforeDropDB(0),
WithOpenedCacheSize(1000),
WithMoveBatchSize(3))
require.NoError(t, b.Open(false))
require.NoError(t, b.Open(mode.ComponentReadWrite))
require.NoError(t, b.Init())
metaStub := &storageIDUpdateStub{
@ -120,7 +121,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta
WithWaitBeforeDropDB(0),
WithOpenedCacheSize(1000),
WithMoveBatchSize(3))
require.NoError(t, b.Open(false))
require.NoError(t, b.Open(mode.ComponentReadWrite))
require.NoError(t, b.Init())
eg, egCtx := errgroup.WithContext(context.Background())
@ -161,7 +162,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta
WithWaitBeforeDropDB(0),
WithOpenedCacheSize(1000),
WithMoveBatchSize(50))
require.NoError(t, b.Open(false))
require.NoError(t, b.Open(mode.ComponentReadWrite))
require.NoError(t, b.Init())
for addr, storageID := range storageIDs {

3
pkg/local_object_storage/blobstor/common/storage.go
View file

@ -4,12 +4,13 @@ import (
"context"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
// Storage represents key-value object storage.
// It is used as a building block for a blobstor of a shard.
type Storage interface {
Open(readOnly bool) error
Open(mode mode.ComponentMode) error
Init() error
Close() error

5
pkg/local_object_storage/blobstor/control.go
View file

@ -27,15 +27,14 @@ func (b *BlobStor) Open(ctx context.Context, mode mode.Mode) error {
return nil
}
func (b *BlobStor) openBlobStor(ctx context.Context, mode mode.Mode) error {
readOnly := mode.ReadOnly()
func (b *BlobStor) openBlobStor(ctx context.Context, mod mode.Mode) error {
for i := range b.storage {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
err := b.storage[i].Storage.Open(readOnly)
err := b.storage[i].Storage.Open(mode.ConvertToComponentMode(mod))
if err != nil {
return err
}

7
pkg/local_object_storage/blobstor/fstree/control.go
View file

@ -1,13 +1,14 @@
package fstree
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
)
// Open implements common.Storage.
func (t *FSTree) Open(ro bool) error {
t.readOnly = ro
t.metrics.SetMode(ro)
func (t *FSTree) Open(mode mode.ComponentMode) error {
t.readOnly = mode.ReadOnly()
t.metrics.SetMode(mode)
return nil
}

3
pkg/local_object_storage/blobstor/fstree/fstree_test.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
@ -43,7 +44,7 @@ func TestObjectCounter(t *testing.T) {
WithDepth(2),
WithDirNameLen(2),
WithFileCounter(counter))
require.NoError(t, fst.Open(false))
require.NoError(t, fst.Open(mode.ComponentReadWrite))
require.NoError(t, fst.Init())
counterValue := counter.Value()

10
pkg/local_object_storage/blobstor/fstree/metrics.go
View file

@ -1,11 +1,15 @@
package fstree
import "time"
import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
type Metrics interface {
SetParentID(parentID string)
SetMode(readOnly bool)
SetMode(mode mode.ComponentMode)
Close()
Iterate(d time.Duration, success bool)
@ -20,7 +24,7 @@ type Metrics interface {
type noopMetrics struct{}
func (m *noopMetrics) SetParentID(string) {}
func (m *noopMetrics) SetMode(bool) {}
func (m *noopMetrics) SetMode(mode.ComponentMode) {}
func (m *noopMetrics) Close() {}
func (m *noopMetrics) Iterate(time.Duration, bool) {}
func (m *noopMetrics) Delete(time.Duration, bool) {}

5
pkg/local_object_storage/blobstor/internal/blobstortest/control.go
View file

@ -7,6 +7,7 @@ import (
objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require"
)
@ -14,13 +15,13 @@ import (
// cons must return a storage which is NOT opened.
func TestControl(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
objects := prepare(t, 10, s, min, max)
require.NoError(t, s.Close())
require.NoError(t, s.Open(true))
require.NoError(t, s.Open(mode.ComponentReadOnly))
for i := range objects {
var prm common.GetPrm
prm.Address = objects[i].addr

3
pkg/local_object_storage/blobstor/internal/blobstortest/delete.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
@ -12,7 +13,7 @@ import (
func TestDelete(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()

3
pkg/local_object_storage/blobstor/internal/blobstortest/exists.go
View file

@ -5,13 +5,14 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestExists(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()

3
pkg/local_object_storage/blobstor/internal/blobstortest/get.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
@ -12,7 +13,7 @@ import (
func TestGet(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()

3
pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
@ -14,7 +15,7 @@ import (
func TestGetRange(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()

3
pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go
View file

@ -6,12 +6,13 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require"
)
func TestIterate(t *testing.T, cons Constructor, min, max uint64) {
s := cons(t)
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()

9
pkg/local_object_storage/blobstor/memstore/control.go
View file

@ -1,9 +1,12 @@
package memstore
import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
func (s *memstoreImpl) Open(readOnly bool) error {
s.readOnly = readOnly
func (s *memstoreImpl) Open(mod mode.ComponentMode) error {
s.readOnly = mod.ReadOnly()
return nil
}

3
pkg/local_object_storage/blobstor/memstore/memstore_test.go
View file

@ -7,6 +7,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"github.com/stretchr/testify/require"
@ -18,7 +19,7 @@ func TestSimpleLifecycle(t *testing.T) {
WithLogger(test.NewLogger(t)),
)
defer func() { require.NoError(t, s.Close()) }()
require.NoError(t, s.Open(false))
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
obj := blobstortest.NewObject(1024)

3
pkg/local_object_storage/blobstor/perf_test.go
View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
@ -22,7 +23,7 @@ type storage struct {
func (s storage) open(b *testing.B) common.Storage {
st := s.create(b.TempDir())
require.NoError(b, st.Open(false))
require.NoError(b, st.Open(mode.ComponentReadWrite))
require.NoError(b, st.Init())
return st

9
pkg/local_object_storage/blobstor/teststore/option.go
View file

@ -3,12 +3,13 @@ package teststore
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
type cfg struct {
st common.Storage
overrides struct {
Open func(readOnly bool) error
Open func(mode mode.ComponentMode) error
Init func() error
Close func() error
@ -35,9 +36,9 @@ func WithSubstorage(st common.Storage) Option {
}
}
func WithOpen(f func(bool) error) Option { return func(c *cfg) { c.overrides.Open = f } }
func WithInit(f func() error) Option { return func(c *cfg) { c.overrides.Init = f } }
func WithClose(f func() error) Option { return func(c *cfg) { c.overrides.Close = f } }
func WithOpen(f func(mode.ComponentMode) error) Option { return func(c *cfg) { c.overrides.Open = f } }
func WithInit(f func() error) Option { return func(c *cfg) { c.overrides.Init = f } }
func WithClose(f func() error) Option { return func(c *cfg) { c.overrides.Close = f } }
func WithType(f func() string) Option { return func(c *cfg) { c.overrides.Type = f } }
func WithPath(f func() string) Option { return func(c *cfg) { c.overrides.Path = f } }

9
pkg/local_object_storage/blobstor/teststore/teststore.go
View file

@ -20,6 +20,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
// TestStore is a common.Storage implementation for testing/mocking purposes.
@ -50,16 +51,16 @@ func (s *TestStore) SetOption(opt Option) {
opt(s.cfg)
}
func (s *TestStore) Open(readOnly bool) error {
func (s *TestStore) Open(mod mode.ComponentMode) error {
s.mu.RLock()
defer s.mu.RUnlock()
switch {
case s.overrides.Open != nil:
return s.overrides.Open(readOnly)
return s.overrides.Open(mod)
case s.st != nil:
return s.st.Open(readOnly)
return s.st.Open(mod)
default:
panic(fmt.Sprintf("unexpected storage call: Open(%v)", readOnly))
panic(fmt.Sprintf("unexpected storage call: Open(%v)", mod.String()))
}
}

2
pkg/local_object_storage/engine/control_test.go
View file

@ -74,7 +74,7 @@ func TestInitializationFailure(t *testing.T) {
openFileMetabase: os.OpenFile,
openFilePilorama: os.OpenFile,
})
largeFileStorage.SetOption(teststore.WithOpen(func(ro bool) error {
largeFileStorage.SetOption(teststore.WithOpen(func(primitiveMode mode.ComponentMode) error {
return teststore.ErrDiskExploded
}))
beforeReload := func() {

4
pkg/local_object_storage/engine/writecache.go
View file

@ -171,8 +171,8 @@ func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) {
m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree)
}
func (m *writeCacheMetrics) SetMode(mode mode.Mode) {
m.metrics.SetMode(m.shardID, mode.String())
func (m *writeCacheMetrics) SetMode(mod mode.ComponentMode) {
m.metrics.SetMode(m.shardID, mod.String())
}
func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) {

16
pkg/local_object_storage/metabase/control.go
View file

@ -38,16 +38,16 @@ var (
)
// Open boltDB instance for metabase.
func (db *DB) Open(_ context.Context, mode mode.Mode) error {
func (db *DB) Open(_ context.Context, m mode.Mode) error {
db.modeMtx.Lock()
defer db.modeMtx.Unlock()
db.mode = mode
db.metrics.SetMode(mode)
db.mode = m
db.metrics.SetMode(mode.ConvertToComponentModeDegraded(m))
if mode.NoMetabase() {
if m.NoMetabase() {
return nil
}
return db.openDB(mode)
return db.openDB(m)
}
func (db *DB) openDB(mode mode.Mode) error {
@ -239,15 +239,15 @@ func (db *DB) Reload(opts ...Option) (bool, error) {
return false, err
}
db.mode = mode.Degraded
db.metrics.SetMode(mode.Degraded)
db.mode = mode.Disabled
db.metrics.SetMode(mode.ComponentDisabled)
db.info.Path = c.info.Path
if err := db.openBolt(); err != nil {
return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err))
}
db.mode = mode.ReadWrite
db.metrics.SetMode(mode.ReadWrite)
db.metrics.SetMode(mode.ComponentReadWrite)
return true, nil
}

4
pkg/local_object_storage/metabase/metrics.go
View file

@ -9,7 +9,7 @@ import (
type Metrics interface {
SetParentID(parentID string)
SetMode(m mode.Mode)
SetMode(m mode.ComponentMode)
Close()
AddMethodDuration(method string, d time.Duration, success bool)
@ -18,6 +18,6 @@ type Metrics interface {
type noopMetrics struct{}
func (m *noopMetrics) SetParentID(string) {}
func (m *noopMetrics) SetMode(mode.Mode) {}
func (m *noopMetrics) SetMode(mode.ComponentMode) {}
func (m *noopMetrics) Close() {}
func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {}

2
pkg/local_object_storage/metabase/mode.go
View file

@ -35,6 +35,6 @@ func (db *DB) SetMode(m mode.Mode) error {
}
db.mode = m
db.metrics.SetMode(m)
db.metrics.SetMode(mode.ConvertToComponentModeDegraded(m))
return nil
}

2
pkg/local_object_storage/metabase/shard_id.go
View file

@ -69,7 +69,7 @@ func (db *DB) SetShardID(id []byte, mode metamode.Mode) error {
err := db.writeShardID(id)
if err == nil {
db.metrics.SetMode(mode)
db.metrics.SetMode(metamode.ConvertToComponentModeDegraded(mode))
}
if cErr := db.close(); cErr != nil {

5
pkg/local_object_storage/metrics/blobovnicza.go
View file

@ -5,6 +5,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
)
@ -34,8 +35,8 @@ func (m *blobovniczaTreeMetrics) SetParentID(parentID string) {
m.shardID = parentID
}
func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) {
m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly)
func (m *blobovniczaTreeMetrics) SetMode(mod mode.ComponentMode) {
m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, mod)
}
func (m *blobovniczaTreeMetrics) Close() {

5
pkg/local_object_storage/metrics/fstree.go
View file

@ -4,6 +4,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
)
@ -25,8 +26,8 @@ func (m *fstreeMetrics) SetParentID(parentID string) {
m.shardID = parentID
}
func (m *fstreeMetrics) SetMode(readOnly bool) {
m.m.SetMode(m.shardID, m.path, readOnly)
func (m *fstreeMetrics) SetMode(mod mode.ComponentMode) {
m.m.SetMode(m.shardID, m.path, mod)
}
func (m *fstreeMetrics) Close() {

2
pkg/local_object_storage/metrics/metabase.go
View file

@ -26,7 +26,7 @@ func (m *metabaseMetrics) SetParentID(parentID string) {
m.shardID = parentID
}
func (m *metabaseMetrics) SetMode(mode mode.Mode) {
func (m *metabaseMetrics) SetMode(mode mode.ComponentMode) {
m.m.SetMode(m.shardID, m.path, mode.String())
}

4
pkg/local_object_storage/metrics/pilorama.go
View file

@ -24,8 +24,8 @@ func (m *piloramaMetrics) SetParentID(id string) {
m.shardID = id
}
func (m *piloramaMetrics) SetMode(mode mode.Mode) {
m.m.SetMode(m.shardID, mode)
func (m *piloramaMetrics) SetMode(mod mode.ComponentMode) {
m.m.SetMode(m.shardID, mod)
}
func (m *piloramaMetrics) Close() {

8
pkg/local_object_storage/pilorama/boltdb.go
View file

@ -109,7 +109,7 @@ func (t *boltForest) SetMode(m mode.Mode) error {
}
t.mode = m
t.metrics.SetMode(m)
t.metrics.SetMode(mode.ConvertToComponentModeDegraded(m))
return nil
}
@ -123,8 +123,8 @@ func (t *boltForest) Open(_ context.Context, mode mode.Mode) error {
return t.openBolt(mode)
}
func (t *boltForest) openBolt(mode mode.Mode) error {
readOnly := mode.ReadOnly()
func (t *boltForest) openBolt(m mode.Mode) error {
readOnly := m.ReadOnly()
err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
if err != nil {
return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err))
@ -143,7 +143,7 @@ func (t *boltForest) openBolt(mode mode.Mode) error {
t.db.MaxBatchSize = t.maxBatchSize
t.db.MaxBatchDelay = t.maxBatchDelay
t.metrics.SetMode(mode)
t.metrics.SetMode(mode.ConvertToComponentModeDegraded(m))
return nil
}

4
pkg/local_object_storage/pilorama/metrics.go
View file

@ -9,7 +9,7 @@ import (
type Metrics interface {
SetParentID(id string)
SetMode(m mode.Mode)
SetMode(m mode.ComponentMode)
Close()
AddMethodDuration(method string, d time.Duration, success bool)
@ -18,6 +18,6 @@ type Metrics interface {
type noopMetrics struct{}
func (m *noopMetrics) SetParentID(string) {}
func (m *noopMetrics) SetMode(mode.Mode) {}
func (m *noopMetrics) SetMode(mode.ComponentMode) {}
func (m *noopMetrics) Close() {}
func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {}

2
pkg/local_object_storage/shard/control.go
View file

@ -37,7 +37,7 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error {
err = s.SetMode(mode.DegradedReadOnly)
if err != nil {
return fmt.Errorf("could not switch to mode %s", mode.DegradedReadOnly)
return fmt.Errorf("could not switch to mode %s", mode.Mode(mode.DegradedReadOnly))
}
return nil
}

62
pkg/local_object_storage/shard/mode/mode.go
View file

@ -29,6 +29,22 @@ const (
DegradedReadOnly Mode = Degraded | ReadOnly
)
// ComponentMode represents basic operation modes for shared components, including READ, READ_WRITE, and DISABLED.
type ComponentMode uint32
const (
// ComponentReadWrite is a Mode value for component that is available
// for read and write operations. Default component mode.
ComponentReadWrite ComponentMode = 0
// ComponentReadOnly is a Mode value for component that does not
// accept write operation but is readable.
ComponentReadOnly ComponentMode = 0b001
// ComponentDisabled mode is a mode where a component is disabled.
ComponentDisabled ComponentMode = math.MaxUint32
)
func (m Mode) String() string {
switch m {
default:
@ -46,6 +62,19 @@ func (m Mode) String() string {
}
}
func (m ComponentMode) String() string {
switch m {
default:
return "UNDEFINED"
case ComponentReadWrite:
return "READ_WRITE"
case ComponentReadOnly:
return "READ_ONLY"
case ComponentDisabled:
return "CLOSED"
}
}
// NoMetabase returns true iff m is operating without the metabase.
func (m Mode) NoMetabase() bool {
return m&Degraded != 0
@ -56,6 +85,39 @@ func (m Mode) ReadOnly() bool {
return m&ReadOnly != 0
}
// ReadOnly returns true iff m prohibits modifying operations with shard.
func (m ComponentMode) ReadOnly() bool {
return m&ComponentReadOnly != 0
}
func (m Mode) Disabled() bool {
return m == Disabled
}
func (m ComponentMode) Disabled() bool {
return m == ComponentDisabled
}
// ConvertToComponentModeDegraded converts a ShardMode to a corresponding ComponentMode.
// Disables the component if the node is in degraded mode. Used in Metabase, Writecache, Pilorama.
func ConvertToComponentModeDegraded(m Mode) ComponentMode {
if m.NoMetabase() || m.Disabled() {
return ComponentDisabled
}
if m.ReadOnly() {
return ComponentReadOnly
}
return ComponentReadWrite
}
// ConvertToComponentMode converts a ShardMode to a corresponding ComponentMode.
// Ignores the degraded mode of the node. Used in Blobstore.
func ConvertToComponentMode(m Mode) ComponentMode {
if m.Disabled() {
return ComponentDisabled
}
if m.ReadOnly() {
return ComponentReadOnly
}
return ComponentReadWrite
}

10
pkg/local_object_storage/writecache/cachebbolt.go
View file

@ -95,14 +95,14 @@ func (c *cache) DumpInfo() Info {
}
// Open opens and initializes database. Reads object counters from the ObjectCounters instance.
func (c *cache) Open(_ context.Context, mode mode.Mode) error {
func (c *cache) Open(_ context.Context, mod mode.Mode) error {
c.modeMtx.Lock()
defer c.modeMtx.Unlock()
c.mode = mode
if mode.NoMetabase() {
c.mode = mod
if mod.NoMetabase() {
return nil
}
err := c.openStore(mode.ReadOnly())
err := c.openStore(mode.ConvertToComponentModeDegraded(mod))
if err != nil {
return metaerr.Wrap(err)
}
@ -112,7 +112,7 @@ func (c *cache) Open(_ context.Context, mode mode.Mode) error {
// Init runs necessary services.
func (c *cache) Init() error {
c.metrics.SetMode(c.mode)
c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode))
ctx, cancel := context.WithCancel(context.Background())
c.cancel = cancel
c.runFlushLoop(ctx)

2
pkg/local_object_storage/writecache/flush.go
View file

@ -294,7 +294,7 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error {
if err := c.setMode(ctx, m, ignoreErrors); err != nil {
return err
}
c.metrics.SetMode(m)
c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m))
}
return nil
}

4
pkg/local_object_storage/writecache/metrics.go
View file

@ -27,7 +27,7 @@ type Metrics interface {
Evict(st StorageType)
SetEstimateSize(db, fstree uint64)
SetMode(m mode.Mode)
SetMode(m mode.ComponentMode)
SetActualCounters(db, fstree uint64)
SetPath(path string)
Close()
@ -49,7 +49,7 @@ func (metricsStub) Put(time.Duration, bool, StorageType) {}
func (metricsStub) SetEstimateSize(uint64, uint64) {}
func (metricsStub) SetMode(mode.Mode) {}
func (metricsStub) SetMode(mode.ComponentMode) {}
func (metricsStub) SetActualCounters(uint64, uint64) {}

4
pkg/local_object_storage/writecache/mode.go
View file

@ -27,7 +27,7 @@ func (c *cache) SetMode(m mode.Mode) error {
err := c.setMode(ctx, m, true)
if err == nil {
c.metrics.SetMode(m)
c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m))
}
return err
}
@ -63,7 +63,7 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) err
return nil
}
if err = c.openStore(m.ReadOnly()); err != nil {
if err = c.openStore(mode.ConvertToComponentModeDegraded(m)); err != nil {
return err
}

2
pkg/local_object_storage/writecache/seal.go
View file

@ -22,7 +22,7 @@ func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error {
// flush will be done by setMode
err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors)
if err == nil {
c.metrics.SetMode(mode.DegradedReadOnly)
c.metrics.SetMode(mode.ComponentDisabled)
}
return err
}

9
pkg/local_object_storage/writecache/storage.go
View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -25,13 +26,13 @@ type store struct {
const dbName = "small.bolt"
func (c *cache) openStore(readOnly bool) error {
func (c *cache) openStore(mod mode.ComponentMode) error {
err := util.MkdirAllX(c.path, os.ModePerm)
if err != nil {
return err
}
c.db, err = OpenDB(c.path, readOnly, c.openFile)
c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile)
if err != nil {
return fmt.Errorf("could not open database: %w", err)
}
@ -39,7 +40,7 @@ func (c *cache) openStore(readOnly bool) error {
c.db.MaxBatchSize = c.maxBatchSize
c.db.MaxBatchDelay = c.maxBatchDelay
if !readOnly {
if !mod.ReadOnly() {
err = c.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(defaultBucket)
return err
@ -57,7 +58,7 @@ func (c *cache) openStore(readOnly bool) error {
fstree.WithNoSync(c.noSync),
fstree.WithFileCounter(&c.objCounters),
)
if err := c.fsTree.Open(readOnly); err != nil {
if err := c.fsTree.Open(mod); err != nil {
return fmt.Errorf("could not open FSTree: %w", err)
}
if err := c.fsTree.Init(); err != nil {

7
pkg/metrics/blobovnicza.go
View file

@ -4,12 +4,13 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
type BlobobvnizcaMetrics interface {
SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool)
SetBlobobvnizcaTreeMode(shardID, path string, mode mode.ComponentMode)
CloseBlobobvnizcaTree(shardID, path string)
BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool)
AddBlobobvnizcaTreePut(shardID, path string, size int)
@ -98,8 +99,8 @@ func newBlobovnicza() *blobovnicza {
}
}
func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) {
b.treeMode.SetMode(shardID, path, modeFromBool(readOnly))
func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, mod mode.ComponentMode) {
b.treeMode.SetMode(shardID, path, mod.String())
}
func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {

7
pkg/metrics/fstree.go
View file

@ -4,12 +4,13 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
type FSTreeMetrics interface {
SetMode(shardID, path string, readOnly bool)
SetMode(shardID, path string, mode mode.ComponentMode)
Close(shardID, path string)
MethodDuration(shardID, path string, method string, d time.Duration, success bool)
@ -48,8 +49,8 @@ func newFSTreeMetrics() *fstreeMetrics {
}
}
func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) {
m.mode.SetMode(shardID, path, modeFromBool(readOnly))
func (m *fstreeMetrics) SetMode(shardID, path string, mod mode.ComponentMode) {
m.mode.SetMode(shardID, path, mod.String())
}
func (m *fstreeMetrics) Close(shardID, path string) {

4
pkg/metrics/pilorama.go
View file

@ -10,7 +10,7 @@ import (
)
type PiloramaMetrics interface {
SetMode(shardID string, m mode.Mode)
SetMode(shardID string, m mode.ComponentMode)
Close(shardID string)
AddMethodDuration(shardID string, method string, d time.Duration, success bool)
@ -33,7 +33,7 @@ type piloramaMetrics struct {
reqDuration *prometheus.HistogramVec
}
func (m *piloramaMetrics) SetMode(shardID string, mode mode.Mode) {
func (m *piloramaMetrics) SetMode(shardID string, mode mode.ComponentMode) {
m.mode.SetMode(shardID, mode.String())
}