[#1559] local_object_storage: Allow to set mode for all components

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-05 07:55:46 +03:00 committed by fyrchik
parent 1e786233bf
commit 3df62769c0
16 changed files with 230 additions and 28 deletions

View file

@ -835,6 +835,11 @@ func (b *blobovniczas) init() error {
return zstdD(data) return zstdD(data)
} }
if b.readOnly {
b.log.Debug("read-only mode, skip blobovniczas initialization...")
return nil
}
return b.iterateBlobovniczas(false, func(p string, blz *blobovnicza.Blobovnicza) error { return b.iterateBlobovniczas(false, func(p string, blz *blobovnicza.Blobovnicza) error {
if err := blz.Init(); err != nil { if err := blz.Init(); err != nil {
return fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err) return fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err)

View file

@ -4,9 +4,11 @@ import (
"encoding/hex" "encoding/hex"
"io/fs" "io/fs"
"path/filepath" "path/filepath"
"sync"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/util/logger" "github.com/nspcc-dev/neofs-node/pkg/util/logger"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -16,6 +18,9 @@ type BlobStor struct {
*cfg *cfg
blobovniczas *blobovniczas blobovniczas *blobovniczas
modeMtx sync.RWMutex
mode mode.Mode
} }
type Info = fstree.Info type Info = fstree.Info

View file

@ -0,0 +1,35 @@
package blobstor
import (
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
)
// SetMode sets the blobstor mode of operation.
func (b *BlobStor) SetMode(m mode.Mode) error {
b.modeMtx.Lock()
defer b.modeMtx.Unlock()
if b.mode == m {
return nil
}
if b.mode.ReadOnly() == m.ReadOnly() {
return nil
}
err := b.Close()
if err == nil {
if err = b.Open(m.ReadOnly()); err == nil {
err = b.Init()
}
}
if err != nil {
return fmt.Errorf("can't set blobstor mode (old=%s, new=%s): %w", b.mode, m, err)
}
b.blobovniczas.readOnly = m.ReadOnly()
b.mode = m
return nil
}

View file

@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
} }
checkShardState(t, e, id[0], 2, mode.Degraded) checkShardState(t, e, id[0], 1, mode.Degraded)
checkShardState(t, e, id[1], 0, mode.ReadWrite) checkShardState(t, e, id[1], 0, mode.ReadWrite)
} }

View file

@ -74,7 +74,14 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
var shPrm shard.GetPrm var shPrm shard.GetPrm
shPrm.SetAddress(prm.addr) shPrm.SetAddress(prm.addr)
var hasDegraded bool
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
noMeta := sh.GetMode().NoMetabase()
shPrm.SetIgnoreMeta(noMeta)
hasDegraded = hasDegraded || noMeta
res, err := sh.Get(shPrm) res, err := sh.Get(shPrm)
if err != nil { if err != nil {
if res.HasMeta() { if res.HasMeta() {
@ -122,7 +129,7 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
} }
if obj == nil { if obj == nil {
if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) { if !hasDegraded && shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
return GetRes{}, outError return GetRes{}, outError
} }
@ -132,6 +139,11 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
shPrm.SetIgnoreMeta(true) shPrm.SetIgnoreMeta(true)
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
if sh.GetMode().NoMetabase() {
// Already visited.
return false
}
res, err := sh.Get(shPrm) res, err := sh.Get(shPrm)
obj = res.Object() obj = res.Object()
return err == nil return err == nil
@ -139,9 +151,11 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
if obj == nil { if obj == nil {
return GetRes{}, outError return GetRes{}, outError
} }
if shardWithMeta.Shard != nil {
e.reportShardError(shardWithMeta, "meta info was present, but object is missing", e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
metaError, zap.Stringer("address", prm.addr)) metaError, zap.Stringer("address", prm.addr))
} }
}
return GetRes{ return GetRes{
obj: obj, obj: obj,

View file

@ -70,6 +70,10 @@ func (db *DB) Reset() error {
} }
func (db *DB) init(reset bool) error { func (db *DB) init(reset bool) error {
if db.mode.NoMetabase() || db.mode.ReadOnly() {
return nil
}
mStaticBuckets := map[string]struct{}{ mStaticBuckets := map[string]struct{}{
string(containerVolumeBucketName): {}, string(containerVolumeBucketName): {},
string(graveyardBucketName): {}, string(graveyardBucketName): {},

View file

@ -8,9 +8,11 @@ import (
"os" "os"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
v2object "github.com/nspcc-dev/neofs-api-go/v2/object" v2object "github.com/nspcc-dev/neofs-api-go/v2/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/util/logger" "github.com/nspcc-dev/neofs-node/pkg/util/logger"
"github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
@ -26,6 +28,9 @@ type matcher struct {
type DB struct { type DB struct {
*cfg *cfg
modeMtx sync.RWMutex
mode mode.Mode
matchers map[object.SearchMatchType]matcher matchers map[object.SearchMatchType]matcher
boltDB *bbolt.DB boltDB *bbolt.DB

View file

@ -0,0 +1,44 @@
package meta
import (
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
)
// SetMode sets the metabase mode of operation.
// If the mode assumes no operation metabase, the database is closed.
func (db *DB) SetMode(m mode.Mode) error {
db.modeMtx.Lock()
defer db.modeMtx.Unlock()
if db.mode == m {
return nil
}
if !db.mode.NoMetabase() {
if err := db.Close(); err != nil {
return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err)
}
}
var err error
switch m {
case mode.Degraded:
db.boltDB = nil
case mode.ReadOnly:
err = db.Open(true)
case mode.ReadWrite:
err = db.Open(false)
}
if err == nil && !m.NoMetabase() && !m.ReadOnly() {
err = db.Init()
}
if err != nil {
return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err)
}
db.mode = m
return nil
}

View file

@ -7,9 +7,11 @@ import (
"math/rand" "math/rand"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"time" "time"
"github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/util" "github.com/nspcc-dev/neofs-node/pkg/util"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
@ -17,6 +19,9 @@ import (
type boltForest struct { type boltForest struct {
db *bbolt.DB db *bbolt.DB
modeMtx sync.Mutex
mode mode.Mode
cfg cfg
} }
@ -58,6 +63,30 @@ func NewBoltForest(opts ...Option) ForestStorage {
return &b return &b
} }
func (t *boltForest) SetMode(m mode.Mode) error {
t.modeMtx.Lock()
defer t.modeMtx.Unlock()
if t.mode == m {
return nil
}
if t.mode.ReadOnly() == m.ReadOnly() {
return nil
}
err := t.Close()
if err == nil {
if err = t.Open(m.ReadOnly()); err == nil {
err = t.Init()
}
}
if err != nil {
return fmt.Errorf("can't set pilorama mode (old=%s, new=%s): %w", t.mode, m, err)
}
t.mode = m
return nil
}
func (t *boltForest) Open(readOnly bool) error { func (t *boltForest) Open(readOnly bool) error {
err := util.MkdirAllX(filepath.Dir(t.path), t.perm) err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
if err != nil { if err != nil {

View file

@ -3,6 +3,7 @@ package pilorama
import ( import (
"sort" "sort"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
) )
@ -111,7 +112,9 @@ func (f *memoryForest) Init() error {
func (f *memoryForest) Open(bool) error { func (f *memoryForest) Open(bool) error {
return nil return nil
} }
func (f *memoryForest) SetMode(mode.Mode) error {
return nil
}
func (f *memoryForest) Close() error { func (f *memoryForest) Close() error {
return nil return nil
} }

View file

@ -3,6 +3,7 @@ package pilorama
import ( import (
"errors" "errors"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
) )
@ -41,6 +42,7 @@ type ForestStorage interface {
Init() error Init() error
Open(bool) error Open(bool) error
Close() error Close() error
SetMode(m mode.Mode) error
Forest Forest
} }

View file

@ -88,7 +88,8 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) {
return res.Object(), nil return res.Object(), nil
} }
obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small) skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
return GetRes{ return GetRes{
obj: obj, obj: obj,

View file

@ -21,8 +21,40 @@ func (s *Shard) SetMode(m mode.Mode) error {
s.m.Lock() s.m.Lock()
defer s.m.Unlock() defer s.m.Unlock()
if s.info.Mode == m {
return nil
}
components := []interface{ SetMode(mode.Mode) error }{
s.metaBase, s.blobStor,
}
if s.hasWriteCache() { if s.hasWriteCache() {
s.writeCache.SetMode(m) components = append(components, s.writeCache)
}
if s.pilorama != nil {
components = append(components, s.pilorama)
}
// The usual flow of the requests (pilorama is independent):
// writecache -> blobstor -> metabase
// For mode.ReadOnly and mode.Degraded the order is:
// writecache -> blobstor -> metabase
// For mode.ReadWrite it is the opposite:
// metabase -> blobstor -> writecache
if m != mode.ReadWrite {
if s.hasWriteCache() {
components[0], components[2] = components[2], components[0]
} else {
components[0], components[1] = components[1], components[0]
}
}
for i := range components {
if err := components[i].SetMode(m); err != nil {
return err
}
} }
s.info.Mode = m s.info.Mode = m

View file

@ -2,6 +2,7 @@ package writecache
import ( import (
"errors" "errors"
"fmt"
"time" "time"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
@ -13,22 +14,28 @@ var ErrReadOnly = errors.New("write-cache is in read-only mode")
// SetMode sets write-cache mode of operation. // SetMode sets write-cache mode of operation.
// When shard is put in read-only mode all objects in memory are flushed to disk // When shard is put in read-only mode all objects in memory are flushed to disk
// and all background jobs are suspended. // and all background jobs are suspended.
func (c *cache) SetMode(m mode.Mode) { func (c *cache) SetMode(m mode.Mode) error {
c.modeMtx.Lock() c.modeMtx.Lock()
defer c.modeMtx.Unlock() defer c.modeMtx.Unlock()
if c.mode == m {
return
}
if m.ReadOnly() == c.readOnly() {
c.mode = m c.mode = m
if m == mode.ReadWrite { return nil
return
} }
if !c.readOnly() {
// Because modeMtx is taken no new objects will arrive an all other modifying // Because modeMtx is taken no new objects will arrive an all other modifying
// operations are completed. // operations are completed.
// 1. Persist objects already in memory on disk. // 1. Persist objects already in memory on disk.
c.persistMemoryCache() c.persistMemoryCache()
}
if c.db != nil {
if err := c.db.Close(); err != nil {
return fmt.Errorf("can't close write-cache database: %w", err)
}
c.db = nil
}
// 2. Suspend producers to ensure there are channel send operations in fly. // 2. Suspend producers to ensure there are channel send operations in fly.
// metaCh and directCh can be populated either during Put or in background memory persist thread. // metaCh and directCh can be populated either during Put or in background memory persist thread.
@ -40,10 +47,17 @@ func (c *cache) SetMode(m mode.Mode) {
c.log.Info("waiting for channels to flush") c.log.Info("waiting for channels to flush")
time.Sleep(time.Second) time.Sleep(time.Second)
} }
if err := c.openStore(m.ReadOnly()); err != nil {
return err
}
c.mode = m
return nil
} }
// readOnly returns true if current mode is read-only. // readOnly returns true if current mode is read-only.
// `c.modeMtx` must be taken. // `c.modeMtx` must be taken.
func (c *cache) readOnly() bool { func (c *cache) readOnly() bool {
return c.mode != mode.ReadWrite return c.mode.ReadOnly()
} }

View file

@ -41,6 +41,7 @@ func (c *cache) openStore(readOnly bool) error {
c.db.MaxBatchSize = c.maxBatchSize c.db.MaxBatchSize = c.maxBatchSize
c.db.MaxBatchDelay = c.maxBatchDelay c.db.MaxBatchDelay = c.maxBatchDelay
if !readOnly {
err = c.db.Update(func(tx *bbolt.Tx) error { err = c.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(defaultBucket) _, err := tx.CreateBucketIfNotExists(defaultBucket)
return err return err
@ -48,6 +49,7 @@ func (c *cache) openStore(readOnly bool) error {
if err != nil { if err != nil {
return fmt.Errorf("could not create default bucket: %w", err) return fmt.Errorf("could not create default bucket: %w", err)
} }
}
c.fsTree = &fstree.FSTree{ c.fsTree = &fstree.FSTree{
Info: fstree.Info{ Info: fstree.Info{

View file

@ -24,7 +24,7 @@ type Cache interface {
Delete(oid.Address) error Delete(oid.Address) error
Iterate(IterationPrm) error Iterate(IterationPrm) error
Put(*object.Object) error Put(*object.Object) error
SetMode(mode.Mode) SetMode(mode.Mode) error
SetLogger(*zap.Logger) SetLogger(*zap.Logger)
DumpInfo() Info DumpInfo() Info
@ -153,9 +153,16 @@ func (c *cache) Init() error {
// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op.
func (c *cache) Close() error { func (c *cache) Close() error {
// Finish all in-progress operations. // Finish all in-progress operations.
c.SetMode(mode.ReadOnly) if err := c.SetMode(mode.ReadOnly); err != nil {
return err
}
close(c.closeCh) close(c.closeCh)
if c.objCounters != nil {
c.objCounters.FlushAndClose() c.objCounters.FlushAndClose()
}
if c.db != nil {
return c.db.Close() return c.db.Close()
}
return nil
} }