[#1559] local_object_storage: Allow to set mode for all components

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-05 07:55:46 +03:00 committed by fyrchik
parent 1e786233bf
commit 3df62769c0
16 changed files with 230 additions and 28 deletions

View file

@ -835,6 +835,11 @@ func (b *blobovniczas) init() error {
return zstdD(data)
}
if b.readOnly {
b.log.Debug("read-only mode, skip blobovniczas initialization...")
return nil
}
return b.iterateBlobovniczas(false, func(p string, blz *blobovnicza.Blobovnicza) error {
if err := blz.Init(); err != nil {
return fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err)

View file

@ -4,9 +4,11 @@ import (
"encoding/hex"
"io/fs"
"path/filepath"
"sync"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"go.uber.org/zap"
)
@ -16,6 +18,9 @@ type BlobStor struct {
*cfg
blobovniczas *blobovniczas
modeMtx sync.RWMutex
mode mode.Mode
}
type Info = fstree.Info

View file

@ -0,0 +1,35 @@
package blobstor
import (
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
)
// SetMode sets the blobstor mode of operation.
func (b *BlobStor) SetMode(m mode.Mode) error {
b.modeMtx.Lock()
defer b.modeMtx.Unlock()
if b.mode == m {
return nil
}
if b.mode.ReadOnly() == m.ReadOnly() {
return nil
}
err := b.Close()
if err == nil {
if err = b.Open(m.ReadOnly()); err == nil {
err = b.Init()
}
}
if err != nil {
return fmt.Errorf("can't set blobstor mode (old=%s, new=%s): %w", b.mode, m, err)
}
b.blobovniczas.readOnly = m.ReadOnly()
b.mode = m
return nil
}

View file

@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
}
checkShardState(t, e, id[0], 2, mode.Degraded)
checkShardState(t, e, id[0], 1, mode.Degraded)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
}

View file

@ -74,7 +74,14 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
var shPrm shard.GetPrm
shPrm.SetAddress(prm.addr)
var hasDegraded bool
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
noMeta := sh.GetMode().NoMetabase()
shPrm.SetIgnoreMeta(noMeta)
hasDegraded = hasDegraded || noMeta
res, err := sh.Get(shPrm)
if err != nil {
if res.HasMeta() {
@ -122,7 +129,7 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
}
if obj == nil {
if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
if !hasDegraded && shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
return GetRes{}, outError
}
@ -132,6 +139,11 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
shPrm.SetIgnoreMeta(true)
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
if sh.GetMode().NoMetabase() {
// Already visited.
return false
}
res, err := sh.Get(shPrm)
obj = res.Object()
return err == nil
@ -139,8 +151,10 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
if obj == nil {
return GetRes{}, outError
}
e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
metaError, zap.Stringer("address", prm.addr))
if shardWithMeta.Shard != nil {
e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
metaError, zap.Stringer("address", prm.addr))
}
}
return GetRes{

View file

@ -70,6 +70,10 @@ func (db *DB) Reset() error {
}
func (db *DB) init(reset bool) error {
if db.mode.NoMetabase() || db.mode.ReadOnly() {
return nil
}
mStaticBuckets := map[string]struct{}{
string(containerVolumeBucketName): {},
string(graveyardBucketName): {},

View file

@ -8,9 +8,11 @@ import (
"os"
"strconv"
"strings"
"sync"
"time"
v2object "github.com/nspcc-dev/neofs-api-go/v2/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"github.com/nspcc-dev/neofs-sdk-go/object"
"go.etcd.io/bbolt"
@ -26,6 +28,9 @@ type matcher struct {
type DB struct {
*cfg
modeMtx sync.RWMutex
mode mode.Mode
matchers map[object.SearchMatchType]matcher
boltDB *bbolt.DB

View file

@ -0,0 +1,44 @@
package meta
import (
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
)
// SetMode sets the metabase mode of operation.
// If the mode assumes no operation metabase, the database is closed.
func (db *DB) SetMode(m mode.Mode) error {
db.modeMtx.Lock()
defer db.modeMtx.Unlock()
if db.mode == m {
return nil
}
if !db.mode.NoMetabase() {
if err := db.Close(); err != nil {
return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err)
}
}
var err error
switch m {
case mode.Degraded:
db.boltDB = nil
case mode.ReadOnly:
err = db.Open(true)
case mode.ReadWrite:
err = db.Open(false)
}
if err == nil && !m.NoMetabase() && !m.ReadOnly() {
err = db.Init()
}
if err != nil {
return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err)
}
db.mode = m
return nil
}

View file

@ -7,9 +7,11 @@ import (
"math/rand"
"os"
"path/filepath"
"sync"
"time"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/util"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.etcd.io/bbolt"
@ -17,6 +19,9 @@ import (
type boltForest struct {
db *bbolt.DB
modeMtx sync.Mutex
mode mode.Mode
cfg
}
@ -58,6 +63,30 @@ func NewBoltForest(opts ...Option) ForestStorage {
return &b
}
func (t *boltForest) SetMode(m mode.Mode) error {
t.modeMtx.Lock()
defer t.modeMtx.Unlock()
if t.mode == m {
return nil
}
if t.mode.ReadOnly() == m.ReadOnly() {
return nil
}
err := t.Close()
if err == nil {
if err = t.Open(m.ReadOnly()); err == nil {
err = t.Init()
}
}
if err != nil {
return fmt.Errorf("can't set pilorama mode (old=%s, new=%s): %w", t.mode, m, err)
}
t.mode = m
return nil
}
func (t *boltForest) Open(readOnly bool) error {
err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
if err != nil {

View file

@ -3,6 +3,7 @@ package pilorama
import (
"sort"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
)
@ -111,7 +112,9 @@ func (f *memoryForest) Init() error {
func (f *memoryForest) Open(bool) error {
return nil
}
func (f *memoryForest) SetMode(mode.Mode) error {
return nil
}
func (f *memoryForest) Close() error {
return nil
}

View file

@ -3,6 +3,7 @@ package pilorama
import (
"errors"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
)
@ -41,6 +42,7 @@ type ForestStorage interface {
Init() error
Open(bool) error
Close() error
SetMode(m mode.Mode) error
Forest
}

View file

@ -88,7 +88,8 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) {
return res.Object(), nil
}
obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small)
skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
return GetRes{
obj: obj,

View file

@ -21,8 +21,40 @@ func (s *Shard) SetMode(m mode.Mode) error {
s.m.Lock()
defer s.m.Unlock()
if s.info.Mode == m {
return nil
}
components := []interface{ SetMode(mode.Mode) error }{
s.metaBase, s.blobStor,
}
if s.hasWriteCache() {
s.writeCache.SetMode(m)
components = append(components, s.writeCache)
}
if s.pilorama != nil {
components = append(components, s.pilorama)
}
// The usual flow of the requests (pilorama is independent):
// writecache -> blobstor -> metabase
// For mode.ReadOnly and mode.Degraded the order is:
// writecache -> blobstor -> metabase
// For mode.ReadWrite it is the opposite:
// metabase -> blobstor -> writecache
if m != mode.ReadWrite {
if s.hasWriteCache() {
components[0], components[2] = components[2], components[0]
} else {
components[0], components[1] = components[1], components[0]
}
}
for i := range components {
if err := components[i].SetMode(m); err != nil {
return err
}
}
s.info.Mode = m

View file

@ -2,6 +2,7 @@ package writecache
import (
"errors"
"fmt"
"time"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
@ -13,22 +14,28 @@ var ErrReadOnly = errors.New("write-cache is in read-only mode")
// SetMode sets write-cache mode of operation.
// When shard is put in read-only mode all objects in memory are flushed to disk
// and all background jobs are suspended.
func (c *cache) SetMode(m mode.Mode) {
func (c *cache) SetMode(m mode.Mode) error {
c.modeMtx.Lock()
defer c.modeMtx.Unlock()
if c.mode == m {
return
if m.ReadOnly() == c.readOnly() {
c.mode = m
return nil
}
c.mode = m
if m == mode.ReadWrite {
return
if !c.readOnly() {
// Because modeMtx is taken no new objects will arrive an all other modifying
// operations are completed.
// 1. Persist objects already in memory on disk.
c.persistMemoryCache()
}
// Because modeMtx is taken no new objects will arrive an all other modifying
// operations are completed.
// 1. Persist objects already in memory on disk.
c.persistMemoryCache()
if c.db != nil {
if err := c.db.Close(); err != nil {
return fmt.Errorf("can't close write-cache database: %w", err)
}
c.db = nil
}
// 2. Suspend producers to ensure there are channel send operations in fly.
// metaCh and directCh can be populated either during Put or in background memory persist thread.
@ -40,10 +47,17 @@ func (c *cache) SetMode(m mode.Mode) {
c.log.Info("waiting for channels to flush")
time.Sleep(time.Second)
}
if err := c.openStore(m.ReadOnly()); err != nil {
return err
}
c.mode = m
return nil
}
// readOnly returns true if current mode is read-only.
// `c.modeMtx` must be taken.
func (c *cache) readOnly() bool {
return c.mode != mode.ReadWrite
return c.mode.ReadOnly()
}

View file

@ -41,12 +41,14 @@ func (c *cache) openStore(readOnly bool) error {
c.db.MaxBatchSize = c.maxBatchSize
c.db.MaxBatchDelay = c.maxBatchDelay
err = c.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(defaultBucket)
return err
})
if err != nil {
return fmt.Errorf("could not create default bucket: %w", err)
if !readOnly {
err = c.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(defaultBucket)
return err
})
if err != nil {
return fmt.Errorf("could not create default bucket: %w", err)
}
}
c.fsTree = &fstree.FSTree{

View file

@ -24,7 +24,7 @@ type Cache interface {
Delete(oid.Address) error
Iterate(IterationPrm) error
Put(*object.Object) error
SetMode(mode.Mode)
SetMode(mode.Mode) error
SetLogger(*zap.Logger)
DumpInfo() Info
@ -153,9 +153,16 @@ func (c *cache) Init() error {
// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op.
func (c *cache) Close() error {
// Finish all in-progress operations.
c.SetMode(mode.ReadOnly)
if err := c.SetMode(mode.ReadOnly); err != nil {
return err
}
close(c.closeCh)
c.objCounters.FlushAndClose()
return c.db.Close()
if c.objCounters != nil {
c.objCounters.FlushAndClose()
}
if c.db != nil {
return c.db.Close()
}
return nil
}