From 3df62769c0542c00c65b18edf0c8423126126559 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 5 Jul 2022 07:55:46 +0300 Subject: [PATCH] [#1559] local_object_storage: Allow to set mode for all components Signed-off-by: Evgenii Stratonikov --- .../blobstor/blobovnicza.go | 5 +++ pkg/local_object_storage/blobstor/blobstor.go | 5 +++ pkg/local_object_storage/blobstor/mode.go | 35 +++++++++++++++ pkg/local_object_storage/engine/error_test.go | 2 +- pkg/local_object_storage/engine/get.go | 20 +++++++-- pkg/local_object_storage/metabase/control.go | 4 ++ pkg/local_object_storage/metabase/db.go | 5 +++ pkg/local_object_storage/metabase/mode.go | 44 +++++++++++++++++++ pkg/local_object_storage/pilorama/boltdb.go | 29 ++++++++++++ pkg/local_object_storage/pilorama/forest.go | 5 ++- .../pilorama/interface.go | 2 + pkg/local_object_storage/shard/get.go | 3 +- pkg/local_object_storage/shard/mode.go | 34 +++++++++++++- pkg/local_object_storage/writecache/mode.go | 36 ++++++++++----- .../writecache/storage.go | 14 +++--- .../writecache/writecache.go | 15 +++++-- 16 files changed, 230 insertions(+), 28 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/mode.go create mode 100644 pkg/local_object_storage/metabase/mode.go diff --git a/pkg/local_object_storage/blobstor/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovnicza.go index f132306bc..2afde24e3 100644 --- a/pkg/local_object_storage/blobstor/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovnicza.go @@ -835,6 +835,11 @@ func (b *blobovniczas) init() error { return zstdD(data) } + if b.readOnly { + b.log.Debug("read-only mode, skip blobovniczas initialization...") + return nil + } + return b.iterateBlobovniczas(false, func(p string, blz *blobovnicza.Blobovnicza) error { if err := blz.Init(); err != nil { return fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err) diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 64c68d764..d048f31e2 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -4,9 +4,11 @@ import ( "encoding/hex" "io/fs" "path/filepath" + "sync" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -16,6 +18,9 @@ type BlobStor struct { *cfg blobovniczas *blobovniczas + + modeMtx sync.RWMutex + mode mode.Mode } type Info = fstree.Info diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go new file mode 100644 index 000000000..fe5d9ee4d --- /dev/null +++ b/pkg/local_object_storage/blobstor/mode.go @@ -0,0 +1,35 @@ +package blobstor + +import ( + "fmt" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" +) + +// SetMode sets the blobstor mode of operation. +func (b *BlobStor) SetMode(m mode.Mode) error { + b.modeMtx.Lock() + defer b.modeMtx.Unlock() + + if b.mode == m { + return nil + } + + if b.mode.ReadOnly() == m.ReadOnly() { + return nil + } + + err := b.Close() + if err == nil { + if err = b.Open(m.ReadOnly()); err == nil { + err = b.Init() + } + } + if err != nil { + return fmt.Errorf("can't set blobstor mode (old=%s, new=%s): %w", b.mode, m, err) + } + + b.blobovniczas.readOnly = m.ReadOnly() + b.mode = m + return nil +} diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 4a22ae25b..9f049132d 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, e, id[0], 2, mode.Degraded) + checkShardState(t, e, id[0], 1, mode.Degraded) checkShardState(t, e, id[1], 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 400eb59dc..b11e72cdf 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -74,7 +74,14 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { var shPrm shard.GetPrm shPrm.SetAddress(prm.addr) + var hasDegraded bool + e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + noMeta := sh.GetMode().NoMetabase() + shPrm.SetIgnoreMeta(noMeta) + + hasDegraded = hasDegraded || noMeta + res, err := sh.Get(shPrm) if err != nil { if res.HasMeta() { @@ -122,7 +129,7 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { } if obj == nil { - if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) { + if !hasDegraded && shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) { return GetRes{}, outError } @@ -132,6 +139,11 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { shPrm.SetIgnoreMeta(true) e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + if sh.GetMode().NoMetabase() { + // Already visited. + return false + } + res, err := sh.Get(shPrm) obj = res.Object() return err == nil @@ -139,8 +151,10 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { if obj == nil { return GetRes{}, outError } - e.reportShardError(shardWithMeta, "meta info was present, but object is missing", - metaError, zap.Stringer("address", prm.addr)) + if shardWithMeta.Shard != nil { + e.reportShardError(shardWithMeta, "meta info was present, but object is missing", + metaError, zap.Stringer("address", prm.addr)) + } } return GetRes{ diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 044936fe6..647f39321 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -70,6 +70,10 @@ func (db *DB) Reset() error { } func (db *DB) init(reset bool) error { + if db.mode.NoMetabase() || db.mode.ReadOnly() { + return nil + } + mStaticBuckets := map[string]struct{}{ string(containerVolumeBucketName): {}, string(graveyardBucketName): {}, diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 4e3ce1cf3..20c874da7 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -8,9 +8,11 @@ import ( "os" "strconv" "strings" + "sync" "time" v2object "github.com/nspcc-dev/neofs-api-go/v2/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/util/logger" "github.com/nspcc-dev/neofs-sdk-go/object" "go.etcd.io/bbolt" @@ -26,6 +28,9 @@ type matcher struct { type DB struct { *cfg + modeMtx sync.RWMutex + mode mode.Mode + matchers map[object.SearchMatchType]matcher boltDB *bbolt.DB diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go new file mode 100644 index 000000000..9267bff3f --- /dev/null +++ b/pkg/local_object_storage/metabase/mode.go @@ -0,0 +1,44 @@ +package meta + +import ( + "fmt" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" +) + +// SetMode sets the metabase mode of operation. +// If the mode assumes no operation metabase, the database is closed. +func (db *DB) SetMode(m mode.Mode) error { + db.modeMtx.Lock() + defer db.modeMtx.Unlock() + + if db.mode == m { + return nil + } + + if !db.mode.NoMetabase() { + if err := db.Close(); err != nil { + return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + } + } + + var err error + switch m { + case mode.Degraded: + db.boltDB = nil + case mode.ReadOnly: + err = db.Open(true) + case mode.ReadWrite: + err = db.Open(false) + } + if err == nil && !m.NoMetabase() && !m.ReadOnly() { + err = db.Init() + } + + if err != nil { + return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + } + + db.mode = m + return nil +} diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 2f3a3677d..5ddd5c66c 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -7,9 +7,11 @@ import ( "math/rand" "os" "path/filepath" + "sync" "time" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/util" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" "go.etcd.io/bbolt" @@ -17,6 +19,9 @@ import ( type boltForest struct { db *bbolt.DB + + modeMtx sync.Mutex + mode mode.Mode cfg } @@ -58,6 +63,30 @@ func NewBoltForest(opts ...Option) ForestStorage { return &b } +func (t *boltForest) SetMode(m mode.Mode) error { + t.modeMtx.Lock() + defer t.modeMtx.Unlock() + + if t.mode == m { + return nil + } + if t.mode.ReadOnly() == m.ReadOnly() { + return nil + } + + err := t.Close() + if err == nil { + if err = t.Open(m.ReadOnly()); err == nil { + err = t.Init() + } + } + if err != nil { + return fmt.Errorf("can't set pilorama mode (old=%s, new=%s): %w", t.mode, m, err) + } + + t.mode = m + return nil +} func (t *boltForest) Open(readOnly bool) error { err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 2d781cffc..87e57fcc2 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -3,6 +3,7 @@ package pilorama import ( "sort" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" ) @@ -111,7 +112,9 @@ func (f *memoryForest) Init() error { func (f *memoryForest) Open(bool) error { return nil } - +func (f *memoryForest) SetMode(mode.Mode) error { + return nil +} func (f *memoryForest) Close() error { return nil } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index b7e98e328..b96c36dfa 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -3,6 +3,7 @@ package pilorama import ( "errors" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" ) @@ -41,6 +42,7 @@ type ForestStorage interface { Init() error Open(bool) error Close() error + SetMode(m mode.Mode) error Forest } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 73f7934fb..16bd54c02 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -88,7 +88,8 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) { return res.Object(), nil } - obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small) + skipMeta := prm.skipMeta || s.GetMode().NoMetabase() + obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small) return GetRes{ obj: obj, diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index d3a1f34f0..0d38a6c5c 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -21,8 +21,40 @@ func (s *Shard) SetMode(m mode.Mode) error { s.m.Lock() defer s.m.Unlock() + if s.info.Mode == m { + return nil + } + + components := []interface{ SetMode(mode.Mode) error }{ + s.metaBase, s.blobStor, + } + if s.hasWriteCache() { - s.writeCache.SetMode(m) + components = append(components, s.writeCache) + } + + if s.pilorama != nil { + components = append(components, s.pilorama) + } + + // The usual flow of the requests (pilorama is independent): + // writecache -> blobstor -> metabase + // For mode.ReadOnly and mode.Degraded the order is: + // writecache -> blobstor -> metabase + // For mode.ReadWrite it is the opposite: + // metabase -> blobstor -> writecache + if m != mode.ReadWrite { + if s.hasWriteCache() { + components[0], components[2] = components[2], components[0] + } else { + components[0], components[1] = components[1], components[0] + } + } + + for i := range components { + if err := components[i].SetMode(m); err != nil { + return err + } } s.info.Mode = m diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 4719be698..bf6685a45 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -2,6 +2,7 @@ package writecache import ( "errors" + "fmt" "time" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" @@ -13,22 +14,28 @@ var ErrReadOnly = errors.New("write-cache is in read-only mode") // SetMode sets write-cache mode of operation. // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. -func (c *cache) SetMode(m mode.Mode) { +func (c *cache) SetMode(m mode.Mode) error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - if c.mode == m { - return + + if m.ReadOnly() == c.readOnly() { + c.mode = m + return nil } - c.mode = m - if m == mode.ReadWrite { - return + if !c.readOnly() { + // Because modeMtx is taken no new objects will arrive an all other modifying + // operations are completed. + // 1. Persist objects already in memory on disk. + c.persistMemoryCache() } - // Because modeMtx is taken no new objects will arrive an all other modifying - // operations are completed. - // 1. Persist objects already in memory on disk. - c.persistMemoryCache() + if c.db != nil { + if err := c.db.Close(); err != nil { + return fmt.Errorf("can't close write-cache database: %w", err) + } + c.db = nil + } // 2. Suspend producers to ensure there are channel send operations in fly. // metaCh and directCh can be populated either during Put or in background memory persist thread. @@ -40,10 +47,17 @@ func (c *cache) SetMode(m mode.Mode) { c.log.Info("waiting for channels to flush") time.Sleep(time.Second) } + + if err := c.openStore(m.ReadOnly()); err != nil { + return err + } + + c.mode = m + return nil } // readOnly returns true if current mode is read-only. // `c.modeMtx` must be taken. func (c *cache) readOnly() bool { - return c.mode != mode.ReadWrite + return c.mode.ReadOnly() } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 6279ded3c..76e8fbe68 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -41,12 +41,14 @@ func (c *cache) openStore(readOnly bool) error { c.db.MaxBatchSize = c.maxBatchSize c.db.MaxBatchDelay = c.maxBatchDelay - err = c.db.Update(func(tx *bbolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(defaultBucket) - return err - }) - if err != nil { - return fmt.Errorf("could not create default bucket: %w", err) + if !readOnly { + err = c.db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(defaultBucket) + return err + }) + if err != nil { + return fmt.Errorf("could not create default bucket: %w", err) + } } c.fsTree = &fstree.FSTree{ diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index a4b7248b7..40902be0f 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -24,7 +24,7 @@ type Cache interface { Delete(oid.Address) error Iterate(IterationPrm) error Put(*object.Object) error - SetMode(mode.Mode) + SetMode(mode.Mode) error SetLogger(*zap.Logger) DumpInfo() Info @@ -153,9 +153,16 @@ func (c *cache) Init() error { // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. func (c *cache) Close() error { // Finish all in-progress operations. - c.SetMode(mode.ReadOnly) + if err := c.SetMode(mode.ReadOnly); err != nil { + return err + } close(c.closeCh) - c.objCounters.FlushAndClose() - return c.db.Close() + if c.objCounters != nil { + c.objCounters.FlushAndClose() + } + if c.db != nil { + return c.db.Close() + } + return nil }