From 3df62769c0542c00c65b18edf0c8423126126559 Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <evgeniy@nspcc.ru>
Date: Tue, 5 Jul 2022 07:55:46 +0300
Subject: [PATCH] [#1559] local_object_storage: Allow to set mode for all
 components

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
---
 .../blobstor/blobovnicza.go                   |  5 +++
 pkg/local_object_storage/blobstor/blobstor.go |  5 +++
 pkg/local_object_storage/blobstor/mode.go     | 35 +++++++++++++++
 pkg/local_object_storage/engine/error_test.go |  2 +-
 pkg/local_object_storage/engine/get.go        | 20 +++++++--
 pkg/local_object_storage/metabase/control.go  |  4 ++
 pkg/local_object_storage/metabase/db.go       |  5 +++
 pkg/local_object_storage/metabase/mode.go     | 44 +++++++++++++++++++
 pkg/local_object_storage/pilorama/boltdb.go   | 29 ++++++++++++
 pkg/local_object_storage/pilorama/forest.go   |  5 ++-
 .../pilorama/interface.go                     |  2 +
 pkg/local_object_storage/shard/get.go         |  3 +-
 pkg/local_object_storage/shard/mode.go        | 34 +++++++++++++-
 pkg/local_object_storage/writecache/mode.go   | 36 ++++++++++-----
 .../writecache/storage.go                     | 14 +++---
 .../writecache/writecache.go                  | 15 +++++--
 16 files changed, 230 insertions(+), 28 deletions(-)
 create mode 100644 pkg/local_object_storage/blobstor/mode.go
 create mode 100644 pkg/local_object_storage/metabase/mode.go

diff --git a/pkg/local_object_storage/blobstor/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovnicza.go
index f132306bc..2afde24e3 100644
--- a/pkg/local_object_storage/blobstor/blobovnicza.go
+++ b/pkg/local_object_storage/blobstor/blobovnicza.go
@@ -835,6 +835,11 @@ func (b *blobovniczas) init() error {
 		return zstdD(data)
 	}
 
+	if b.readOnly {
+		b.log.Debug("read-only mode, skip blobovniczas initialization...")
+		return nil
+	}
+
 	return b.iterateBlobovniczas(false, func(p string, blz *blobovnicza.Blobovnicza) error {
 		if err := blz.Init(); err != nil {
 			return fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err)
diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go
index 64c68d764..d048f31e2 100644
--- a/pkg/local_object_storage/blobstor/blobstor.go
+++ b/pkg/local_object_storage/blobstor/blobstor.go
@@ -4,9 +4,11 @@ import (
 	"encoding/hex"
 	"io/fs"
 	"path/filepath"
+	"sync"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/util/logger"
 	"go.uber.org/zap"
 )
@@ -16,6 +18,9 @@ type BlobStor struct {
 	*cfg
 
 	blobovniczas *blobovniczas
+
+	modeMtx sync.RWMutex
+	mode    mode.Mode
 }
 
 type Info = fstree.Info
diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go
new file mode 100644
index 000000000..fe5d9ee4d
--- /dev/null
+++ b/pkg/local_object_storage/blobstor/mode.go
@@ -0,0 +1,35 @@
+package blobstor
+
+import (
+	"fmt"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
+)
+
+// SetMode sets the blobstor mode of operation.
+func (b *BlobStor) SetMode(m mode.Mode) error {
+	b.modeMtx.Lock()
+	defer b.modeMtx.Unlock()
+
+	if b.mode == m {
+		return nil
+	}
+
+	if b.mode.ReadOnly() == m.ReadOnly() {
+		return nil
+	}
+
+	err := b.Close()
+	if err == nil {
+		if err = b.Open(m.ReadOnly()); err == nil {
+			err = b.Init()
+		}
+	}
+	if err != nil {
+		return fmt.Errorf("can't set blobstor mode (old=%s, new=%s): %w", b.mode, m, err)
+	}
+
+	b.blobovniczas.readOnly = m.ReadOnly()
+	b.mode = m
+	return nil
+}
diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go
index 4a22ae25b..9f049132d 100644
--- a/pkg/local_object_storage/engine/error_test.go
+++ b/pkg/local_object_storage/engine/error_test.go
@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 2, mode.Degraded)
+	checkShardState(t, e, id[0], 1, mode.Degraded)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 
diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go
index 400eb59dc..b11e72cdf 100644
--- a/pkg/local_object_storage/engine/get.go
+++ b/pkg/local_object_storage/engine/get.go
@@ -74,7 +74,14 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
 	var shPrm shard.GetPrm
 	shPrm.SetAddress(prm.addr)
 
+	var hasDegraded bool
+
 	e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
+		noMeta := sh.GetMode().NoMetabase()
+		shPrm.SetIgnoreMeta(noMeta)
+
+		hasDegraded = hasDegraded || noMeta
+
 		res, err := sh.Get(shPrm)
 		if err != nil {
 			if res.HasMeta() {
@@ -122,7 +129,7 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
 	}
 
 	if obj == nil {
-		if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
+		if !hasDegraded && shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
 			return GetRes{}, outError
 		}
 
@@ -132,6 +139,11 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
 		shPrm.SetIgnoreMeta(true)
 
 		e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
+			if sh.GetMode().NoMetabase() {
+				// Already visited.
+				return false
+			}
+
 			res, err := sh.Get(shPrm)
 			obj = res.Object()
 			return err == nil
@@ -139,8 +151,10 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) {
 		if obj == nil {
 			return GetRes{}, outError
 		}
-		e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
-			metaError, zap.Stringer("address", prm.addr))
+		if shardWithMeta.Shard != nil {
+			e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
+				metaError, zap.Stringer("address", prm.addr))
+		}
 	}
 
 	return GetRes{
diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go
index 044936fe6..647f39321 100644
--- a/pkg/local_object_storage/metabase/control.go
+++ b/pkg/local_object_storage/metabase/control.go
@@ -70,6 +70,10 @@ func (db *DB) Reset() error {
 }
 
 func (db *DB) init(reset bool) error {
+	if db.mode.NoMetabase() || db.mode.ReadOnly() {
+		return nil
+	}
+
 	mStaticBuckets := map[string]struct{}{
 		string(containerVolumeBucketName): {},
 		string(graveyardBucketName):       {},
diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go
index 4e3ce1cf3..20c874da7 100644
--- a/pkg/local_object_storage/metabase/db.go
+++ b/pkg/local_object_storage/metabase/db.go
@@ -8,9 +8,11 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	v2object "github.com/nspcc-dev/neofs-api-go/v2/object"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/util/logger"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	"go.etcd.io/bbolt"
@@ -26,6 +28,9 @@ type matcher struct {
 type DB struct {
 	*cfg
 
+	modeMtx sync.RWMutex
+	mode    mode.Mode
+
 	matchers map[object.SearchMatchType]matcher
 
 	boltDB *bbolt.DB
diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go
new file mode 100644
index 000000000..9267bff3f
--- /dev/null
+++ b/pkg/local_object_storage/metabase/mode.go
@@ -0,0 +1,44 @@
+package meta
+
+import (
+	"fmt"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
+)
+
+// SetMode sets the metabase mode of operation.
+// If the mode assumes no operation metabase, the database is closed.
+func (db *DB) SetMode(m mode.Mode) error {
+	db.modeMtx.Lock()
+	defer db.modeMtx.Unlock()
+
+	if db.mode == m {
+		return nil
+	}
+
+	if !db.mode.NoMetabase() {
+		if err := db.Close(); err != nil {
+			return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err)
+		}
+	}
+
+	var err error
+	switch m {
+	case mode.Degraded:
+		db.boltDB = nil
+	case mode.ReadOnly:
+		err = db.Open(true)
+	case mode.ReadWrite:
+		err = db.Open(false)
+	}
+	if err == nil && !m.NoMetabase() && !m.ReadOnly() {
+		err = db.Init()
+	}
+
+	if err != nil {
+		return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err)
+	}
+
+	db.mode = m
+	return nil
+}
diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go
index 2f3a3677d..5ddd5c66c 100644
--- a/pkg/local_object_storage/pilorama/boltdb.go
+++ b/pkg/local_object_storage/pilorama/boltdb.go
@@ -7,9 +7,11 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/nspcc-dev/neo-go/pkg/io"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/util"
 	cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	"go.etcd.io/bbolt"
@@ -17,6 +19,9 @@ import (
 
 type boltForest struct {
 	db *bbolt.DB
+
+	modeMtx sync.Mutex
+	mode    mode.Mode
 	cfg
 }
 
@@ -58,6 +63,30 @@ func NewBoltForest(opts ...Option) ForestStorage {
 	return &b
 }
 
+func (t *boltForest) SetMode(m mode.Mode) error {
+	t.modeMtx.Lock()
+	defer t.modeMtx.Unlock()
+
+	if t.mode == m {
+		return nil
+	}
+	if t.mode.ReadOnly() == m.ReadOnly() {
+		return nil
+	}
+
+	err := t.Close()
+	if err == nil {
+		if err = t.Open(m.ReadOnly()); err == nil {
+			err = t.Init()
+		}
+	}
+	if err != nil {
+		return fmt.Errorf("can't set pilorama mode (old=%s, new=%s): %w", t.mode, m, err)
+	}
+
+	t.mode = m
+	return nil
+}
 func (t *boltForest) Open(readOnly bool) error {
 	err := util.MkdirAllX(filepath.Dir(t.path), t.perm)
 	if err != nil {
diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go
index 2d781cffc..87e57fcc2 100644
--- a/pkg/local_object_storage/pilorama/forest.go
+++ b/pkg/local_object_storage/pilorama/forest.go
@@ -3,6 +3,7 @@ package pilorama
 import (
 	"sort"
 
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
 )
 
@@ -111,7 +112,9 @@ func (f *memoryForest) Init() error {
 func (f *memoryForest) Open(bool) error {
 	return nil
 }
-
+func (f *memoryForest) SetMode(mode.Mode) error {
+	return nil
+}
 func (f *memoryForest) Close() error {
 	return nil
 }
diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go
index b7e98e328..b96c36dfa 100644
--- a/pkg/local_object_storage/pilorama/interface.go
+++ b/pkg/local_object_storage/pilorama/interface.go
@@ -3,6 +3,7 @@ package pilorama
 import (
 	"errors"
 
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
 )
 
@@ -41,6 +42,7 @@ type ForestStorage interface {
 	Init() error
 	Open(bool) error
 	Close() error
+	SetMode(m mode.Mode) error
 	Forest
 }
 
diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go
index 73f7934fb..16bd54c02 100644
--- a/pkg/local_object_storage/shard/get.go
+++ b/pkg/local_object_storage/shard/get.go
@@ -88,7 +88,8 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) {
 		return res.Object(), nil
 	}
 
-	obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small)
+	skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
+	obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
 
 	return GetRes{
 		obj:     obj,
diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go
index d3a1f34f0..0d38a6c5c 100644
--- a/pkg/local_object_storage/shard/mode.go
+++ b/pkg/local_object_storage/shard/mode.go
@@ -21,8 +21,40 @@ func (s *Shard) SetMode(m mode.Mode) error {
 	s.m.Lock()
 	defer s.m.Unlock()
 
+	if s.info.Mode == m {
+		return nil
+	}
+
+	components := []interface{ SetMode(mode.Mode) error }{
+		s.metaBase, s.blobStor,
+	}
+
 	if s.hasWriteCache() {
-		s.writeCache.SetMode(m)
+		components = append(components, s.writeCache)
+	}
+
+	if s.pilorama != nil {
+		components = append(components, s.pilorama)
+	}
+
+	// The usual flow of the requests (pilorama is independent):
+	// writecache -> blobstor -> metabase
+	// For mode.ReadOnly and mode.Degraded the order is:
+	// writecache -> blobstor -> metabase
+	// For mode.ReadWrite it is the opposite:
+	// metabase -> blobstor -> writecache
+	if m != mode.ReadWrite {
+		if s.hasWriteCache() {
+			components[0], components[2] = components[2], components[0]
+		} else {
+			components[0], components[1] = components[1], components[0]
+		}
+	}
+
+	for i := range components {
+		if err := components[i].SetMode(m); err != nil {
+			return err
+		}
 	}
 
 	s.info.Mode = m
diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go
index 4719be698..bf6685a45 100644
--- a/pkg/local_object_storage/writecache/mode.go
+++ b/pkg/local_object_storage/writecache/mode.go
@@ -2,6 +2,7 @@ package writecache
 
 import (
 	"errors"
+	"fmt"
 	"time"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
@@ -13,22 +14,28 @@ var ErrReadOnly = errors.New("write-cache is in read-only mode")
 // SetMode sets write-cache mode of operation.
 // When shard is put in read-only mode all objects in memory are flushed to disk
 // and all background jobs are suspended.
-func (c *cache) SetMode(m mode.Mode) {
+func (c *cache) SetMode(m mode.Mode) error {
 	c.modeMtx.Lock()
 	defer c.modeMtx.Unlock()
-	if c.mode == m {
-		return
+
+	if m.ReadOnly() == c.readOnly() {
+		c.mode = m
+		return nil
 	}
 
-	c.mode = m
-	if m == mode.ReadWrite {
-		return
+	if !c.readOnly() {
+		// Because modeMtx is taken no new objects will arrive an all other modifying
+		// operations are completed.
+		// 1. Persist objects already in memory on disk.
+		c.persistMemoryCache()
 	}
 
-	// Because modeMtx is taken no new objects will arrive an all other modifying
-	// operations are completed.
-	// 1. Persist objects already in memory on disk.
-	c.persistMemoryCache()
+	if c.db != nil {
+		if err := c.db.Close(); err != nil {
+			return fmt.Errorf("can't close write-cache database: %w", err)
+		}
+		c.db = nil
+	}
 
 	// 2. Suspend producers to ensure there are channel send operations in fly.
 	// metaCh and directCh can be populated either during Put or in background memory persist thread.
@@ -40,10 +47,17 @@ func (c *cache) SetMode(m mode.Mode) {
 		c.log.Info("waiting for channels to flush")
 		time.Sleep(time.Second)
 	}
+
+	if err := c.openStore(m.ReadOnly()); err != nil {
+		return err
+	}
+
+	c.mode = m
+	return nil
 }
 
 // readOnly returns true if current mode is read-only.
 // `c.modeMtx` must be taken.
 func (c *cache) readOnly() bool {
-	return c.mode != mode.ReadWrite
+	return c.mode.ReadOnly()
 }
diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go
index 6279ded3c..76e8fbe68 100644
--- a/pkg/local_object_storage/writecache/storage.go
+++ b/pkg/local_object_storage/writecache/storage.go
@@ -41,12 +41,14 @@ func (c *cache) openStore(readOnly bool) error {
 	c.db.MaxBatchSize = c.maxBatchSize
 	c.db.MaxBatchDelay = c.maxBatchDelay
 
-	err = c.db.Update(func(tx *bbolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists(defaultBucket)
-		return err
-	})
-	if err != nil {
-		return fmt.Errorf("could not create default bucket: %w", err)
+	if !readOnly {
+		err = c.db.Update(func(tx *bbolt.Tx) error {
+			_, err := tx.CreateBucketIfNotExists(defaultBucket)
+			return err
+		})
+		if err != nil {
+			return fmt.Errorf("could not create default bucket: %w", err)
+		}
 	}
 
 	c.fsTree = &fstree.FSTree{
diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go
index a4b7248b7..40902be0f 100644
--- a/pkg/local_object_storage/writecache/writecache.go
+++ b/pkg/local_object_storage/writecache/writecache.go
@@ -24,7 +24,7 @@ type Cache interface {
 	Delete(oid.Address) error
 	Iterate(IterationPrm) error
 	Put(*object.Object) error
-	SetMode(mode.Mode)
+	SetMode(mode.Mode) error
 	SetLogger(*zap.Logger)
 	DumpInfo() Info
 
@@ -153,9 +153,16 @@ func (c *cache) Init() error {
 // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op.
 func (c *cache) Close() error {
 	// Finish all in-progress operations.
-	c.SetMode(mode.ReadOnly)
+	if err := c.SetMode(mode.ReadOnly); err != nil {
+		return err
+	}
 
 	close(c.closeCh)
-	c.objCounters.FlushAndClose()
-	return c.db.Close()
+	if c.objCounters != nil {
+		c.objCounters.FlushAndClose()
+	}
+	if c.db != nil {
+		return c.db.Close()
+	}
+	return nil
 }