From c8911d72d015752bbcf365d98b52fae4dc3ee499 Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <evgeniy@nspcc.ru>
Date: Wed, 29 Jun 2022 14:27:36 +0300
Subject: [PATCH] [#1559] shard: Do not consult metabase in a degraded mode

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
---
 pkg/local_object_storage/engine/error_test.go |  2 +-
 pkg/local_object_storage/engine/range.go      | 24 ++++++++++---
 pkg/local_object_storage/shard/delete.go      |  6 ++--
 pkg/local_object_storage/shard/dump.go        |  3 +-
 pkg/local_object_storage/shard/exists.go      | 36 ++++++++-----------
 pkg/local_object_storage/shard/head.go        | 27 +++++++++-----
 pkg/local_object_storage/shard/inhume.go      |  6 ++--
 pkg/local_object_storage/shard/list.go        |  4 +++
 pkg/local_object_storage/shard/lock.go        |  6 ++--
 pkg/local_object_storage/shard/mode.go        |  3 ++
 pkg/local_object_storage/shard/mode/mode.go   | 20 ++++++++---
 pkg/local_object_storage/shard/move.go        |  6 ++--
 pkg/local_object_storage/shard/put.go         | 21 +++++------
 pkg/local_object_storage/shard/range.go       |  3 +-
 pkg/local_object_storage/shard/restore.go     |  3 +-
 pkg/local_object_storage/shard/select.go      |  4 +++
 pkg/local_object_storage/shard/tree.go        |  7 ++--
 17 files changed, 114 insertions(+), 67 deletions(-)

diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go
index 0161ae62f..4a22ae25b 100644
--- a/pkg/local_object_storage/engine/error_test.go
+++ b/pkg/local_object_storage/engine/error_test.go
@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 4, mode.Degraded)
+	checkShardState(t, e, id[0], 2, mode.Degraded)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 
diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go
index 7b3c9d091..554fc9850 100644
--- a/pkg/local_object_storage/engine/range.go
+++ b/pkg/local_object_storage/engine/range.go
@@ -86,11 +86,17 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 		metaError     error
 	)
 
+	var hasDegraded bool
+
 	var shPrm shard.RngPrm
 	shPrm.SetAddress(prm.addr)
 	shPrm.SetRange(prm.off, prm.ln)
 
 	e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
+		noMeta := sh.GetMode().NoMetabase()
+		hasDegraded = hasDegraded || noMeta
+		shPrm.SetIgnoreMeta(noMeta)
+
 		res, err := sh.GetRange(shPrm)
 		if err != nil {
 			if res.HasMeta() {
@@ -140,7 +146,9 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 	}
 
 	if obj == nil {
-		if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
+		// If any shard is in a degraded mode, we should assume that metabase could store
+		// info about some object.
+		if shardWithMeta.Shard == nil && !hasDegraded || !shard.IsErrNotFound(outError) {
 			return RngRes{}, outError
 		}
 
@@ -150,6 +158,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 		shPrm.SetIgnoreMeta(true)
 
 		e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
+			if sh.GetMode().NoMetabase() {
+				// Already processed it without a metabase.
+				return false
+			}
+
 			res, err := sh.GetRange(shPrm)
 			if shard.IsErrOutOfRange(err) {
 				var errOutOfRange apistatus.ObjectOutOfRange
@@ -163,10 +176,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 		if obj == nil {
 			return RngRes{}, outError
 		}
-		e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
-			metaError,
-			zap.Stringer("address", prm.addr),
-		)
+		if shardWithMeta.Shard != nil {
+			e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
+				metaError,
+				zap.Stringer("address", prm.addr))
+		}
 	}
 
 	return RngRes{
diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go
index 465688967..0cb13699c 100644
--- a/pkg/local_object_storage/shard/delete.go
+++ b/pkg/local_object_storage/shard/delete.go
@@ -4,7 +4,6 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
@@ -28,8 +27,11 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) {
 // Delete removes data from the shard's writeCache, metaBase and
 // blobStor.
 func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return DeleteRes{}, ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return DeleteRes{}, ErrDegradedMode
 	}
 
 	ln := len(prm.addr)
diff --git a/pkg/local_object_storage/shard/dump.go b/pkg/local_object_storage/shard/dump.go
index 2b81cfc4a..d62149dbb 100644
--- a/pkg/local_object_storage/shard/dump.go
+++ b/pkg/local_object_storage/shard/dump.go
@@ -7,7 +7,6 @@ import (
 	"os"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 )
 
@@ -56,7 +55,7 @@ func (s *Shard) Dump(prm DumpPrm) (DumpRes, error) {
 	s.m.RLock()
 	defer s.m.RUnlock()
 
-	if s.info.Mode != mode.ReadOnly {
+	if !s.info.Mode.ReadOnly() {
 		return DumpRes{}, ErrMustBeReadOnly
 	}
 
diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go
index 4cfb8143d..dd7594a85 100644
--- a/pkg/local_object_storage/shard/exists.go
+++ b/pkg/local_object_storage/shard/exists.go
@@ -3,9 +3,7 @@ package shard
 import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
-	"go.uber.org/zap"
 )
 
 // ExistsPrm groups the parameters of Exists operation.
@@ -35,27 +33,23 @@ func (p ExistsRes) Exists() bool {
 //
 // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed.
 func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) {
-	var existsPrm meta.ExistsPrm
-	existsPrm.SetAddress(prm.addr)
+	var exists bool
+	var err error
 
-	res, err := s.metaBase.Exists(existsPrm)
-	exists := res.Exists()
-	if err != nil {
-		// If the shard is in degraded mode, try to consult blobstor directly.
-		// Otherwise, just return an error.
-		if s.GetMode() == mode.Degraded {
-			var p blobstor.ExistsPrm
-			p.SetAddress(prm.addr)
+	if s.GetMode().NoMetabase() {
+		var p blobstor.ExistsPrm
+		p.SetAddress(prm.addr)
 
-			res, bErr := s.blobStor.Exists(p)
-			if bErr == nil {
-				exists = res.Exists()
-				s.log.Warn("metabase existence check finished with error",
-					zap.Stringer("address", prm.addr),
-					zap.String("error", err.Error()))
-				err = nil
-			}
-		}
+		var res blobstor.ExistsRes
+		res, err = s.blobStor.Exists(p)
+		exists = res.Exists()
+	} else {
+		var existsPrm meta.ExistsPrm
+		existsPrm.SetAddress(prm.addr)
+
+		var res meta.ExistsRes
+		res, err = s.metaBase.Exists(existsPrm)
+		exists = res.Exists()
 	}
 
 	return ExistsRes{
diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go
index 4699c55fd..80d85df19 100644
--- a/pkg/local_object_storage/shard/head.go
+++ b/pkg/local_object_storage/shard/head.go
@@ -63,16 +63,27 @@ func (s *Shard) Head(prm HeadPrm) (HeadRes, error) {
 		// otherwise object seems to be flushed to metabase
 	}
 
-	var headParams meta.GetPrm
-	headParams.SetAddress(prm.addr)
-	headParams.SetRaw(prm.raw)
+	var obj *objectSDK.Object
+	var err error
+	if s.GetMode().NoMetabase() {
+		var getPrm GetPrm
+		getPrm.SetAddress(prm.addr)
+		getPrm.SetIgnoreMeta(true)
 
-	res, err := s.metaBase.Get(headParams)
-	if err != nil {
-		return HeadRes{}, err
+		var res GetRes
+		res, err = s.Get(getPrm)
+		obj = res.Object()
+	} else {
+		var headParams meta.GetPrm
+		headParams.SetAddress(prm.addr)
+		headParams.SetRaw(prm.raw)
+
+		var res meta.GetRes
+		res, err = s.metaBase.Get(headParams)
+		obj = res.Header()
 	}
 
 	return HeadRes{
-		obj: res.Header(),
-	}, nil
+		obj: obj,
+	}, err
 }
diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go
index 6811d5f93..4df938223 100644
--- a/pkg/local_object_storage/shard/inhume.go
+++ b/pkg/local_object_storage/shard/inhume.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
 )
@@ -62,8 +61,11 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval
 //
 // Returns ErrReadOnlyMode error if shard is in "read-only" mode.
 func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return InhumeRes{}, ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return InhumeRes{}, ErrDegradedMode
 	}
 
 	if s.hasWriteCache() {
diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go
index 4ac240bc4..128847b4c 100644
--- a/pkg/local_object_storage/shard/list.go
+++ b/pkg/local_object_storage/shard/list.go
@@ -110,6 +110,10 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) {
 // Returns ErrEndOfListing if there are no more objects to return or count
 // parameter set to zero.
 func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) {
+	if s.GetMode().NoMetabase() {
+		return ListWithCursorRes{}, ErrDegradedMode
+	}
+
 	var metaPrm meta.ListPrm
 	metaPrm.SetCount(prm.count)
 	metaPrm.SetCursor(prm.cursor)
diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go
index 197a09388..0b8ffeb60 100644
--- a/pkg/local_object_storage/shard/lock.go
+++ b/pkg/local_object_storage/shard/lock.go
@@ -3,7 +3,6 @@ package shard
 import (
 	"fmt"
 
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 )
@@ -15,8 +14,11 @@ import (
 //
 // Locked list should be unique. Panics if it is empty.
 func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return ErrDegradedMode
 	}
 
 	err := s.metaBase.Lock(idCnr, locker, locked)
diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go
index b7ede5e72..d3a1f34f0 100644
--- a/pkg/local_object_storage/shard/mode.go
+++ b/pkg/local_object_storage/shard/mode.go
@@ -10,6 +10,9 @@ import (
 // that changes shard's memory due to the "read-only" shard's mode.
 var ErrReadOnlyMode = errors.New("shard is in read-only mode")
 
+// ErrDegradedMode is returned when operation requiring metabase is executed in degraded mode.
+var ErrDegradedMode = errors.New("shard is in degraded mode")
+
 // SetMode sets mode of the shard.
 //
 // Returns any error encountered that did not allow
diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go
index dffaf5b41..a0aef6e61 100644
--- a/pkg/local_object_storage/shard/mode/mode.go
+++ b/pkg/local_object_storage/shard/mode/mode.go
@@ -3,14 +3,14 @@ package mode
 // Mode represents enumeration of Shard work modes.
 type Mode uint32
 
-const (
-	// ReadWrite is a Mode value for shard that is available
-	// for read and write operations. Default shard mode.
-	ReadWrite Mode = iota
+// ReadWrite is a Mode value for shard that is available
+// for read and write operations. Default shard mode.
+const ReadWrite Mode = 0
 
+const (
 	// ReadOnly is a Mode value for shard that does not
 	// accept write operation but is readable.
-	ReadOnly
+	ReadOnly Mode = 1 << iota
 
 	// Degraded is a Mode value for shard that is set automatically
 	// after a certain number of errors is encountered. It is the same as
@@ -31,3 +31,13 @@ func (m Mode) String() string {
 		return "DEGRADED"
 	}
 }
+
+// NoMetabase returns true iff m is operating without the metabase.
+func (m Mode) NoMetabase() bool {
+	return m&Degraded != 0
+}
+
+// ReadOnly returns true iff m prohibits modifying operations with shard.
+func (m Mode) ReadOnly() bool {
+	return m&ReadOnly != 0
+}
diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go
index 5b203d0ca..0068dc7dd 100644
--- a/pkg/local_object_storage/shard/move.go
+++ b/pkg/local_object_storage/shard/move.go
@@ -2,7 +2,6 @@ package shard
 
 import (
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
 )
@@ -24,8 +23,11 @@ func (p *ToMoveItPrm) SetAddress(addr oid.Address) {
 // ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to
 // another shard.
 func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return ToMoveItRes{}, ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return ToMoveItRes{}, ErrDegradedMode
 	}
 
 	var toMovePrm meta.ToMoveItPrm
diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go
index db59cf285..87fcfcd25 100644
--- a/pkg/local_object_storage/shard/put.go
+++ b/pkg/local_object_storage/shard/put.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	"go.uber.org/zap"
 )
@@ -30,7 +29,8 @@ func (p *PutPrm) SetObject(obj *object.Object) {
 //
 // Returns ErrReadOnlyMode error if shard is in "read-only" mode.
 func (s *Shard) Put(prm PutPrm) (PutRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return PutRes{}, ErrReadOnlyMode
 	}
 
@@ -58,14 +58,15 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
 		return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err)
 	}
 
-	// put to metabase
-	var pPrm meta.PutPrm
-	pPrm.SetObject(prm.obj)
-	pPrm.SetBlobovniczaID(res.BlobovniczaID())
-	if _, err := s.metaBase.Put(pPrm); err != nil {
-		// may we need to handle this case in a special way
-		// since the object has been successfully written to BlobStor
-		return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err)
+	if !m.NoMetabase() {
+		var pPrm meta.PutPrm
+		pPrm.SetObject(prm.obj)
+		pPrm.SetBlobovniczaID(res.BlobovniczaID())
+		if _, err := s.metaBase.Put(pPrm); err != nil {
+			// may we need to handle this case in a special way
+			// since the object has been successfully written to BlobStor
+			return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err)
+		}
 	}
 
 	return PutRes{}, nil
diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go
index 284561af4..84b7dda97 100644
--- a/pkg/local_object_storage/shard/range.go
+++ b/pkg/local_object_storage/shard/range.go
@@ -102,7 +102,8 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) {
 		return obj, nil
 	}
 
-	obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small)
+	skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
+	obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
 
 	return RngRes{
 		obj:     obj,
diff --git a/pkg/local_object_storage/shard/restore.go b/pkg/local_object_storage/shard/restore.go
index a996252c5..ae37b159c 100644
--- a/pkg/local_object_storage/shard/restore.go
+++ b/pkg/local_object_storage/shard/restore.go
@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 )
 
@@ -62,7 +61,7 @@ func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) {
 	s.m.RLock()
 	defer s.m.RUnlock()
 
-	if s.info.Mode != mode.ReadWrite {
+	if s.info.Mode.ReadOnly() {
 		return RestoreRes{}, ErrReadOnlyMode
 	}
 
diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go
index 58f7f1501..7d675c203 100644
--- a/pkg/local_object_storage/shard/select.go
+++ b/pkg/local_object_storage/shard/select.go
@@ -40,6 +40,10 @@ func (r SelectRes) AddressList() []oid.Address {
 // Returns any error encountered that
 // did not allow to completely select the objects.
 func (s *Shard) Select(prm SelectPrm) (SelectRes, error) {
+	if s.GetMode().NoMetabase() {
+		return SelectRes{}, ErrDegradedMode
+	}
+
 	var selectPrm meta.SelectPrm
 	selectPrm.SetFilters(prm.filters)
 	selectPrm.SetContainerID(prm.cnr)
diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go
index 2bc272852..df4ffd85d 100644
--- a/pkg/local_object_storage/shard/tree.go
+++ b/pkg/local_object_storage/shard/tree.go
@@ -4,7 +4,6 @@ import (
 	"errors"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
 )
 
@@ -18,7 +17,7 @@ func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Mo
 	if s.pilorama == nil {
 		return nil, ErrPiloramaDisabled
 	}
-	if s.GetMode() != mode.ReadWrite {
+	if s.GetMode().ReadOnly() {
 		return nil, ErrReadOnlyMode
 	}
 	return s.pilorama.TreeMove(d, treeID, m)
@@ -29,7 +28,7 @@ func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr stri
 	if s.pilorama == nil {
 		return nil, ErrPiloramaDisabled
 	}
-	if s.GetMode() != mode.ReadWrite {
+	if s.GetMode().ReadOnly() {
 		return nil, ErrReadOnlyMode
 	}
 	return s.pilorama.TreeAddByPath(d, treeID, attr, path, meta)
@@ -40,7 +39,7 @@ func (s *Shard) TreeApply(d pilorama.CIDDescriptor, treeID string, m *pilorama.M
 	if s.pilorama == nil {
 		return ErrPiloramaDisabled
 	}
-	if s.GetMode() != mode.ReadWrite {
+	if s.GetMode().ReadOnly() {
 		return ErrReadOnlyMode
 	}
 	return s.pilorama.TreeApply(d, treeID, m)