[#1559] shard: Do not consult metabase in a degraded mode

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

pkg/local_object_storage/engine/error_test.go

@@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
 		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
 	}
 
-	checkShardState(t, e, id[0], 4, mode.Degraded)
+	checkShardState(t, e, id[0], 2, mode.Degraded)
 	checkShardState(t, e, id[1], 0, mode.ReadWrite)
 }
 

pkg/local_object_storage/engine/range.go

@@ -86,11 +86,17 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 		metaError     error
 	)
 
+	var hasDegraded bool
+
 	var shPrm shard.RngPrm
 	shPrm.SetAddress(prm.addr)
 	shPrm.SetRange(prm.off, prm.ln)
 
 	e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
+		noMeta := sh.GetMode().NoMetabase()
+		hasDegraded = hasDegraded || noMeta
+		shPrm.SetIgnoreMeta(noMeta)
+
 		res, err := sh.GetRange(shPrm)
 		if err != nil {
 			if res.HasMeta() {
@@ -140,7 +146,9 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 	}
 
 	if obj == nil {
-		if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
+		// If any shard is in a degraded mode, we should assume that metabase could store
+		// info about some object.
+		if shardWithMeta.Shard == nil && !hasDegraded || !shard.IsErrNotFound(outError) {
 			return RngRes{}, outError
 		}
 
@@ -150,6 +158,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 		shPrm.SetIgnoreMeta(true)
 
 		e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
+			if sh.GetMode().NoMetabase() {
+				// Already processed it without a metabase.
+				return false
+			}
+
 			res, err := sh.GetRange(shPrm)
 			if shard.IsErrOutOfRange(err) {
 				var errOutOfRange apistatus.ObjectOutOfRange
@@ -163,10 +176,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
 		if obj == nil {
 			return RngRes{}, outError
 		}
+		if shardWithMeta.Shard != nil {
 			e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
 				metaError,
-			zap.Stringer("address", prm.addr),
+				zap.Stringer("address", prm.addr))
-		)
+		}
 	}
 
 	return RngRes{

pkg/local_object_storage/shard/delete.go

@@ -4,7 +4,6 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
@@ -28,8 +27,11 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) {
 // Delete removes data from the shard's writeCache, metaBase and
 // blobStor.
 func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return DeleteRes{}, ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return DeleteRes{}, ErrDegradedMode
 	}
 
 	ln := len(prm.addr)

pkg/local_object_storage/shard/dump.go

@@ -7,7 +7,6 @@ import (
 	"os"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 )
 
@@ -56,7 +55,7 @@ func (s *Shard) Dump(prm DumpPrm) (DumpRes, error) {
 	s.m.RLock()
 	defer s.m.RUnlock()
 
-	if s.info.Mode != mode.ReadOnly {
+	if !s.info.Mode.ReadOnly() {
 		return DumpRes{}, ErrMustBeReadOnly
 	}
 

pkg/local_object_storage/shard/exists.go

@@ -3,9 +3,7 @@ package shard
 import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
-	"go.uber.org/zap"
 )
 
 // ExistsPrm groups the parameters of Exists operation.
@@ -35,27 +33,23 @@ func (p ExistsRes) Exists() bool {
 //
 // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed.
 func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) {
-	var existsPrm meta.ExistsPrm
+	var exists bool
-	existsPrm.SetAddress(prm.addr)
+	var err error
 
-	res, err := s.metaBase.Exists(existsPrm)
+	if s.GetMode().NoMetabase() {
-	exists := res.Exists()
-	if err != nil {
-		// If the shard is in degraded mode, try to consult blobstor directly.
-		// Otherwise, just return an error.
-		if s.GetMode() == mode.Degraded {
 		var p blobstor.ExistsPrm
 		p.SetAddress(prm.addr)
 
-			res, bErr := s.blobStor.Exists(p)
+		var res blobstor.ExistsRes
-			if bErr == nil {
+		res, err = s.blobStor.Exists(p)
+		exists = res.Exists()
+	} else {
+		var existsPrm meta.ExistsPrm
+		existsPrm.SetAddress(prm.addr)
+
+		var res meta.ExistsRes
+		res, err = s.metaBase.Exists(existsPrm)
 		exists = res.Exists()
-				s.log.Warn("metabase existence check finished with error",
-					zap.Stringer("address", prm.addr),
-					zap.String("error", err.Error()))
-				err = nil
-			}
-		}
 	}
 
 	return ExistsRes{

pkg/local_object_storage/shard/head.go

@@ -63,16 +63,27 @@ func (s *Shard) Head(prm HeadPrm) (HeadRes, error) {
 		// otherwise object seems to be flushed to metabase
 	}
 
+	var obj *objectSDK.Object
+	var err error
+	if s.GetMode().NoMetabase() {
+		var getPrm GetPrm
+		getPrm.SetAddress(prm.addr)
+		getPrm.SetIgnoreMeta(true)
+
+		var res GetRes
+		res, err = s.Get(getPrm)
+		obj = res.Object()
+	} else {
 		var headParams meta.GetPrm
 		headParams.SetAddress(prm.addr)
 		headParams.SetRaw(prm.raw)
 
-	res, err := s.metaBase.Get(headParams)
+		var res meta.GetRes
-	if err != nil {
+		res, err = s.metaBase.Get(headParams)
-		return HeadRes{}, err
+		obj = res.Header()
 	}
 
 	return HeadRes{
-		obj: res.Header(),
+		obj: obj,
-	}, nil
+	}, err
 }

pkg/local_object_storage/shard/inhume.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
 )
@@ -62,8 +61,11 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval
 //
 // Returns ErrReadOnlyMode error if shard is in "read-only" mode.
 func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return InhumeRes{}, ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return InhumeRes{}, ErrDegradedMode
 	}
 
 	if s.hasWriteCache() {

pkg/local_object_storage/shard/list.go

@@ -110,6 +110,10 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) {
 // Returns ErrEndOfListing if there are no more objects to return or count
 // parameter set to zero.
 func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) {
+	if s.GetMode().NoMetabase() {
+		return ListWithCursorRes{}, ErrDegradedMode
+	}
+
 	var metaPrm meta.ListPrm
 	metaPrm.SetCount(prm.count)
 	metaPrm.SetCursor(prm.cursor)

pkg/local_object_storage/shard/lock.go

@@ -3,7 +3,6 @@ package shard
 import (
 	"fmt"
 
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 )
@@ -15,8 +14,11 @@ import (
 //
 // Locked list should be unique. Panics if it is empty.
 func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return ErrDegradedMode
 	}
 
 	err := s.metaBase.Lock(idCnr, locker, locked)

pkg/local_object_storage/shard/mode.go

@@ -10,6 +10,9 @@ import (
 // that changes shard's memory due to the "read-only" shard's mode.
 var ErrReadOnlyMode = errors.New("shard is in read-only mode")
 
+// ErrDegradedMode is returned when operation requiring metabase is executed in degraded mode.
+var ErrDegradedMode = errors.New("shard is in degraded mode")
+
 // SetMode sets mode of the shard.
 //
 // Returns any error encountered that did not allow

pkg/local_object_storage/shard/mode/mode.go

@@ -3,14 +3,14 @@ package mode
 // Mode represents enumeration of Shard work modes.
 type Mode uint32
 
-const (
+// ReadWrite is a Mode value for shard that is available
-	// ReadWrite is a Mode value for shard that is available
+// for read and write operations. Default shard mode.
-	// for read and write operations. Default shard mode.
+const ReadWrite Mode = 0
-	ReadWrite Mode = iota
 
+const (
 	// ReadOnly is a Mode value for shard that does not
 	// accept write operation but is readable.
-	ReadOnly
+	ReadOnly Mode = 1 << iota
 
 	// Degraded is a Mode value for shard that is set automatically
 	// after a certain number of errors is encountered. It is the same as
@@ -31,3 +31,13 @@ func (m Mode) String() string {
 		return "DEGRADED"
 	}
 }
+
+// NoMetabase returns true iff m is operating without the metabase.
+func (m Mode) NoMetabase() bool {
+	return m&Degraded != 0
+}
+
+// ReadOnly returns true iff m prohibits modifying operations with shard.
+func (m Mode) ReadOnly() bool {
+	return m&ReadOnly != 0
+}

pkg/local_object_storage/shard/move.go

@@ -2,7 +2,6 @@ package shard
 
 import (
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"go.uber.org/zap"
 )
@@ -24,8 +23,11 @@ func (p *ToMoveItPrm) SetAddress(addr oid.Address) {
 // ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to
 // another shard.
 func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return ToMoveItRes{}, ErrReadOnlyMode
+	} else if m.NoMetabase() {
+		return ToMoveItRes{}, ErrDegradedMode
 	}
 
 	var toMovePrm meta.ToMoveItPrm

pkg/local_object_storage/shard/put.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	"go.uber.org/zap"
 )
@@ -30,7 +29,8 @@ func (p *PutPrm) SetObject(obj *object.Object) {
 //
 // Returns ErrReadOnlyMode error if shard is in "read-only" mode.
 func (s *Shard) Put(prm PutPrm) (PutRes, error) {
-	if s.GetMode() != mode.ReadWrite {
+	m := s.GetMode()
+	if m.ReadOnly() {
 		return PutRes{}, ErrReadOnlyMode
 	}
 
@@ -58,7 +58,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
 		return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err)
 	}
 
-	// put to metabase
+	if !m.NoMetabase() {
 		var pPrm meta.PutPrm
 		pPrm.SetObject(prm.obj)
 		pPrm.SetBlobovniczaID(res.BlobovniczaID())
@@ -67,6 +67,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
 			// since the object has been successfully written to BlobStor
 			return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err)
 		}
+	}
 
 	return PutRes{}, nil
 }

pkg/local_object_storage/shard/range.go

@@ -102,7 +102,8 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) {
 		return obj, nil
 	}
 
-	obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small)
+	skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
+	obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
 
 	return RngRes{
 		obj:     obj,

pkg/local_object_storage/shard/restore.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 )
 
@@ -62,7 +61,7 @@ func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) {
 	s.m.RLock()
 	defer s.m.RUnlock()
 
-	if s.info.Mode != mode.ReadWrite {
+	if s.info.Mode.ReadOnly() {
 		return RestoreRes{}, ErrReadOnlyMode
 	}
 

pkg/local_object_storage/shard/select.go

@@ -40,6 +40,10 @@ func (r SelectRes) AddressList() []oid.Address {
 // Returns any error encountered that
 // did not allow to completely select the objects.
 func (s *Shard) Select(prm SelectPrm) (SelectRes, error) {
+	if s.GetMode().NoMetabase() {
+		return SelectRes{}, ErrDegradedMode
+	}
+
 	var selectPrm meta.SelectPrm
 	selectPrm.SetFilters(prm.filters)
 	selectPrm.SetContainerID(prm.cnr)

pkg/local_object_storage/shard/tree.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
 )
 
@@ -18,7 +17,7 @@ func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Mo
 	if s.pilorama == nil {
 		return nil, ErrPiloramaDisabled
 	}
-	if s.GetMode() != mode.ReadWrite {
+	if s.GetMode().ReadOnly() {
 		return nil, ErrReadOnlyMode
 	}
 	return s.pilorama.TreeMove(d, treeID, m)
@@ -29,7 +28,7 @@ func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr stri
 	if s.pilorama == nil {
 		return nil, ErrPiloramaDisabled
 	}
-	if s.GetMode() != mode.ReadWrite {
+	if s.GetMode().ReadOnly() {
 		return nil, ErrReadOnlyMode
 	}
 	return s.pilorama.TreeAddByPath(d, treeID, attr, path, meta)
@@ -40,7 +39,7 @@ func (s *Shard) TreeApply(d pilorama.CIDDescriptor, treeID string, m *pilorama.M
 	if s.pilorama == nil {
 		return ErrPiloramaDisabled
 	}
-	if s.GetMode() != mode.ReadWrite {
+	if s.GetMode().ReadOnly() {
 		return ErrReadOnlyMode
 	}
 	return s.pilorama.TreeApply(d, treeID, m)