[#1559] shard: Do not consult metabase in a degraded mode

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-06-29 14:27:36 +03:00 committed by fyrchik
parent 339864b720
commit c8911d72d0
17 changed files with 114 additions and 67 deletions

View file

@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) {
require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
}
checkShardState(t, e, id[0], 4, mode.Degraded)
checkShardState(t, e, id[0], 2, mode.Degraded)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
}

View file

@ -86,11 +86,17 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
metaError error
)
var hasDegraded bool
var shPrm shard.RngPrm
shPrm.SetAddress(prm.addr)
shPrm.SetRange(prm.off, prm.ln)
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
noMeta := sh.GetMode().NoMetabase()
hasDegraded = hasDegraded || noMeta
shPrm.SetIgnoreMeta(noMeta)
res, err := sh.GetRange(shPrm)
if err != nil {
if res.HasMeta() {
@ -140,7 +146,9 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
}
if obj == nil {
if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) {
// If any shard is in a degraded mode, we should assume that metabase could store
// info about some object.
if shardWithMeta.Shard == nil && !hasDegraded || !shard.IsErrNotFound(outError) {
return RngRes{}, outError
}
@ -150,6 +158,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
shPrm.SetIgnoreMeta(true)
e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) {
if sh.GetMode().NoMetabase() {
// Already processed it without a metabase.
return false
}
res, err := sh.GetRange(shPrm)
if shard.IsErrOutOfRange(err) {
var errOutOfRange apistatus.ObjectOutOfRange
@ -163,10 +176,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) {
if obj == nil {
return RngRes{}, outError
}
if shardWithMeta.Shard != nil {
e.reportShardError(shardWithMeta, "meta info was present, but object is missing",
metaError,
zap.Stringer("address", prm.addr),
)
zap.Stringer("address", prm.addr))
}
}
return RngRes{

View file

@ -4,7 +4,6 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
@ -28,8 +27,11 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) {
// Delete removes data from the shard's writeCache, metaBase and
// blobStor.
func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) {
if s.GetMode() != mode.ReadWrite {
m := s.GetMode()
if m.ReadOnly() {
return DeleteRes{}, ErrReadOnlyMode
} else if m.NoMetabase() {
return DeleteRes{}, ErrDegradedMode
}
ln := len(prm.addr)

View file

@ -7,7 +7,6 @@ import (
"os"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
)
@ -56,7 +55,7 @@ func (s *Shard) Dump(prm DumpPrm) (DumpRes, error) {
s.m.RLock()
defer s.m.RUnlock()
if s.info.Mode != mode.ReadOnly {
if !s.info.Mode.ReadOnly() {
return DumpRes{}, ErrMustBeReadOnly
}

View file

@ -3,9 +3,7 @@ package shard
import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
// ExistsPrm groups the parameters of Exists operation.
@ -35,27 +33,23 @@ func (p ExistsRes) Exists() bool {
//
// Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed.
func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) {
var existsPrm meta.ExistsPrm
existsPrm.SetAddress(prm.addr)
var exists bool
var err error
res, err := s.metaBase.Exists(existsPrm)
exists := res.Exists()
if err != nil {
// If the shard is in degraded mode, try to consult blobstor directly.
// Otherwise, just return an error.
if s.GetMode() == mode.Degraded {
if s.GetMode().NoMetabase() {
var p blobstor.ExistsPrm
p.SetAddress(prm.addr)
res, bErr := s.blobStor.Exists(p)
if bErr == nil {
var res blobstor.ExistsRes
res, err = s.blobStor.Exists(p)
exists = res.Exists()
} else {
var existsPrm meta.ExistsPrm
existsPrm.SetAddress(prm.addr)
var res meta.ExistsRes
res, err = s.metaBase.Exists(existsPrm)
exists = res.Exists()
s.log.Warn("metabase existence check finished with error",
zap.Stringer("address", prm.addr),
zap.String("error", err.Error()))
err = nil
}
}
}
return ExistsRes{

View file

@ -63,16 +63,27 @@ func (s *Shard) Head(prm HeadPrm) (HeadRes, error) {
// otherwise object seems to be flushed to metabase
}
var obj *objectSDK.Object
var err error
if s.GetMode().NoMetabase() {
var getPrm GetPrm
getPrm.SetAddress(prm.addr)
getPrm.SetIgnoreMeta(true)
var res GetRes
res, err = s.Get(getPrm)
obj = res.Object()
} else {
var headParams meta.GetPrm
headParams.SetAddress(prm.addr)
headParams.SetRaw(prm.raw)
res, err := s.metaBase.Get(headParams)
if err != nil {
return HeadRes{}, err
var res meta.GetRes
res, err = s.metaBase.Get(headParams)
obj = res.Header()
}
return HeadRes{
obj: res.Header(),
}, nil
obj: obj,
}, err
}

View file

@ -6,7 +6,6 @@ import (
"fmt"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
@ -62,8 +61,11 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval
//
// Returns ErrReadOnlyMode error if shard is in "read-only" mode.
func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) {
if s.GetMode() != mode.ReadWrite {
m := s.GetMode()
if m.ReadOnly() {
return InhumeRes{}, ErrReadOnlyMode
} else if m.NoMetabase() {
return InhumeRes{}, ErrDegradedMode
}
if s.hasWriteCache() {

View file

@ -110,6 +110,10 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) {
// Returns ErrEndOfListing if there are no more objects to return or count
// parameter set to zero.
func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) {
if s.GetMode().NoMetabase() {
return ListWithCursorRes{}, ErrDegradedMode
}
var metaPrm meta.ListPrm
metaPrm.SetCount(prm.count)
metaPrm.SetCursor(prm.cursor)

View file

@ -3,7 +3,6 @@ package shard
import (
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
@ -15,8 +14,11 @@ import (
//
// Locked list should be unique. Panics if it is empty.
func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error {
if s.GetMode() != mode.ReadWrite {
m := s.GetMode()
if m.ReadOnly() {
return ErrReadOnlyMode
} else if m.NoMetabase() {
return ErrDegradedMode
}
err := s.metaBase.Lock(idCnr, locker, locked)

View file

@ -10,6 +10,9 @@ import (
// that changes shard's memory due to the "read-only" shard's mode.
var ErrReadOnlyMode = errors.New("shard is in read-only mode")
// ErrDegradedMode is returned when operation requiring metabase is executed in degraded mode.
var ErrDegradedMode = errors.New("shard is in degraded mode")
// SetMode sets mode of the shard.
//
// Returns any error encountered that did not allow

View file

@ -3,14 +3,14 @@ package mode
// Mode represents enumeration of Shard work modes.
type Mode uint32
const (
// ReadWrite is a Mode value for shard that is available
// for read and write operations. Default shard mode.
ReadWrite Mode = iota
const ReadWrite Mode = 0
const (
// ReadOnly is a Mode value for shard that does not
// accept write operation but is readable.
ReadOnly
ReadOnly Mode = 1 << iota
// Degraded is a Mode value for shard that is set automatically
// after a certain number of errors is encountered. It is the same as
@ -31,3 +31,13 @@ func (m Mode) String() string {
return "DEGRADED"
}
}
// NoMetabase returns true iff m is operating without the metabase.
func (m Mode) NoMetabase() bool {
return m&Degraded != 0
}
// ReadOnly returns true iff m prohibits modifying operations with shard.
func (m Mode) ReadOnly() bool {
return m&ReadOnly != 0
}

View file

@ -2,7 +2,6 @@ package shard
import (
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
@ -24,8 +23,11 @@ func (p *ToMoveItPrm) SetAddress(addr oid.Address) {
// ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to
// another shard.
func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) {
if s.GetMode() != mode.ReadWrite {
m := s.GetMode()
if m.ReadOnly() {
return ToMoveItRes{}, ErrReadOnlyMode
} else if m.NoMetabase() {
return ToMoveItRes{}, ErrDegradedMode
}
var toMovePrm meta.ToMoveItPrm

View file

@ -5,7 +5,6 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-sdk-go/object"
"go.uber.org/zap"
)
@ -30,7 +29,8 @@ func (p *PutPrm) SetObject(obj *object.Object) {
//
// Returns ErrReadOnlyMode error if shard is in "read-only" mode.
func (s *Shard) Put(prm PutPrm) (PutRes, error) {
if s.GetMode() != mode.ReadWrite {
m := s.GetMode()
if m.ReadOnly() {
return PutRes{}, ErrReadOnlyMode
}
@ -58,7 +58,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err)
}
// put to metabase
if !m.NoMetabase() {
var pPrm meta.PutPrm
pPrm.SetObject(prm.obj)
pPrm.SetBlobovniczaID(res.BlobovniczaID())
@ -67,6 +67,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
// since the object has been successfully written to BlobStor
return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err)
}
}
return PutRes{}, nil
}

View file

@ -102,7 +102,8 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) {
return obj, nil
}
obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small)
skipMeta := prm.skipMeta || s.GetMode().NoMetabase()
obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small)
return RngRes{
obj: obj,

View file

@ -7,7 +7,6 @@ import (
"io"
"os"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-sdk-go/object"
)
@ -62,7 +61,7 @@ func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) {
s.m.RLock()
defer s.m.RUnlock()
if s.info.Mode != mode.ReadWrite {
if s.info.Mode.ReadOnly() {
return RestoreRes{}, ErrReadOnlyMode
}

View file

@ -40,6 +40,10 @@ func (r SelectRes) AddressList() []oid.Address {
// Returns any error encountered that
// did not allow to completely select the objects.
func (s *Shard) Select(prm SelectPrm) (SelectRes, error) {
if s.GetMode().NoMetabase() {
return SelectRes{}, ErrDegradedMode
}
var selectPrm meta.SelectPrm
selectPrm.SetFilters(prm.filters)
selectPrm.SetContainerID(prm.cnr)

View file

@ -4,7 +4,6 @@ import (
"errors"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
)
@ -18,7 +17,7 @@ func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Mo
if s.pilorama == nil {
return nil, ErrPiloramaDisabled
}
if s.GetMode() != mode.ReadWrite {
if s.GetMode().ReadOnly() {
return nil, ErrReadOnlyMode
}
return s.pilorama.TreeMove(d, treeID, m)
@ -29,7 +28,7 @@ func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr stri
if s.pilorama == nil {
return nil, ErrPiloramaDisabled
}
if s.GetMode() != mode.ReadWrite {
if s.GetMode().ReadOnly() {
return nil, ErrReadOnlyMode
}
return s.pilorama.TreeAddByPath(d, treeID, attr, path, meta)
@ -40,7 +39,7 @@ func (s *Shard) TreeApply(d pilorama.CIDDescriptor, treeID string, m *pilorama.M
if s.pilorama == nil {
return ErrPiloramaDisabled
}
if s.GetMode() != mode.ReadWrite {
if s.GetMode().ReadOnly() {
return ErrReadOnlyMode
}
return s.pilorama.TreeApply(d, treeID, m)