diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 0161ae62ff..4a22ae25b6 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -193,7 +193,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, e, id[0], 4, mode.Degraded) + checkShardState(t, e, id[0], 2, mode.Degraded) checkShardState(t, e, id[1], 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 7b3c9d0910..554fc9850b 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -86,11 +86,17 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { metaError error ) + var hasDegraded bool + var shPrm shard.RngPrm shPrm.SetAddress(prm.addr) shPrm.SetRange(prm.off, prm.ln) e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + noMeta := sh.GetMode().NoMetabase() + hasDegraded = hasDegraded || noMeta + shPrm.SetIgnoreMeta(noMeta) + res, err := sh.GetRange(shPrm) if err != nil { if res.HasMeta() { @@ -140,7 +146,9 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { } if obj == nil { - if shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) { + // If any shard is in a degraded mode, we should assume that metabase could store + // info about some object. + if shardWithMeta.Shard == nil && !hasDegraded || !shard.IsErrNotFound(outError) { return RngRes{}, outError } @@ -150,6 +158,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { shPrm.SetIgnoreMeta(true) e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + if sh.GetMode().NoMetabase() { + // Already processed it without a metabase. + return false + } + res, err := sh.GetRange(shPrm) if shard.IsErrOutOfRange(err) { var errOutOfRange apistatus.ObjectOutOfRange @@ -163,10 +176,11 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { if obj == nil { return RngRes{}, outError } - e.reportShardError(shardWithMeta, "meta info was present, but object is missing", - metaError, - zap.Stringer("address", prm.addr), - ) + if shardWithMeta.Shard != nil { + e.reportShardError(shardWithMeta, "meta info was present, but object is missing", + metaError, + zap.Stringer("address", prm.addr)) + } } return RngRes{ diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 4656889673..0cb13699cd 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -4,7 +4,6 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" @@ -28,8 +27,11 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) { // Delete removes data from the shard's writeCache, metaBase and // blobStor. func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { - if s.GetMode() != mode.ReadWrite { + m := s.GetMode() + if m.ReadOnly() { return DeleteRes{}, ErrReadOnlyMode + } else if m.NoMetabase() { + return DeleteRes{}, ErrDegradedMode } ln := len(prm.addr) diff --git a/pkg/local_object_storage/shard/dump.go b/pkg/local_object_storage/shard/dump.go index 2b81cfc4aa..d62149dbb7 100644 --- a/pkg/local_object_storage/shard/dump.go +++ b/pkg/local_object_storage/shard/dump.go @@ -7,7 +7,6 @@ import ( "os" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" ) @@ -56,7 +55,7 @@ func (s *Shard) Dump(prm DumpPrm) (DumpRes, error) { s.m.RLock() defer s.m.RUnlock() - if s.info.Mode != mode.ReadOnly { + if !s.info.Mode.ReadOnly() { return DumpRes{}, ErrMustBeReadOnly } diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 4cfb8143d2..dd7594a85d 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -3,9 +3,7 @@ package shard import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "go.uber.org/zap" ) // ExistsPrm groups the parameters of Exists operation. @@ -35,27 +33,23 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) { - var existsPrm meta.ExistsPrm - existsPrm.SetAddress(prm.addr) + var exists bool + var err error - res, err := s.metaBase.Exists(existsPrm) - exists := res.Exists() - if err != nil { - // If the shard is in degraded mode, try to consult blobstor directly. - // Otherwise, just return an error. - if s.GetMode() == mode.Degraded { - var p blobstor.ExistsPrm - p.SetAddress(prm.addr) + if s.GetMode().NoMetabase() { + var p blobstor.ExistsPrm + p.SetAddress(prm.addr) - res, bErr := s.blobStor.Exists(p) - if bErr == nil { - exists = res.Exists() - s.log.Warn("metabase existence check finished with error", - zap.Stringer("address", prm.addr), - zap.String("error", err.Error())) - err = nil - } - } + var res blobstor.ExistsRes + res, err = s.blobStor.Exists(p) + exists = res.Exists() + } else { + var existsPrm meta.ExistsPrm + existsPrm.SetAddress(prm.addr) + + var res meta.ExistsRes + res, err = s.metaBase.Exists(existsPrm) + exists = res.Exists() } return ExistsRes{ diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index 4699c55fde..80d85df19d 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -63,16 +63,27 @@ func (s *Shard) Head(prm HeadPrm) (HeadRes, error) { // otherwise object seems to be flushed to metabase } - var headParams meta.GetPrm - headParams.SetAddress(prm.addr) - headParams.SetRaw(prm.raw) + var obj *objectSDK.Object + var err error + if s.GetMode().NoMetabase() { + var getPrm GetPrm + getPrm.SetAddress(prm.addr) + getPrm.SetIgnoreMeta(true) - res, err := s.metaBase.Get(headParams) - if err != nil { - return HeadRes{}, err + var res GetRes + res, err = s.Get(getPrm) + obj = res.Object() + } else { + var headParams meta.GetPrm + headParams.SetAddress(prm.addr) + headParams.SetRaw(prm.raw) + + var res meta.GetRes + res, err = s.metaBase.Get(headParams) + obj = res.Header() } return HeadRes{ - obj: res.Header(), - }, nil + obj: obj, + }, err } diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 6811d5f93f..4df9382239 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -6,7 +6,6 @@ import ( "fmt" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" ) @@ -62,8 +61,11 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) { - if s.GetMode() != mode.ReadWrite { + m := s.GetMode() + if m.ReadOnly() { return InhumeRes{}, ErrReadOnlyMode + } else if m.NoMetabase() { + return InhumeRes{}, ErrDegradedMode } if s.hasWriteCache() { diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 4ac240bc4b..128847b4c0 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -110,6 +110,10 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { + if s.GetMode().NoMetabase() { + return ListWithCursorRes{}, ErrDegradedMode + } + var metaPrm meta.ListPrm metaPrm.SetCount(prm.count) metaPrm.SetCursor(prm.cursor) diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index 197a093887..0b8ffeb602 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -3,7 +3,6 @@ package shard import ( "fmt" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) @@ -15,8 +14,11 @@ import ( // // Locked list should be unique. Panics if it is empty. func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { - if s.GetMode() != mode.ReadWrite { + m := s.GetMode() + if m.ReadOnly() { return ErrReadOnlyMode + } else if m.NoMetabase() { + return ErrDegradedMode } err := s.metaBase.Lock(idCnr, locker, locked) diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index b7ede5e72e..d3a1f34f00 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -10,6 +10,9 @@ import ( // that changes shard's memory due to the "read-only" shard's mode. var ErrReadOnlyMode = errors.New("shard is in read-only mode") +// ErrDegradedMode is returned when operation requiring metabase is executed in degraded mode. +var ErrDegradedMode = errors.New("shard is in degraded mode") + // SetMode sets mode of the shard. // // Returns any error encountered that did not allow diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index dffaf5b41b..a0aef6e618 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -3,14 +3,14 @@ package mode // Mode represents enumeration of Shard work modes. type Mode uint32 -const ( - // ReadWrite is a Mode value for shard that is available - // for read and write operations. Default shard mode. - ReadWrite Mode = iota +// ReadWrite is a Mode value for shard that is available +// for read and write operations. Default shard mode. +const ReadWrite Mode = 0 +const ( // ReadOnly is a Mode value for shard that does not // accept write operation but is readable. - ReadOnly + ReadOnly Mode = 1 << iota // Degraded is a Mode value for shard that is set automatically // after a certain number of errors is encountered. It is the same as @@ -31,3 +31,13 @@ func (m Mode) String() string { return "DEGRADED" } } + +// NoMetabase returns true iff m is operating without the metabase. +func (m Mode) NoMetabase() bool { + return m&Degraded != 0 +} + +// ReadOnly returns true iff m prohibits modifying operations with shard. +func (m Mode) ReadOnly() bool { + return m&ReadOnly != 0 +} diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index 5b203d0caa..0068dc7dd5 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -2,7 +2,6 @@ package shard import ( meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" ) @@ -24,8 +23,11 @@ func (p *ToMoveItPrm) SetAddress(addr oid.Address) { // ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to // another shard. func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) { - if s.GetMode() != mode.ReadWrite { + m := s.GetMode() + if m.ReadOnly() { return ToMoveItRes{}, ErrReadOnlyMode + } else if m.NoMetabase() { + return ToMoveItRes{}, ErrDegradedMode } var toMovePrm meta.ToMoveItPrm diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index db59cf2850..87fcfcd256 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -5,7 +5,6 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-sdk-go/object" "go.uber.org/zap" ) @@ -30,7 +29,8 @@ func (p *PutPrm) SetObject(obj *object.Object) { // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. func (s *Shard) Put(prm PutPrm) (PutRes, error) { - if s.GetMode() != mode.ReadWrite { + m := s.GetMode() + if m.ReadOnly() { return PutRes{}, ErrReadOnlyMode } @@ -58,14 +58,15 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err) } - // put to metabase - var pPrm meta.PutPrm - pPrm.SetObject(prm.obj) - pPrm.SetBlobovniczaID(res.BlobovniczaID()) - if _, err := s.metaBase.Put(pPrm); err != nil { - // may we need to handle this case in a special way - // since the object has been successfully written to BlobStor - return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) + if !m.NoMetabase() { + var pPrm meta.PutPrm + pPrm.SetObject(prm.obj) + pPrm.SetBlobovniczaID(res.BlobovniczaID()) + if _, err := s.metaBase.Put(pPrm); err != nil { + // may we need to handle this case in a special way + // since the object has been successfully written to BlobStor + return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) + } } return PutRes{}, nil diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 284561af47..84b7dda97a 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -102,7 +102,8 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) { return obj, nil } - obj, hasMeta, err := s.fetchObjectData(prm.addr, prm.skipMeta, big, small) + skipMeta := prm.skipMeta || s.GetMode().NoMetabase() + obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, big, small) return RngRes{ obj: obj, diff --git a/pkg/local_object_storage/shard/restore.go b/pkg/local_object_storage/shard/restore.go index a996252c59..ae37b159c3 100644 --- a/pkg/local_object_storage/shard/restore.go +++ b/pkg/local_object_storage/shard/restore.go @@ -7,7 +7,6 @@ import ( "io" "os" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-sdk-go/object" ) @@ -62,7 +61,7 @@ func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) { s.m.RLock() defer s.m.RUnlock() - if s.info.Mode != mode.ReadWrite { + if s.info.Mode.ReadOnly() { return RestoreRes{}, ErrReadOnlyMode } diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 58f7f1501a..7d675c2031 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -40,6 +40,10 @@ func (r SelectRes) AddressList() []oid.Address { // Returns any error encountered that // did not allow to completely select the objects. func (s *Shard) Select(prm SelectPrm) (SelectRes, error) { + if s.GetMode().NoMetabase() { + return SelectRes{}, ErrDegradedMode + } + var selectPrm meta.SelectPrm selectPrm.SetFilters(prm.filters) selectPrm.SetContainerID(prm.cnr) diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 2bc2728521..df4ffd85d5 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -4,7 +4,6 @@ import ( "errors" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" - "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id" ) @@ -18,7 +17,7 @@ func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Mo if s.pilorama == nil { return nil, ErrPiloramaDisabled } - if s.GetMode() != mode.ReadWrite { + if s.GetMode().ReadOnly() { return nil, ErrReadOnlyMode } return s.pilorama.TreeMove(d, treeID, m) @@ -29,7 +28,7 @@ func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr stri if s.pilorama == nil { return nil, ErrPiloramaDisabled } - if s.GetMode() != mode.ReadWrite { + if s.GetMode().ReadOnly() { return nil, ErrReadOnlyMode } return s.pilorama.TreeAddByPath(d, treeID, attr, path, meta) @@ -40,7 +39,7 @@ func (s *Shard) TreeApply(d pilorama.CIDDescriptor, treeID string, m *pilorama.M if s.pilorama == nil { return ErrPiloramaDisabled } - if s.GetMode() != mode.ReadWrite { + if s.GetMode().ReadOnly() { return ErrReadOnlyMode } return s.pilorama.TreeApply(d, treeID, m)