[#922] storage engine: Support operation blocking

There is a need to disable execution of local data operation on storage
engine in runtime. If storage engine ops are blocked, node will act like
always but all local object operations will be denied.

Implement `BlockExecution` / `ResumeExecution` methods on `StorageEngine`
which blocks / resumes the execution of data ops. Wait for the completion of
all operations executed at the time of the call. Return error passed to
`BlockExecution` from all data-related methods until `ResumeExecution` call.
Make `Close` to block operations as well.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-11-09 18:46:12 +03:00 committed by Alex Vanin
parent cea1de3a27
commit ec04e787aa
10 changed files with 237 additions and 29 deletions

View file

@ -37,14 +37,24 @@ func (r *ListContainersRes) Containers() []*cid.ID {
} }
// ContainerSize returns sum of estimation container sizes among all shards. // ContainerSize returns sum of estimation container sizes among all shards.
func (e *StorageEngine) ContainerSize(prm *ContainerSizePrm) *ContainerSizeRes { //
if e.metrics != nil { // Returns empty result if executions are blocked (see BlockExecution).
defer elapsed(e.metrics.AddEstimateContainerSizeDuration)() func (e *StorageEngine) ContainerSize(prm *ContainerSizePrm) (res *ContainerSizeRes) {
err := e.exec(func() error {
res = e.containerSize(prm)
return nil
})
if err != nil {
e.log.Debug("container size exec failure",
zap.String("err", err.Error()),
)
} }
return &ContainerSizeRes{ if res == nil {
size: e.containerSize(prm.cid), res = new(ContainerSizeRes)
} }
return
} }
// ContainerSize returns sum of estimation container sizes among all shards. // ContainerSize returns sum of estimation container sizes among all shards.
@ -52,35 +62,51 @@ func ContainerSize(e *StorageEngine, id *cid.ID) uint64 {
return e.ContainerSize(&ContainerSizePrm{cid: id}).Size() return e.ContainerSize(&ContainerSizePrm{cid: id}).Size()
} }
func (e *StorageEngine) containerSize(id *cid.ID) (total uint64) { func (e *StorageEngine) containerSize(prm *ContainerSizePrm) *ContainerSizeRes {
if e.metrics != nil {
defer elapsed(e.metrics.AddEstimateContainerSizeDuration)()
}
var res ContainerSizeRes
e.iterateOverUnsortedShards(func(s *shard.Shard) (stop bool) { e.iterateOverUnsortedShards(func(s *shard.Shard) (stop bool) {
size, err := shard.ContainerSize(s, id) size, err := shard.ContainerSize(s, prm.cid)
if err != nil { if err != nil {
e.log.Warn("can't get container size", e.log.Warn("can't get container size",
zap.Stringer("shard_id", s.ID()), zap.Stringer("shard_id", s.ID()),
zap.Stringer("container_id", id), zap.Stringer("container_id", prm.cid),
zap.String("error", err.Error())) zap.String("error", err.Error()))
return false return false
} }
total += size res.size += size
return false return false
}) })
return total return &res
} }
// ListContainers returns unique container IDs presented in the engine objects. // ListContainers returns unique container IDs presented in the engine objects.
func (e *StorageEngine) ListContainers(_ *ListContainersPrm) *ListContainersRes { //
if e.metrics != nil { // Returns empty result if executions are blocked (see BlockExecution).
defer elapsed(e.metrics.AddListContainersDuration)() func (e *StorageEngine) ListContainers(_ *ListContainersPrm) (res *ListContainersRes) {
err := e.exec(func() error {
res = e.listContainers()
return nil
})
if err != nil {
e.log.Debug("list containers exec failure",
zap.String("err", err.Error()),
)
} }
return &ListContainersRes{ if res == nil {
containers: e.listContainers(), res = new(ListContainersRes)
} }
return
} }
// ListContainers returns unique container IDs presented in the engine objects. // ListContainers returns unique container IDs presented in the engine objects.
@ -88,7 +114,11 @@ func ListContainers(e *StorageEngine) []*cid.ID {
return e.ListContainers(&ListContainersPrm{}).Containers() return e.ListContainers(&ListContainersPrm{}).Containers()
} }
func (e *StorageEngine) listContainers() []*cid.ID { func (e *StorageEngine) listContainers() *ListContainersRes {
if e.metrics != nil {
defer elapsed(e.metrics.AddListContainersDuration)()
}
uniqueIDs := make(map[string]*cid.ID) uniqueIDs := make(map[string]*cid.ID)
e.iterateOverUnsortedShards(func(s *shard.Shard) (stop bool) { e.iterateOverUnsortedShards(func(s *shard.Shard) (stop bool) {
@ -116,5 +146,7 @@ func (e *StorageEngine) listContainers() []*cid.ID {
result = append(result, v) result = append(result, v)
} }
return result return &ListContainersRes{
containers: result,
}
} }

View file

@ -1,6 +1,7 @@
package engine package engine
import ( import (
"errors"
"fmt" "fmt"
"go.uber.org/zap" "go.uber.org/zap"
@ -8,6 +9,10 @@ import (
// Open opens all StorageEngine's components. // Open opens all StorageEngine's components.
func (e *StorageEngine) Open() error { func (e *StorageEngine) Open() error {
return e.open()
}
func (e *StorageEngine) open() error {
e.mtx.RLock() e.mtx.RLock()
defer e.mtx.RUnlock() defer e.mtx.RUnlock()
@ -34,8 +39,18 @@ func (e *StorageEngine) Init() error {
return nil return nil
} }
// Close releases all StorageEngine's components. var errClosed = errors.New("storage engine is closed")
// Close releases all StorageEngine's components. Waits for all data-related operations to complete.
// After the call, all the next ones will fail until the ResumeExecution call.
//
// Еhe method is supposed to be called when the application exits.
func (e *StorageEngine) Close() error { func (e *StorageEngine) Close() error {
return e.setBlockExecErr(errClosed)
}
// closes all shards. Never returns an error, shard errors are logged.
func (e *StorageEngine) close() error {
e.mtx.RLock() e.mtx.RLock()
defer e.mtx.RUnlock() defer e.mtx.RUnlock()
@ -54,3 +69,70 @@ func (e *StorageEngine) Close() error {
return nil return nil
} }
// executes op if execution is not blocked, otherwise returns blocking error.
//
// Can be called concurrently with setBlockExecErr.
func (e *StorageEngine) exec(op func() error) error {
e.blockExec.mtx.RLock()
defer e.blockExec.mtx.RUnlock()
if e.blockExec.err != nil {
return e.blockExec.err
}
return op()
}
// sets the flag of blocking execution of all data operations according to err:
// * err != nil, then blocks the execution. If exec wasn't blocked, calls close method.
// * otherwise, resumes execution. If exec was blocked, calls open method.
//
// Can be called concurrently with exec. In this case it waits for all executions to complete.
func (e *StorageEngine) setBlockExecErr(err error) error {
e.blockExec.mtx.Lock()
defer e.blockExec.mtx.Unlock()
prevErr := e.blockExec.err
e.blockExec.err = err
if err == nil {
if prevErr != nil { // block -> ok
return e.open()
}
} else if prevErr == nil { // ok -> block
return e.close()
}
// otherwise do nothing
return nil
}
// BlockExecution block blocks the execution of any data-related operation. All blocked ops will return err.
// To resume the execution, use ResumeExecution method.
//
// Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources
// similar to Close. Can be called concurrently with Close and any data related method (waits for all executions
// to complete).
//
// Must not be called concurrently with either Open or Init.
//
// Note: technically passing nil error will resume the execution, otherwise, it is recommended to call ResumeExecution
// for this.
func (e *StorageEngine) BlockExecution(err error) error {
return e.setBlockExecErr(err)
}
// ResumeExecution resumes the execution of any data-related operation.
// To block the execution, use BlockExecution method.
//
// Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources
// similar to Open. Can be called concurrently with Close and any data related method (waits for all executions
// to complete).
//
// Must not be called concurrently with either Open or Init.
func (e *StorageEngine) ResumeExecution() error {
return e.setBlockExecErr(nil)
}

View file

@ -26,7 +26,18 @@ func (p *DeletePrm) WithAddresses(addr ...*objectSDK.Address) *DeletePrm {
} }
// Delete marks the objects to be removed. // Delete marks the objects to be removed.
func (e *StorageEngine) Delete(prm *DeletePrm) (*DeleteRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) Delete(prm *DeletePrm) (res *DeleteRes, err error) {
err = e.exec(func() error {
res, err = e.delete(prm)
return err
})
return
}
func (e *StorageEngine) delete(prm *DeletePrm) (*DeleteRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddDeleteDuration)() defer elapsed(e.metrics.AddDeleteDuration)()
} }

View file

@ -18,6 +18,12 @@ type StorageEngine struct {
shards map[string]*shard.Shard shards map[string]*shard.Shard
shardPools map[string]util.WorkerPool shardPools map[string]util.WorkerPool
blockExec struct {
mtx sync.RWMutex
err error
}
} }
// Option represents StorageEngine's constructor option. // Option represents StorageEngine's constructor option.

View file

@ -42,7 +42,18 @@ func (r *GetRes) Object() *object.Object {
// did not allow to completely read the object part. // did not allow to completely read the object part.
// //
// Returns ErrNotFound if requested object is missing in local storage. // Returns ErrNotFound if requested object is missing in local storage.
func (e *StorageEngine) Get(prm *GetPrm) (*GetRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) Get(prm *GetPrm) (res *GetRes, err error) {
err = e.exec(func() error {
res, err = e.get(prm)
return err
})
return
}
func (e *StorageEngine) get(prm *GetPrm) (*GetRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddGetDuration)() defer elapsed(e.metrics.AddGetDuration)()
} }

View file

@ -57,7 +57,18 @@ func (r *HeadRes) Header() *object.Object {
// //
// Returns object.ErrNotFound if requested object is missing in local storage. // Returns object.ErrNotFound if requested object is missing in local storage.
// Returns object.ErrAlreadyRemoved if requested object was inhumed. // Returns object.ErrAlreadyRemoved if requested object was inhumed.
func (e *StorageEngine) Head(prm *HeadPrm) (*HeadRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) Head(prm *HeadPrm) (res *HeadRes, err error) {
err = e.exec(func() error {
res, err = e.head(prm)
return err
})
return
}
func (e *StorageEngine) head(prm *HeadPrm) (*HeadRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddHeadDuration)() defer elapsed(e.metrics.AddHeadDuration)()
} }

View file

@ -49,7 +49,18 @@ var errInhumeFailure = errors.New("inhume operation failed")
// Inhume calls metabase. Inhume method to mark object as removed. It won't be // Inhume calls metabase. Inhume method to mark object as removed. It won't be
// removed physically from shard until `Delete` operation. // removed physically from shard until `Delete` operation.
func (e *StorageEngine) Inhume(prm *InhumePrm) (*InhumeRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) Inhume(prm *InhumePrm) (res *InhumeRes, err error) {
err = e.exec(func() error {
res, err = e.inhume(prm)
return err
})
return
}
func (e *StorageEngine) inhume(prm *InhumePrm) (*InhumeRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddInhumeDuration)() defer elapsed(e.metrics.AddInhumeDuration)()
} }
@ -63,9 +74,9 @@ func (e *StorageEngine) Inhume(prm *InhumePrm) (*InhumeRes, error) {
shPrm.MarkAsGarbage(prm.addrs[i]) shPrm.MarkAsGarbage(prm.addrs[i])
} }
ok := e.inhume(prm.addrs[i], shPrm, true) ok := e.inhumeAddr(prm.addrs[i], shPrm, true)
if !ok { if !ok {
ok = e.inhume(prm.addrs[i], shPrm, false) ok = e.inhumeAddr(prm.addrs[i], shPrm, false)
if !ok { if !ok {
return nil, errInhumeFailure return nil, errInhumeFailure
} }
@ -75,7 +86,7 @@ func (e *StorageEngine) Inhume(prm *InhumePrm) (*InhumeRes, error) {
return new(InhumeRes), nil return new(InhumeRes), nil
} }
func (e *StorageEngine) inhume(addr *objectSDK.Address, prm *shard.InhumePrm, checkExists bool) (ok bool) { func (e *StorageEngine) inhumeAddr(addr *objectSDK.Address, prm *shard.InhumePrm, checkExists bool) (ok bool) {
root := false root := false
e.iterateOverSortedShards(addr, func(_ int, sh *shard.Shard) (stop bool) { e.iterateOverSortedShards(addr, func(_ int, sh *shard.Shard) (stop bool) {

View file

@ -33,7 +33,18 @@ func (p *PutPrm) WithObject(obj *object.Object) *PutPrm {
// //
// Returns any error encountered that // Returns any error encountered that
// did not allow to completely save the object. // did not allow to completely save the object.
func (e *StorageEngine) Put(prm *PutPrm) (*PutRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) Put(prm *PutPrm) (res *PutRes, err error) {
err = e.exec(func() error {
res, err = e.put(prm)
return err
})
return
}
func (e *StorageEngine) put(prm *PutPrm) (*PutRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddPutDuration)() defer elapsed(e.metrics.AddPutDuration)()
} }

View file

@ -60,7 +60,18 @@ func (r *RngRes) Object() *object.Object {
// Returns ErrNotFound if requested object is missing in local storage. // Returns ErrNotFound if requested object is missing in local storage.
// Returns ErrAlreadyRemoved if requested object is inhumed. // Returns ErrAlreadyRemoved if requested object is inhumed.
// Returns ErrRangeOutOfBounds if requested object range is out of bounds. // Returns ErrRangeOutOfBounds if requested object range is out of bounds.
func (e *StorageEngine) GetRange(prm *RngPrm) (*RngRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) GetRange(prm *RngPrm) (res *RngRes, err error) {
err = e.exec(func() error {
res, err = e.getRange(prm)
return err
})
return
}
func (e *StorageEngine) getRange(prm *RngPrm) (*RngRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddRangeDuration)() defer elapsed(e.metrics.AddRangeDuration)()
} }

View file

@ -47,7 +47,18 @@ func (r *SelectRes) AddressList() []*object.Address {
// Select selects the objects from local storage that match select parameters. // Select selects the objects from local storage that match select parameters.
// //
// Returns any error encountered that did not allow to completely select the objects. // Returns any error encountered that did not allow to completely select the objects.
func (e *StorageEngine) Select(prm *SelectPrm) (*SelectRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) Select(prm *SelectPrm) (res *SelectRes, err error) {
err = e.exec(func() error {
res, err = e._select(prm)
return err
})
return
}
func (e *StorageEngine) _select(prm *SelectPrm) (*SelectRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddSearchDuration)() defer elapsed(e.metrics.AddSearchDuration)()
} }
@ -97,7 +108,18 @@ func (e *StorageEngine) Select(prm *SelectPrm) (*SelectRes, error) {
// List returns `limit` available physically storage object addresses in engine. // List returns `limit` available physically storage object addresses in engine.
// If limit is zero, then returns all available object addresses. // If limit is zero, then returns all available object addresses.
func (e *StorageEngine) List(limit uint64) (*SelectRes, error) { //
// Returns an error if executions are blocked (see BlockExecution).
func (e *StorageEngine) List(limit uint64) (res *SelectRes, err error) {
err = e.exec(func() error {
res, err = e.list(limit)
return err
})
return
}
func (e *StorageEngine) list(limit uint64) (*SelectRes, error) {
if e.metrics != nil { if e.metrics != nil {
defer elapsed(e.metrics.AddListObjectsDuration)() defer elapsed(e.metrics.AddListObjectsDuration)()
} }