frostfs-node/pkg/local_object_storage/engine/control.go
Leonard Lyubich ec04e787aa [#922] storage engine: Support operation blocking
There is a need to disable execution of local data operation on storage
engine in runtime. If storage engine ops are blocked, node will act like
always but all local object operations will be denied.

Implement `BlockExecution` / `ResumeExecution` methods on `StorageEngine`
which blocks / resumes the execution of data ops. Wait for the completion of
all operations executed at the time of the call. Return error passed to
`BlockExecution` from all data-related methods until `ResumeExecution` call.
Make `Close` to block operations as well.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
2021-11-12 17:28:38 +03:00

138 lines
3.6 KiB
Go

package engine
import (
"errors"
"fmt"
"go.uber.org/zap"
)
// Open opens all StorageEngine's components.
func (e *StorageEngine) Open() error {
return e.open()
}
func (e *StorageEngine) open() error {
e.mtx.RLock()
defer e.mtx.RUnlock()
for id, sh := range e.shards {
if err := sh.Open(); err != nil {
return fmt.Errorf("could not open shard %s: %w", id, err)
}
}
return nil
}
// Init initializes all StorageEngine's components.
func (e *StorageEngine) Init() error {
e.mtx.RLock()
defer e.mtx.RUnlock()
for id, sh := range e.shards {
if err := sh.Init(); err != nil {
return fmt.Errorf("could not initialize shard %s: %w", id, err)
}
}
return nil
}
var errClosed = errors.New("storage engine is closed")
// Close releases all StorageEngine's components. Waits for all data-related operations to complete.
// After the call, all the next ones will fail until the ResumeExecution call.
//
// Еhe method is supposed to be called when the application exits.
func (e *StorageEngine) Close() error {
return e.setBlockExecErr(errClosed)
}
// closes all shards. Never returns an error, shard errors are logged.
func (e *StorageEngine) close() error {
e.mtx.RLock()
defer e.mtx.RUnlock()
for _, p := range e.shardPools {
p.Release()
}
for id, sh := range e.shards {
if err := sh.Close(); err != nil {
e.log.Debug("could not close shard",
zap.String("id", id),
zap.String("error", err.Error()),
)
}
}
return nil
}
// executes op if execution is not blocked, otherwise returns blocking error.
//
// Can be called concurrently with setBlockExecErr.
func (e *StorageEngine) exec(op func() error) error {
e.blockExec.mtx.RLock()
defer e.blockExec.mtx.RUnlock()
if e.blockExec.err != nil {
return e.blockExec.err
}
return op()
}
// sets the flag of blocking execution of all data operations according to err:
// * err != nil, then blocks the execution. If exec wasn't blocked, calls close method.
// * otherwise, resumes execution. If exec was blocked, calls open method.
//
// Can be called concurrently with exec. In this case it waits for all executions to complete.
func (e *StorageEngine) setBlockExecErr(err error) error {
e.blockExec.mtx.Lock()
defer e.blockExec.mtx.Unlock()
prevErr := e.blockExec.err
e.blockExec.err = err
if err == nil {
if prevErr != nil { // block -> ok
return e.open()
}
} else if prevErr == nil { // ok -> block
return e.close()
}
// otherwise do nothing
return nil
}
// BlockExecution block blocks the execution of any data-related operation. All blocked ops will return err.
// To resume the execution, use ResumeExecution method.
//
// Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources
// similar to Close. Can be called concurrently with Close and any data related method (waits for all executions
// to complete).
//
// Must not be called concurrently with either Open or Init.
//
// Note: technically passing nil error will resume the execution, otherwise, it is recommended to call ResumeExecution
// for this.
func (e *StorageEngine) BlockExecution(err error) error {
return e.setBlockExecErr(err)
}
// ResumeExecution resumes the execution of any data-related operation.
// To block the execution, use BlockExecution method.
//
// Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources
// similar to Open. Can be called concurrently with Close and any data related method (waits for all executions
// to complete).
//
// Must not be called concurrently with either Open or Init.
func (e *StorageEngine) ResumeExecution() error {
return e.setBlockExecErr(nil)
}