[#922] storage engine: Prevent any operations after first Close call

Make `BlockExecution` / `ResumeExecution` to not release per-shard worker
pools. Make `StorageEngine.Close` to block these methods and any
data-related operations. It is still releases the pools.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-11-11 16:58:07 +03:00 committed by Alex Vanin
parent d6457ee485
commit 6c0b29e3e3
4 changed files with 73 additions and 9 deletions

View file

@ -42,7 +42,7 @@ func (e *StorageEngine) Init() error {
var errClosed = errors.New("storage engine is closed") var errClosed = errors.New("storage engine is closed")
// Close releases all StorageEngine's components. Waits for all data-related operations to complete. // Close releases all StorageEngine's components. Waits for all data-related operations to complete.
// After the call, all the next ones will fail until the ResumeExecution call. // After the call, all the next ones will fail.
// //
// The method is supposed to be called when the application exits. // The method is supposed to be called when the application exits.
func (e *StorageEngine) Close() error { func (e *StorageEngine) Close() error {
@ -50,13 +50,15 @@ func (e *StorageEngine) Close() error {
} }
// closes all shards. Never returns an error, shard errors are logged. // closes all shards. Never returns an error, shard errors are logged.
func (e *StorageEngine) close() error { func (e *StorageEngine) close(releasePools bool) error {
e.mtx.RLock() e.mtx.RLock()
defer e.mtx.RUnlock() defer e.mtx.RUnlock()
if releasePools {
for _, p := range e.shardPools { for _, p := range e.shardPools {
p.Release() p.Release()
} }
}
for id, sh := range e.shards { for id, sh := range e.shards {
if err := sh.Close(); err != nil { if err := sh.Close(); err != nil {
@ -85,7 +87,8 @@ func (e *StorageEngine) execIfNotBlocked(op func() error) error {
} }
// sets the flag of blocking execution of all data operations according to err: // sets the flag of blocking execution of all data operations according to err:
// * err != nil, then blocks the execution. If exec wasn't blocked, calls close method. // * err != nil, then blocks the execution. If exec wasn't blocked, calls close method
// (if err == errClosed => additionally releases pools and does not allow to resume executions).
// * otherwise, resumes execution. If exec was blocked, calls open method. // * otherwise, resumes execution. If exec was blocked, calls open method.
// //
// Can be called concurrently with exec. In this case it waits for all executions to complete. // Can be called concurrently with exec. In this case it waits for all executions to complete.
@ -95,6 +98,11 @@ func (e *StorageEngine) setBlockExecErr(err error) error {
prevErr := e.blockExec.err prevErr := e.blockExec.err
wasClosed := errors.Is(prevErr, errClosed)
if wasClosed {
return errClosed
}
e.blockExec.err = err e.blockExec.err = err
if err == nil { if err == nil {
@ -102,7 +110,7 @@ func (e *StorageEngine) setBlockExecErr(err error) error {
return e.open() return e.open()
} }
} else if prevErr == nil { // ok -> block } else if prevErr == nil { // ok -> block
return e.close() return e.close(errors.Is(err, errClosed))
} }
// otherwise do nothing // otherwise do nothing
@ -115,7 +123,7 @@ func (e *StorageEngine) setBlockExecErr(err error) error {
// //
// Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources // Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources
// similar to Close. Can be called concurrently with Close and any data related method (waits for all executions // similar to Close. Can be called concurrently with Close and any data related method (waits for all executions
// to complete). // to complete). Returns error if any Close has been called before.
// //
// Must not be called concurrently with either Open or Init. // Must not be called concurrently with either Open or Init.
// //
@ -130,7 +138,7 @@ func (e *StorageEngine) BlockExecution(err error) error {
// //
// Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources // Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources
// similar to Open. Can be called concurrently with Close and any data related method (waits for all executions // similar to Open. Can be called concurrently with Close and any data related method (waits for all executions
// to complete). // to complete). Returns error if any Close has been called before.
// //
// Must not be called concurrently with either Open or Init. // Must not be called concurrently with either Open or Init.
func (e *StorageEngine) ResumeExecution() error { func (e *StorageEngine) ResumeExecution() error {

View file

@ -0,0 +1,46 @@
package engine
import (
"errors"
"testing"
cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test"
"github.com/stretchr/testify/require"
)
func TestExecBlocks(t *testing.T) {
e := testNewEngineWithShardNum(t, 2) // number doesn't matter in this test, 2 is several but not many
defer e.Close()
// put some object
obj := generateRawObjectWithCID(t, cidtest.Generate()).Object()
addr := obj.Address()
require.NoError(t, Put(e, obj))
// block executions
errBlock := errors.New("block exec err")
require.NoError(t, e.BlockExecution(errBlock))
// try to exec some op
_, err := Head(e, addr)
require.ErrorIs(t, err, errBlock)
// resume executions
require.NoError(t, e.ResumeExecution())
_, err = Head(e, addr) // can be any data-related op
require.NoError(t, err)
// close
require.NoError(t, e.Close())
// try exec after close
_, err = Head(e, addr)
require.Error(t, err)
// try to resume
require.Error(t, e.ResumeExecution())
}

View file

@ -116,3 +116,13 @@ func addAttribute(obj *object.RawObject, key, val string) {
attrs = append(attrs, attr) attrs = append(attrs, attr)
obj.SetAttributes(attrs...) obj.SetAttributes(attrs...)
} }
func testNewEngineWithShardNum(t *testing.T, num int) *StorageEngine {
shards := make([]*shard.Shard, 0, num)
for i := 0; i < num; i++ {
shards = append(shards, testNewShard(t, i))
}
return testNewEngineWithShards(shards...)
}

View file

@ -34,7 +34,7 @@ func TestStorageEngine_Inhume(t *testing.T) {
link.SetSplitID(splitID) link.SetSplitID(splitID)
t.Run("delete small object", func(t *testing.T) { t.Run("delete small object", func(t *testing.T) {
e := testNewEngineWithShards(testNewShard(t, 1)) e := testNewEngineWithShardNum(t, 1)
defer e.Close() defer e.Close()
err := Put(e, parent.Object()) err := Put(e, parent.Object())