diff --git a/pkg/local_object_storage/internal/storagetest/storage.go b/pkg/local_object_storage/internal/storagetest/storage.go new file mode 100644 index 000000000..3f9a01242 --- /dev/null +++ b/pkg/local_object_storage/internal/storagetest/storage.go @@ -0,0 +1,120 @@ +package storagetest + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" +) + +// Component represents single storage component. +type Component interface { + Open(bool) error + SetMode(mode.Mode) error + Init() error + Close() error +} + +// Constructor constructs storage component. +// Each call must create a component using different file-system path. +type Constructor = func(t *testing.T) Component + +// TestAll checks that storage component doesn't panic under +// any circumstances during shard operation. +func TestAll(t *testing.T, cons Constructor) { + modes := []mode.Mode{ + mode.ReadWrite, + mode.ReadOnly, + mode.Degraded, + mode.DegradedReadOnly, + } + + t.Run("close after open", func(t *testing.T) { + TestCloseAfterOpen(t, cons) + }) + t.Run("close twice", func(t *testing.T) { + TestCloseTwice(t, cons) + }) + t.Run("set mode", func(t *testing.T) { + for _, m := range modes { + t.Run(m.String(), func(t *testing.T) { + TestSetMode(t, cons, m) + }) + } + }) + t.Run("mode transition", func(t *testing.T) { + for _, from := range modes { + for _, to := range modes { + TestModeTransition(t, cons, from, to) + } + } + }) +} + +// TestCloseAfterOpen checks that `Close` can be done right after `Open`. +// Use-case: open shard, encounter error, close before the initialization. +func TestCloseAfterOpen(t *testing.T, cons Constructor) { + t.Run("RW", func(t *testing.T) { + // Use-case: irrecoverable error on some components, close everything. + s := cons(t) + require.NoError(t, s.Open(false)) + require.NoError(t, s.Close()) + }) + t.Run("RO", func(t *testing.T) { + // Use-case: irrecoverable error on some components, close everything. + // Open in read-only must be done after the db is here. + s := cons(t) + require.NoError(t, s.Open(false)) + require.NoError(t, s.Init()) + require.NoError(t, s.Close()) + + require.NoError(t, s.Open(true)) + require.NoError(t, s.Close()) + }) +} + +// TestCloseTwice checks that `Close` can be done twice. +func TestCloseTwice(t *testing.T, cons Constructor) { + // Use-case: move to maintenance mode twice, first time failed. + s := cons(t) + require.NoError(t, s.Open(false)) + require.NoError(t, s.Init()) + require.NoError(t, s.Close()) + require.NoError(t, s.Close()) // already closed, no-op +} + +// TestSetMode checks that any mode transition can be done safely. +func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { + t.Run("before init", func(t *testing.T) { + // Use-case: metabase `Init` failed, + // call `SetMode` on all not-yet-initialized components. + s := cons(t) + require.NoError(t, s.Open(false)) + require.NoError(t, s.SetMode(m)) + + t.Run("after open in RO", func(t *testing.T) { + require.NoError(t, s.Close()) + require.NoError(t, s.Open(true)) + require.NoError(t, s.SetMode(m)) + }) + + require.NoError(t, s.Close()) + }) + t.Run("after init", func(t *testing.T) { + s := cons(t) + // Use-case: notmal node operation. + require.NoError(t, s.Open(false)) + require.NoError(t, s.Init()) + require.NoError(t, s.SetMode(m)) + }) +} + +func TestModeTransition(t *testing.T, cons Constructor, from, to mode.Mode) { + // Use-case: normal node operation. + s := cons(t) + require.NoError(t, s.Open(false)) + require.NoError(t, s.Init()) + require.NoError(t, s.SetMode(from)) + require.NoError(t, s.SetMode(to)) + require.NoError(t, s.Close()) +} diff --git a/pkg/local_object_storage/writecache/generic_test.go b/pkg/local_object_storage/writecache/generic_test.go new file mode 100644 index 000000000..4e97b21ea --- /dev/null +++ b/pkg/local_object_storage/writecache/generic_test.go @@ -0,0 +1,29 @@ +package writecache + +import ( + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/storagetest" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func TestGeneric(t *testing.T) { + defer func() { _ = os.RemoveAll(t.Name()) }() + + var n int + newCache := func(t *testing.T) storagetest.Component { + n++ + dir := filepath.Join(t.Name(), strconv.Itoa(n)) + require.NoError(t, os.MkdirAll(dir, os.ModePerm)) + return New( + WithLogger(zaptest.NewLogger(t)), + WithFlushWorkersCount(2), + WithPath(dir)) + } + + storagetest.TestAll(t, newCache) +} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index cc00ca597..7bd7c5834 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -80,7 +80,6 @@ var ( func New(opts ...Option) Cache { c := &cache{ flushCh: make(chan *object.Object), - closeCh: make(chan struct{}), mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), @@ -121,11 +120,13 @@ func (c *cache) Open(readOnly bool) error { return err } - if c.objCounters == nil { - c.objCounters = &counters{ - db: c.db, - fs: c.fsTree, - } + // Opening after Close is done during maintenance mode, + // thus we need to create a channel here. + c.closeCh = make(chan struct{}) + + c.objCounters = &counters{ + db: c.db, + fs: c.fsTree, } return c.objCounters.Read() @@ -145,14 +146,25 @@ func (c *cache) Close() error { return err } - close(c.closeCh) + if c.closeCh != nil { + close(c.closeCh) + } c.wg.Wait() + if c.closeCh != nil { + c.closeCh = nil + } if c.objCounters != nil { c.objCounters.FlushAndClose() + c.objCounters = nil } + + var err error if c.db != nil { - return c.db.Close() + err = c.db.Close() + if err != nil { + c.db = nil + } } return nil }