forked from TrueCloudLab/frostfs-node
[#139] test: Add test storage implementation
This aims to reduce the usage of chmod hackery to induce or simulate OS-related failures. Signed-off-by: Alejandro Lopez <a.lopez@yadro.com>
This commit is contained in:
Gitea
parent
e843e7f090
commit
341fe1688f
20 changed files with 617 additions and 208 deletions
|
|
@ -3,14 +3,17 @@ package engine
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
|
||||
|
|
@ -25,92 +28,128 @@ import (
|
|||
)
|
||||
|
||||
// TestInitializationFailure checks that shard is initialized and closed even if media
|
||||
// under any single component is absent. We emulate this with permission denied error.
|
||||
// under any single component is absent.
|
||||
func TestInitializationFailure(t *testing.T) {
|
||||
type paths struct {
|
||||
blobstor string
|
||||
metabase string
|
||||
writecache string
|
||||
pilorama string
|
||||
type openFileFunc func(string, int, fs.FileMode) (*os.File, error)
|
||||
|
||||
type testShardOpts struct {
|
||||
openFileMetabase openFileFunc
|
||||
openFileWriteCache openFileFunc
|
||||
openFilePilorama openFileFunc
|
||||
}
|
||||
|
||||
existsDir := filepath.Join(t.TempDir(), "shard")
|
||||
badDir := filepath.Join(t.TempDir(), "missing")
|
||||
|
||||
testShard := func(c paths) []shard.Option {
|
||||
testShard := func(opts testShardOpts) ([]shard.Option, *teststore.TestStore, *teststore.TestStore) {
|
||||
sid, err := generateShardID()
|
||||
require.NoError(t, err)
|
||||
|
||||
tempDir := t.TempDir()
|
||||
blobstorPath := filepath.Join(tempDir, "bs")
|
||||
metabasePath := filepath.Join(tempDir, "mb")
|
||||
writecachePath := filepath.Join(tempDir, "wc")
|
||||
piloramaPath := filepath.Join(tempDir, "pl")
|
||||
|
||||
storages, smallFileStorage, largeFileStorage := newTestStorages(blobstorPath, 1<<20)
|
||||
|
||||
return []shard.Option{
|
||||
shard.WithID(sid),
|
||||
shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
|
||||
shard.WithBlobStorOptions(
|
||||
blobstor.WithStorages(
|
||||
newStorages(c.blobstor, 1<<20))),
|
||||
blobstor.WithStorages(storages)),
|
||||
shard.WithMetaBaseOptions(
|
||||
meta.WithBoltDBOptions(&bbolt.Options{
|
||||
Timeout: 100 * time.Millisecond,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
OpenFile: opts.openFileMetabase,
|
||||
}),
|
||||
meta.WithPath(c.metabase),
|
||||
meta.WithPath(metabasePath),
|
||||
meta.WithPermissions(0700),
|
||||
meta.WithEpochState(epochState{})),
|
||||
shard.WithWriteCache(true),
|
||||
shard.WithWriteCacheOptions(writecache.WithPath(c.writecache)),
|
||||
shard.WithPiloramaOptions(pilorama.WithPath(c.pilorama)),
|
||||
}
|
||||
shard.WithWriteCacheOptions(
|
||||
writecache.WithPath(writecachePath),
|
||||
writecache.WithOpenFile(opts.openFileWriteCache),
|
||||
),
|
||||
shard.WithPiloramaOptions(
|
||||
pilorama.WithPath(piloramaPath),
|
||||
pilorama.WithOpenFile(opts.openFilePilorama),
|
||||
),
|
||||
}, smallFileStorage, largeFileStorage
|
||||
}
|
||||
|
||||
t.Run("blobstor", func(t *testing.T) {
|
||||
badDir := filepath.Join(badDir, t.Name())
|
||||
require.NoError(t, os.MkdirAll(badDir, os.ModePerm))
|
||||
require.NoError(t, os.Chmod(badDir, 0))
|
||||
testEngineFailInitAndReload(t, badDir, false, testShard(paths{
|
||||
blobstor: filepath.Join(badDir, "0"),
|
||||
metabase: filepath.Join(existsDir, t.Name(), "1"),
|
||||
writecache: filepath.Join(existsDir, t.Name(), "2"),
|
||||
pilorama: filepath.Join(existsDir, t.Name(), "3"),
|
||||
shardOpts, _, largeFileStorage := testShard(testShardOpts{
|
||||
openFileMetabase: os.OpenFile,
|
||||
openFileWriteCache: os.OpenFile,
|
||||
openFilePilorama: os.OpenFile,
|
||||
})
|
||||
largeFileStorage.SetOption(teststore.WithOpen(func(ro bool) error {
|
||||
return teststore.ErrDiskExploded
|
||||
}))
|
||||
beforeReload := func() {
|
||||
largeFileStorage.SetOption(teststore.WithOpen(nil))
|
||||
}
|
||||
testEngineFailInitAndReload(t, false, shardOpts, beforeReload)
|
||||
})
|
||||
t.Run("metabase", func(t *testing.T) {
|
||||
badDir := filepath.Join(badDir, t.Name())
|
||||
require.NoError(t, os.MkdirAll(badDir, os.ModePerm))
|
||||
require.NoError(t, os.Chmod(badDir, 0))
|
||||
testEngineFailInitAndReload(t, badDir, true, testShard(paths{
|
||||
blobstor: filepath.Join(existsDir, t.Name(), "0"),
|
||||
metabase: filepath.Join(badDir, "1"),
|
||||
writecache: filepath.Join(existsDir, t.Name(), "2"),
|
||||
pilorama: filepath.Join(existsDir, t.Name(), "3"),
|
||||
}))
|
||||
var openFileMetabaseSucceed atomic.Bool
|
||||
openFileMetabase := func(p string, f int, mode fs.FileMode) (*os.File, error) {
|
||||
if openFileMetabaseSucceed.Load() {
|
||||
return os.OpenFile(p, f, mode)
|
||||
}
|
||||
return nil, teststore.ErrDiskExploded
|
||||
}
|
||||
beforeReload := func() {
|
||||
openFileMetabaseSucceed.Store(true)
|
||||
}
|
||||
shardOpts, _, _ := testShard(testShardOpts{
|
||||
openFileMetabase: openFileMetabase,
|
||||
openFileWriteCache: os.OpenFile,
|
||||
openFilePilorama: os.OpenFile,
|
||||
})
|
||||
testEngineFailInitAndReload(t, true, shardOpts, beforeReload)
|
||||
})
|
||||
t.Run("write-cache", func(t *testing.T) {
|
||||
badDir := filepath.Join(badDir, t.Name())
|
||||
require.NoError(t, os.MkdirAll(badDir, os.ModePerm))
|
||||
require.NoError(t, os.Chmod(badDir, 0))
|
||||
testEngineFailInitAndReload(t, badDir, false, testShard(paths{
|
||||
blobstor: filepath.Join(existsDir, t.Name(), "0"),
|
||||
metabase: filepath.Join(existsDir, t.Name(), "1"),
|
||||
writecache: filepath.Join(badDir, "2"),
|
||||
pilorama: filepath.Join(existsDir, t.Name(), "3"),
|
||||
}))
|
||||
var openFileWriteCacheSucceed atomic.Bool
|
||||
openFileWriteCache := func(p string, f int, mode fs.FileMode) (*os.File, error) {
|
||||
if openFileWriteCacheSucceed.Load() {
|
||||
return os.OpenFile(p, f, mode)
|
||||
}
|
||||
return nil, teststore.ErrDiskExploded
|
||||
}
|
||||
beforeReload := func() {
|
||||
openFileWriteCacheSucceed.Store(true)
|
||||
}
|
||||
shardOpts, _, _ := testShard(testShardOpts{
|
||||
openFileMetabase: os.OpenFile,
|
||||
openFileWriteCache: openFileWriteCache,
|
||||
openFilePilorama: os.OpenFile,
|
||||
})
|
||||
testEngineFailInitAndReload(t, false, shardOpts, beforeReload)
|
||||
})
|
||||
t.Run("pilorama", func(t *testing.T) {
|
||||
badDir := filepath.Join(badDir, t.Name())
|
||||
require.NoError(t, os.MkdirAll(badDir, os.ModePerm))
|
||||
require.NoError(t, os.Chmod(badDir, 0))
|
||||
testEngineFailInitAndReload(t, badDir, false, testShard(paths{
|
||||
blobstor: filepath.Join(existsDir, t.Name(), "0"),
|
||||
metabase: filepath.Join(existsDir, t.Name(), "1"),
|
||||
writecache: filepath.Join(existsDir, t.Name(), "2"),
|
||||
pilorama: filepath.Join(badDir, "3"),
|
||||
}))
|
||||
var openFilePiloramaSucceed atomic.Bool
|
||||
openFilePilorama := func(p string, f int, mode fs.FileMode) (*os.File, error) {
|
||||
if openFilePiloramaSucceed.Load() {
|
||||
return os.OpenFile(p, f, mode)
|
||||
}
|
||||
return nil, teststore.ErrDiskExploded
|
||||
}
|
||||
beforeReload := func() {
|
||||
openFilePiloramaSucceed.Store(true)
|
||||
}
|
||||
shardOpts, _, _ := testShard(testShardOpts{
|
||||
openFileMetabase: os.OpenFile,
|
||||
openFileWriteCache: os.OpenFile,
|
||||
openFilePilorama: openFilePilorama,
|
||||
})
|
||||
testEngineFailInitAndReload(t, false, shardOpts, beforeReload)
|
||||
})
|
||||
}
|
||||
|
||||
func testEngineFailInitAndReload(t *testing.T, badDir string, errOnAdd bool, s []shard.Option) {
|
||||
func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Option, beforeReload func()) {
|
||||
var configID string
|
||||
|
||||
e := New()
|
||||
_, err := e.AddShard(s...)
|
||||
_, err := e.AddShard(opts...)
|
||||
if errOnAdd {
|
||||
require.Error(t, err)
|
||||
// This branch is only taken when we cannot update shard ID in the metabase.
|
||||
|
|
@ -139,9 +178,10 @@ func testEngineFailInitAndReload(t *testing.T, badDir string, errOnAdd bool, s [
|
|||
e.mtx.RUnlock()
|
||||
require.Equal(t, 0, shardCount)
|
||||
|
||||
require.NoError(t, os.Chmod(badDir, os.ModePerm))
|
||||
beforeReload()
|
||||
|
||||
require.NoError(t, e.Reload(ReConfiguration{
|
||||
shards: map[string][]shard.Option{configID: s},
|
||||
shards: map[string][]shard.Option{configID: opts},
|
||||
}))
|
||||
|
||||
e.mtx.RLock()
|
||||
|
|
@ -193,26 +233,28 @@ func TestPersistentShardID(t *testing.T) {
|
|||
dir, err := os.MkdirTemp("", "*")
|
||||
require.NoError(t, err)
|
||||
|
||||
e, _, id := newEngineWithErrorThreshold(t, dir, 1)
|
||||
te := newEngineWithErrorThreshold(t, dir, 1)
|
||||
|
||||
checkShardState(t, e, id[0], 0, mode.ReadWrite)
|
||||
require.NoError(t, e.Close())
|
||||
checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite)
|
||||
require.NoError(t, te.ng.Close())
|
||||
|
||||
e, _, newID := newEngineWithErrorThreshold(t, dir, 1)
|
||||
require.Equal(t, id, newID)
|
||||
require.NoError(t, e.Close())
|
||||
newTe := newEngineWithErrorThreshold(t, dir, 1)
|
||||
for i := 0; i < len(newTe.shards); i++ {
|
||||
require.Equal(t, te.shards[i].id, newTe.shards[i].id)
|
||||
}
|
||||
require.NoError(t, newTe.ng.Close())
|
||||
|
||||
p1 := e.shards[id[0].String()].Shard.DumpInfo().MetaBaseInfo.Path
|
||||
p2 := e.shards[id[1].String()].Shard.DumpInfo().MetaBaseInfo.Path
|
||||
p1 := newTe.ng.shards[te.shards[0].id.String()].Shard.DumpInfo().MetaBaseInfo.Path
|
||||
p2 := newTe.ng.shards[te.shards[1].id.String()].Shard.DumpInfo().MetaBaseInfo.Path
|
||||
tmp := filepath.Join(dir, "tmp")
|
||||
require.NoError(t, os.Rename(p1, tmp))
|
||||
require.NoError(t, os.Rename(p2, p1))
|
||||
require.NoError(t, os.Rename(tmp, p2))
|
||||
|
||||
e, _, newID = newEngineWithErrorThreshold(t, dir, 1)
|
||||
require.Equal(t, id[1], newID[0])
|
||||
require.Equal(t, id[0], newID[1])
|
||||
require.NoError(t, e.Close())
|
||||
newTe = newEngineWithErrorThreshold(t, dir, 1)
|
||||
require.Equal(t, te.shards[1].id, newTe.shards[0].id)
|
||||
require.Equal(t, te.shards[0].id, newTe.shards[1].id)
|
||||
require.NoError(t, newTe.ng.Close())
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue