frostfs-node/pkg/local_object_storage/shard/dump_test.go

413 lines
12 KiB
Go
Raw Normal View History

package shard_test
import (
"bytes"
"context"
"io"
"math/rand"
"os"
"path/filepath"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
func TestDump(t *testing.T) {
t.Run("without write-cache", func(t *testing.T) {
testDump(t, 10, false)
})
t.Run("with write-cache", func(t *testing.T) {
// Put a bit more objects to write-cache to facilitate race-conditions.
testDump(t, 100, true)
})
}
func testDump(t *testing.T, objCount int, hasWriteCache bool) {
const (
wcSmallObjectSize = 1024 // 1 KiB, goes to write-cache memory
wcBigObjectSize = 4 * 1024 // 4 KiB, goes to write-cache FSTree
bsSmallObjectSize = 10 * 1024 // 10 KiB, goes to blobovnicza DB
bsBigObjectSize = 1024*1024 + 1 // > 1 MiB, goes to blobovnicza FSTree
)
var sh *shard.Shard
if !hasWriteCache {
sh = newShard(t, false)
} else {
sh = newCustomShard(t, t.TempDir(), true,
[]writecache.Option{
writecache.WithSmallObjectSize(wcSmallObjectSize),
writecache.WithMaxObjectSize(wcBigObjectSize),
writecache.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
},
nil)
}
defer releaseShard(sh, t)
out := filepath.Join(t.TempDir(), "dump")
var prm shard.DumpPrm
prm.WithPath(out)
t.Run("must be read-only", func(t *testing.T) {
_, err := sh.Dump(prm)
require.ErrorIs(t, err, shard.ErrMustBeReadOnly)
})
require.NoError(t, sh.SetMode(mode.ReadOnly))
outEmpty := out + ".empty"
var dumpPrm shard.DumpPrm
dumpPrm.WithPath(outEmpty)
res, err := sh.Dump(dumpPrm)
require.NoError(t, err)
require.Equal(t, 0, res.Count())
require.NoError(t, sh.SetMode(mode.ReadWrite))
// Approximate object header size.
const headerSize = 400
objects := make([]*objectSDK.Object, objCount)
for i := 0; i < objCount; i++ {
cnr := cidtest.ID()
var size int
switch i % 6 {
case 0, 1:
size = wcSmallObjectSize - headerSize
case 2, 3:
size = bsSmallObjectSize - headerSize
case 4:
size = wcBigObjectSize - headerSize
default:
size = bsBigObjectSize - headerSize
}
data := make([]byte, size)
rand.Read(data)
obj := testutil.GenerateObjectWithCIDWithPayload(cnr, data)
objects[i] = obj
var prm shard.PutPrm
prm.SetObject(objects[i])
_, err := sh.Put(context.Background(), prm)
require.NoError(t, err)
}
require.NoError(t, sh.SetMode(mode.ReadOnly))
t.Run("invalid path", func(t *testing.T) {
var dumpPrm shard.DumpPrm
dumpPrm.WithPath("\x00")
_, err := sh.Dump(dumpPrm)
require.Error(t, err)
})
res, err = sh.Dump(prm)
require.NoError(t, err)
require.Equal(t, objCount, res.Count())
t.Run("restore", func(t *testing.T) {
sh := newShard(t, false)
defer releaseShard(sh, t)
t.Run("empty dump", func(t *testing.T) {
var restorePrm shard.RestorePrm
restorePrm.WithPath(outEmpty)
res, err := sh.Restore(context.Background(), restorePrm)
require.NoError(t, err)
require.Equal(t, 0, res.Count())
})
t.Run("invalid path", func(t *testing.T) {
_, err := sh.Restore(context.Background(), *new(shard.RestorePrm))
require.ErrorIs(t, err, os.ErrNotExist)
})
t.Run("invalid file", func(t *testing.T) {
t.Run("invalid magic", func(t *testing.T) {
out := out + ".wrongmagic"
require.NoError(t, os.WriteFile(out, []byte{0, 0, 0, 0}, os.ModePerm))
var restorePrm shard.RestorePrm
restorePrm.WithPath(out)
_, err := sh.Restore(context.Background(), restorePrm)
require.ErrorIs(t, err, shard.ErrInvalidMagic)
})
fileData, err := os.ReadFile(out)
require.NoError(t, err)
t.Run("incomplete size", func(t *testing.T) {
out := out + ".wrongsize"
fileData := append(fileData, 1)
require.NoError(t, os.WriteFile(out, fileData, os.ModePerm))
var restorePrm shard.RestorePrm
restorePrm.WithPath(out)
_, err := sh.Restore(context.Background(), restorePrm)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
})
t.Run("incomplete object data", func(t *testing.T) {
out := out + ".wrongsize"
fileData := append(fileData, 1, 0, 0, 0)
require.NoError(t, os.WriteFile(out, fileData, os.ModePerm))
var restorePrm shard.RestorePrm
restorePrm.WithPath(out)
_, err := sh.Restore(context.Background(), restorePrm)
require.ErrorIs(t, err, io.EOF)
})
t.Run("invalid object", func(t *testing.T) {
out := out + ".wrongobj"
fileData := append(fileData, 1, 0, 0, 0, 0xFF, 4, 0, 0, 0, 1, 2, 3, 4)
require.NoError(t, os.WriteFile(out, fileData, os.ModePerm))
var restorePrm shard.RestorePrm
restorePrm.WithPath(out)
_, err := sh.Restore(context.Background(), restorePrm)
require.Error(t, err)
t.Run("skip errors", func(t *testing.T) {
sh := newCustomShard(t, filepath.Join(t.TempDir(), "ignore"), false, nil, nil)
t.Cleanup(func() { require.NoError(t, sh.Close()) })
var restorePrm shard.RestorePrm
restorePrm.WithPath(out)
restorePrm.WithIgnoreErrors(true)
res, err := sh.Restore(context.Background(), restorePrm)
require.NoError(t, err)
require.Equal(t, objCount, res.Count())
require.Equal(t, 2, res.FailCount())
})
})
})
var prm shard.RestorePrm
prm.WithPath(out)
t.Run("must allow write", func(t *testing.T) {
require.NoError(t, sh.SetMode(mode.ReadOnly))
_, err := sh.Restore(context.Background(), prm)
require.ErrorIs(t, err, shard.ErrReadOnlyMode)
})
require.NoError(t, sh.SetMode(mode.ReadWrite))
checkRestore(t, sh, prm, objects)
})
}
func TestStream(t *testing.T) {
sh1 := newCustomShard(t, filepath.Join(t.TempDir(), "shard1"), false, nil, nil)
defer releaseShard(sh1, t)
sh2 := newCustomShard(t, filepath.Join(t.TempDir(), "shard2"), false, nil, nil)
defer releaseShard(sh2, t)
const objCount = 5
objects := make([]*objectSDK.Object, objCount)
for i := 0; i < objCount; i++ {
cnr := cidtest.ID()
obj := testutil.GenerateObjectWithCID(cnr)
objects[i] = obj
var prm shard.PutPrm
prm.SetObject(objects[i])
_, err := sh1.Put(context.Background(), prm)
require.NoError(t, err)
}
require.NoError(t, sh1.SetMode(mode.ReadOnly))
r, w := io.Pipe()
finish := make(chan struct{})
go func() {
var dumpPrm shard.DumpPrm
dumpPrm.WithStream(w)
res, err := sh1.Dump(dumpPrm)
require.NoError(t, err)
require.Equal(t, objCount, res.Count())
require.NoError(t, w.Close())
close(finish)
}()
var restorePrm shard.RestorePrm
restorePrm.WithStream(r)
checkRestore(t, sh2, restorePrm, objects)
require.Eventually(t, func() bool {
select {
case <-finish:
return true
default:
return false
}
}, time.Second, time.Millisecond)
}
func checkRestore(t *testing.T, sh *shard.Shard, prm shard.RestorePrm, objects []*objectSDK.Object) {
res, err := sh.Restore(context.Background(), prm)
require.NoError(t, err)
require.Equal(t, len(objects), res.Count())
var getPrm shard.GetPrm
for i := range objects {
getPrm.SetAddress(object.AddressOf(objects[i]))
res, err := sh.Get(context.Background(), getPrm)
require.NoError(t, err)
require.Equal(t, objects[i], res.Object())
}
}
func TestDumpIgnoreErrors(t *testing.T) {
const (
wcSmallObjectSize = 512 // goes to write-cache memory
wcBigObjectSize = wcSmallObjectSize << 1 // goes to write-cache FSTree
bsSmallObjectSize = wcSmallObjectSize << 2 // goes to blobovnicza DB
objCount = 10
headerSize = 400
)
dir := t.TempDir()
bsPath := filepath.Join(dir, "blob")
bsOpts := func(sw uint64) []blobstor.Option {
return []blobstor.Option{
blobstor.WithCompressObjects(true),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(bsPath, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(sw),
blobovniczatree.WithOpenedCacheSize(1)),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return len(data) < bsSmallObjectSize
},
},
{
Storage: fstree.New(
fstree.WithPath(bsPath),
fstree.WithDepth(1)),
},
}),
}
}
wcPath := filepath.Join(dir, "writecache")
wcOpts := []writecache.Option{
writecache.WithPath(wcPath),
writecache.WithSmallObjectSize(wcSmallObjectSize),
writecache.WithMaxObjectSize(wcBigObjectSize),
}
sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2))
objects := make([]*objectSDK.Object, objCount)
for i := 0; i < objCount; i++ {
size := (wcSmallObjectSize << (i % 4)) - headerSize
obj := testutil.GenerateObjectWithCIDWithPayload(cidtest.ID(), make([]byte, size))
objects[i] = obj
var prm shard.PutPrm
prm.SetObject(objects[i])
_, err := sh.Put(context.Background(), prm)
require.NoError(t, err)
}
releaseShard(sh, t)
b := bytes.NewBuffer(nil)
badObject := make([]byte, 1000)
enc, err := zstd.NewWriter(b)
require.NoError(t, err)
corruptedData := enc.EncodeAll(badObject, nil)
for i := 4; i < len(corruptedData); i++ {
corruptedData[i] ^= 0xFF
}
// There are 3 different types of errors to consider.
// To setup envirionment we use implementation details so this test must be updated
// if any of them are changed.
{
// 1. Invalid object in fs tree.
// 1.1. Invalid compressed data.
addr := cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString()
dirName := filepath.Join(bsPath, addr[:2])
require.NoError(t, os.MkdirAll(dirName, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(dirName, addr[2:]), corruptedData, os.ModePerm))
// 1.2. Unreadable file.
addr = cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString()
dirName = filepath.Join(bsPath, addr[:2])
require.NoError(t, os.MkdirAll(dirName, os.ModePerm))
fname := filepath.Join(dirName, addr[2:])
require.NoError(t, os.WriteFile(fname, []byte{}, 0))
// 1.3. Unreadable dir.
require.NoError(t, os.MkdirAll(filepath.Join(bsPath, "ZZ"), 0))
}
sh = newCustomShard(t, dir, true, wcOpts, bsOpts(3))
require.NoError(t, sh.SetMode(mode.ReadOnly))
{
// 2. Invalid object in blobovnicza.
// 2.1. Invalid blobovnicza.
bTree := filepath.Join(bsPath, "blobovnicza")
data := make([]byte, 1024)
rand.Read(data)
require.NoError(t, os.WriteFile(filepath.Join(bTree, "0", "2"), data, 0))
// 2.2. Invalid object in valid blobovnicza.
var prm blobovnicza.PutPrm
prm.SetAddress(oid.Address{})
prm.SetMarshaledObject(corruptedData)
b := blobovnicza.New(blobovnicza.WithPath(filepath.Join(bTree, "1", "2")))
require.NoError(t, b.Open())
_, err := b.Put(prm)
require.NoError(t, err)
require.NoError(t, b.Close())
}
{
// 3. Invalid object in write-cache. Note that because shard is read-only
// the object won't be flushed.
addr := cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString()
dir := filepath.Join(wcPath, addr[:1])
require.NoError(t, os.MkdirAll(dir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(dir, addr[1:]), nil, 0))
}
out := filepath.Join(t.TempDir(), "out.dump")
var dumpPrm shard.DumpPrm
dumpPrm.WithPath(out)
dumpPrm.WithIgnoreErrors(true)
res, err := sh.Dump(dumpPrm)
require.NoError(t, err)
require.Equal(t, objCount, res.Count())
}