[#1085] shard: dump data from write-cache

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-01-18 16:40:11 +03:00 committed by LeL
parent ad01aaf8bf
commit c0f6c988f0
6 changed files with 105 additions and 12 deletions

View file

@ -72,7 +72,23 @@ func (s *Shard) Evacuate(prm *EvacuatePrm) (*EvacuateRes, error) {
var count int var count int
if s.hasWriteCache() { if s.hasWriteCache() {
// TODO evacuate objects from write cache err := s.writeCache.Iterate(func(data []byte) error {
var size [4]byte
binary.LittleEndian.PutUint32(size[:], uint32(len(data)))
if _, err := w.Write(size[:]); err != nil {
return err
}
if _, err := w.Write(data); err != nil {
return err
}
count++
return nil
})
if err != nil {
return nil, err
}
} }
var pi blobstor.IteratePrm var pi blobstor.IteratePrm

View file

@ -4,21 +4,44 @@ import (
"errors" "errors"
"io" "io"
"io/ioutil" "io/ioutil"
"math/rand"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"testing" "testing"
"time" "time"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestEvacuate(t *testing.T) { func TestEvacuate(t *testing.T) {
sh := newShard(t, false) t.Run("without write-cache", func(t *testing.T) {
testEvacuate(t, 10, false)
})
t.Run("with write-cache", func(t *testing.T) {
// Put a bit more objects to write-cache to facilitate race-conditions.
testEvacuate(t, 100, true)
})
}
func testEvacuate(t *testing.T, objCount int, hasWriteCache bool) {
const (
wcSmallObjectSize = 1024 // 1 KiB, goes to write-cache memory
wcBigObjectSize = 4 * 1024 // 4 KiB, goes to write-cache FSTree
bsSmallObjectSize = 10 * 1024 // 10 KiB, goes to blobovnicza DB
bsBigObjectSize = 1024*1024 + 1 // > 1 MiB, goes to blobovnicza FSTree
)
var sh *shard.Shard
if !hasWriteCache {
sh = newShard(t, false)
} else {
sh = newCustomShard(t, true,
writecache.WithSmallObjectSize(wcSmallObjectSize),
writecache.WithMaxObjectSize(wcBigObjectSize))
}
defer releaseShard(sh, t) defer releaseShard(sh, t)
out := filepath.Join(t.TempDir(), "dump") out := filepath.Join(t.TempDir(), "dump")
@ -36,12 +59,25 @@ func TestEvacuate(t *testing.T) {
require.Equal(t, 0, res.Count()) require.Equal(t, 0, res.Count())
require.NoError(t, sh.SetMode(shard.ModeReadWrite)) require.NoError(t, sh.SetMode(shard.ModeReadWrite))
const objCount = 10 // Approximate object header size.
const headerSize = 400
objects := make([]*object.Object, objCount) objects := make([]*object.Object, objCount)
for i := 0; i < objCount; i++ { for i := 0; i < objCount; i++ {
cid := cidtest.ID() cid := cidtest.ID()
obj := generateRawObjectWithCID(t, cid) var size int
addAttribute(obj, "foo", strconv.FormatUint(rand.Uint64(), 10)) switch i % 6 {
case 0, 1:
size = wcSmallObjectSize - headerSize
case 2, 3:
size = bsSmallObjectSize - headerSize
case 4:
size = wcBigObjectSize - headerSize
default:
size = bsBigObjectSize - headerSize
}
data := make([]byte, size)
obj := generateRawObjectWithPayload(cid, data)
objects[i] = obj.Object() objects[i] = obj.Object()
prm := new(shard.PutPrm).WithObject(objects[i]) prm := new(shard.PutPrm).WithObject(objects[i])

View file

@ -42,7 +42,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) {
res, err := testHead(t, sh, headPrm, hasWriteCache) res, err := testHead(t, sh, headPrm, hasWriteCache)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, obj.Object(), res.Object()) require.Equal(t, obj.CutPayload().Object(), res.Object())
}) })
t.Run("virtual object", func(t *testing.T) { t.Run("virtual object", func(t *testing.T) {
@ -75,7 +75,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) {
head, err := sh.Head(headPrm) head, err := sh.Head(headPrm)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, parent.Object(), head.Object()) require.Equal(t, parent.CutPayload().Object(), head.Object())
}) })
} }

View file

@ -27,6 +27,10 @@ import (
) )
func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { func newShard(t testing.TB, enableWriteCache bool) *shard.Shard {
return newCustomShard(t, enableWriteCache, writecache.WithMaxMemSize(0))
}
func newCustomShard(t testing.TB, enableWriteCache bool, wcOpts ...writecache.Option) *shard.Shard {
rootPath := t.Name() rootPath := t.Name()
if enableWriteCache { if enableWriteCache {
rootPath = path.Join(rootPath, "wc") rootPath = path.Join(rootPath, "wc")
@ -46,8 +50,7 @@ func newShard(t testing.TB, enableWriteCache bool) *shard.Shard {
), ),
shard.WithWriteCache(enableWriteCache), shard.WithWriteCache(enableWriteCache),
shard.WithWriteCacheOptions( shard.WithWriteCacheOptions(
writecache.WithMaxMemSize(0), // disable memory batches append(wcOpts, writecache.WithPath(path.Join(rootPath, "wcache")))...,
writecache.WithPath(path.Join(rootPath, "wcache")),
), ),
} }
@ -69,12 +72,17 @@ func generateRawObject(t *testing.T) *object.RawObject {
} }
func generateRawObjectWithCID(t *testing.T, cid *cid.ID) *object.RawObject { func generateRawObjectWithCID(t *testing.T, cid *cid.ID) *object.RawObject {
data := owner.PublicKeyToIDBytes(&test.DecodeKey(-1).PublicKey)
return generateRawObjectWithPayload(cid, data)
}
func generateRawObjectWithPayload(cid *cid.ID, data []byte) *object.RawObject {
version := version.New() version := version.New()
version.SetMajor(2) version.SetMajor(2)
version.SetMinor(1) version.SetMinor(1)
csum := new(checksum.Checksum) csum := new(checksum.Checksum)
csum.SetSHA256(sha256.Sum256(owner.PublicKeyToIDBytes(&test.DecodeKey(-1).PublicKey))) csum.SetSHA256(sha256.Sum256(data))
csumTZ := new(checksum.Checksum) csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum())) csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
@ -84,6 +92,7 @@ func generateRawObjectWithCID(t *testing.T, cid *cid.ID) *object.RawObject {
obj.SetOwnerID(ownertest.ID()) obj.SetOwnerID(ownertest.ID())
obj.SetContainerID(cid) obj.SetContainerID(cid)
obj.SetVersion(version) obj.SetVersion(version)
obj.SetPayload(data)
obj.SetPayloadChecksum(csum) obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ) obj.SetPayloadHomomorphicHash(csumTZ)

View file

@ -11,6 +11,37 @@ import (
// ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing. // ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing.
var ErrNoDefaultBucket = errors.New("no default bucket") var ErrNoDefaultBucket = errors.New("no default bucket")
// Iterate iterates over all objects present in write cache.
// This is very difficult to do correctly unless write-cache is put in read-only mode.
// Thus we silently fail if shard is not in read-only mode to avoid reporting misleading results.
func (c *cache) Iterate(f func([]byte) error) error {
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
if c.mode != ModeReadOnly {
return nil
}
err := c.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(defaultBucket)
return b.ForEach(func(k, data []byte) error {
if _, ok := c.flushed.Peek(string(k)); ok {
return nil
}
return f(data)
})
})
if err != nil {
return err
}
return c.fsTree.Iterate(func(addr *object.Address, data []byte) error {
if _, ok := c.flushed.Peek(addr.String()); ok {
return nil
}
return f(data)
})
}
// IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return. // IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return.
// It is assumed that db is an underlying database of some WriteCache instance. // It is assumed that db is an underlying database of some WriteCache instance.
// //

View file

@ -20,6 +20,7 @@ type Cache interface {
Get(*objectSDK.Address) (*object.Object, error) Get(*objectSDK.Address) (*object.Object, error)
Head(*objectSDK.Address) (*object.Object, error) Head(*objectSDK.Address) (*object.Object, error)
Delete(*objectSDK.Address) error Delete(*objectSDK.Address) error
Iterate(func(data []byte) error) error
Put(*object.Object) error Put(*object.Object) error
SetMode(Mode) SetMode(Mode)
DumpInfo() Info DumpInfo() Info