forked from TrueCloudLab/frostfs-node
[#86] node: Move testing utils to one package
Move testing utils from tests in local_object_storage package to unified testutil package Signed-off-by: Airat Arifullin <aarifullin@yadro.com>
This commit is contained in:
Gitea
parent
342e571d89
commit
9808dec591
41 changed files with 495 additions and 520 deletions
|
|
@ -1,7 +1,6 @@
|
|||
package blobstor
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
|
@ -10,15 +9,8 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/atomic"
|
||||
"golang.org/x/exp/rand"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// The storages to benchmark. Each storage has a description and a function which returns the actual
|
||||
|
|
@ -83,20 +75,20 @@ func BenchmarkSubstorageReadPerf(b *testing.B) {
|
|||
readTests := []struct {
|
||||
desc string
|
||||
size int
|
||||
objGen func() objectGenerator
|
||||
addrGen func() addressGenerator
|
||||
objGen func() testutil.ObjectGenerator
|
||||
addrGen func() testutil.AddressGenerator
|
||||
}{
|
||||
{
|
||||
desc: "seq100",
|
||||
size: 10000,
|
||||
objGen: func() objectGenerator { return &seqObjGenerator{objSize: 100} },
|
||||
addrGen: func() addressGenerator { return &seqAddrGenerator{maxID: 100} },
|
||||
objGen: func() testutil.ObjectGenerator { return &testutil.SeqObjGenerator{ObjSize: 100} },
|
||||
addrGen: func() testutil.AddressGenerator { return &testutil.SeqAddrGenerator{MaxID: 100} },
|
||||
},
|
||||
{
|
||||
desc: "rand100",
|
||||
size: 10000,
|
||||
objGen: func() objectGenerator { return &seqObjGenerator{objSize: 100} },
|
||||
addrGen: func() addressGenerator { return randAddrGenerator(10000) },
|
||||
objGen: func() testutil.ObjectGenerator { return &testutil.SeqObjGenerator{ObjSize: 100} },
|
||||
addrGen: func() testutil.AddressGenerator { return testutil.RandAddrGenerator(10000) },
|
||||
},
|
||||
}
|
||||
for _, tt := range readTests {
|
||||
|
|
@ -111,7 +103,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) {
|
|||
// Fill database
|
||||
for i := 0; i < tt.size; i++ {
|
||||
obj := objGen.Next()
|
||||
addr := addressFromObject(obj)
|
||||
addr := testutil.AddressFromObject(obj)
|
||||
raw, err := obj.Marshal()
|
||||
require.NoError(b, err)
|
||||
if _, err := st.Put(common.PutPrm{
|
||||
|
|
@ -142,14 +134,16 @@ func BenchmarkSubstorageReadPerf(b *testing.B) {
|
|||
func BenchmarkSubstorageWritePerf(b *testing.B) {
|
||||
generators := []struct {
|
||||
desc string
|
||||
create func() objectGenerator
|
||||
create func() testutil.ObjectGenerator
|
||||
}{
|
||||
{desc: "rand10", create: func() objectGenerator { return &randObjGenerator{objSize: 10} }},
|
||||
{desc: "rand100", create: func() objectGenerator { return &randObjGenerator{objSize: 100} }},
|
||||
{desc: "rand1000", create: func() objectGenerator { return &randObjGenerator{objSize: 1000} }},
|
||||
{desc: "overwrite10", create: func() objectGenerator { return &overwriteObjGenerator{objSize: 10, maxObjects: 100} }},
|
||||
{desc: "overwrite100", create: func() objectGenerator { return &overwriteObjGenerator{objSize: 100, maxObjects: 100} }},
|
||||
{desc: "overwrite1000", create: func() objectGenerator { return &overwriteObjGenerator{objSize: 1000, maxObjects: 100} }},
|
||||
{desc: "rand10", create: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 10} }},
|
||||
{desc: "rand100", create: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 100} }},
|
||||
{desc: "rand1000", create: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 1000} }},
|
||||
{desc: "overwrite10", create: func() testutil.ObjectGenerator { return &testutil.OverwriteObjGenerator{ObjSize: 10, MaxObjects: 100} }},
|
||||
{desc: "overwrite100", create: func() testutil.ObjectGenerator { return &testutil.OverwriteObjGenerator{ObjSize: 100, MaxObjects: 100} }},
|
||||
{desc: "overwrite1000", create: func() testutil.ObjectGenerator {
|
||||
return &testutil.OverwriteObjGenerator{ObjSize: 1000, MaxObjects: 100}
|
||||
}},
|
||||
}
|
||||
|
||||
for _, genEntry := range generators {
|
||||
|
|
@ -165,7 +159,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) {
|
|||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
obj := gen.Next()
|
||||
addr := addressFromObject(obj)
|
||||
addr := testutil.AddressFromObject(obj)
|
||||
raw, err := obj.Marshal()
|
||||
require.NoError(b, err)
|
||||
if _, err := st.Put(common.PutPrm{
|
||||
|
|
@ -188,12 +182,12 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) {
|
|||
iterateTests := []struct {
|
||||
desc string
|
||||
size int
|
||||
objGen func() objectGenerator
|
||||
objGen func() testutil.ObjectGenerator
|
||||
}{
|
||||
{
|
||||
desc: "rand100",
|
||||
size: 10000,
|
||||
objGen: func() objectGenerator { return &randObjGenerator{objSize: 100} },
|
||||
objGen: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 100} },
|
||||
},
|
||||
}
|
||||
for _, tt := range iterateTests {
|
||||
|
|
@ -208,7 +202,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) {
|
|||
// Fill database
|
||||
for i := 0; i < tt.size; i++ {
|
||||
obj := objGen.Next()
|
||||
addr := addressFromObject(obj)
|
||||
addr := testutil.AddressFromObject(obj)
|
||||
raw, err := obj.Marshal()
|
||||
require.NoError(b, err)
|
||||
if _, err := st.Put(common.PutPrm{
|
||||
|
|
@ -238,165 +232,3 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func addressFromObject(obj *objectSDK.Object) oid.Address {
|
||||
var addr oid.Address
|
||||
if id, isSet := obj.ID(); isSet {
|
||||
addr.SetObject(id)
|
||||
} else {
|
||||
panic("object ID is not set")
|
||||
}
|
||||
if cid, isSet := obj.ContainerID(); isSet {
|
||||
addr.SetContainer(cid)
|
||||
} else {
|
||||
panic("container ID is not set")
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
// addressGenerator is the interface of types that generate object addresses.
|
||||
type addressGenerator interface {
|
||||
Next() oid.Address
|
||||
}
|
||||
|
||||
// seqAddrGenerator is an addressGenerator that generates addresses sequentially and wraps around the given max ID.
|
||||
type seqAddrGenerator struct {
|
||||
cnt atomic.Uint64
|
||||
maxID uint64
|
||||
}
|
||||
|
||||
func (g *seqAddrGenerator) Next() oid.Address {
|
||||
var id oid.ID
|
||||
binary.LittleEndian.PutUint64(id[:], ((g.cnt.Inc()-1)%g.maxID)+1)
|
||||
var addr oid.Address
|
||||
addr.SetContainer(cid.ID{})
|
||||
addr.SetObject(id)
|
||||
return addr
|
||||
}
|
||||
|
||||
func TestSeqAddrGenerator(t *testing.T) {
|
||||
gen := &seqAddrGenerator{maxID: 10}
|
||||
for i := 1; i <= 20; i++ {
|
||||
addr := gen.Next()
|
||||
id := addr.Object()
|
||||
|
||||
require.Equal(t, uint64((i-1)%int(gen.maxID)+1), binary.LittleEndian.Uint64(id[:]))
|
||||
}
|
||||
}
|
||||
|
||||
// randAddrGenerator is an addressGenerator that generates random addresses in the given range.
|
||||
type randAddrGenerator uint64
|
||||
|
||||
func (g randAddrGenerator) Next() oid.Address {
|
||||
var id oid.ID
|
||||
binary.LittleEndian.PutUint64(id[:], uint64(1+int(rand.Int63n(int64(g)))))
|
||||
var addr oid.Address
|
||||
addr.SetContainer(cid.ID{})
|
||||
addr.SetObject(id)
|
||||
return addr
|
||||
}
|
||||
|
||||
func TestRandAddrGenerator(t *testing.T) {
|
||||
gen := randAddrGenerator(5)
|
||||
for i := 0; i < 50; i++ {
|
||||
addr := gen.Next()
|
||||
id := addr.Object()
|
||||
k := binary.LittleEndian.Uint64(id[:])
|
||||
|
||||
require.True(t, 1 <= k && k <= uint64(gen))
|
||||
}
|
||||
}
|
||||
|
||||
// objectGenerator is the interface of types that generate object entries.
|
||||
type objectGenerator interface {
|
||||
Next() *objectSDK.Object
|
||||
}
|
||||
|
||||
// seqObjGenerator is an objectGenerator that generates entries with random payloads of size objSize and sequential IDs.
|
||||
type seqObjGenerator struct {
|
||||
cnt atomic.Uint64
|
||||
objSize uint64
|
||||
}
|
||||
|
||||
func (g *seqObjGenerator) Next() *objectSDK.Object {
|
||||
var id oid.ID
|
||||
binary.LittleEndian.PutUint64(id[:], g.cnt.Inc())
|
||||
return genObject(id, cid.ID{}, g.objSize)
|
||||
}
|
||||
|
||||
func TestSeqObjGenerator(t *testing.T) {
|
||||
gen := &seqObjGenerator{objSize: 10}
|
||||
var addrs []string
|
||||
for i := 1; i <= 10; i++ {
|
||||
obj := gen.Next()
|
||||
id, isSet := obj.ID()
|
||||
addrs = append(addrs, addressFromObject(obj).EncodeToString())
|
||||
|
||||
require.True(t, isSet)
|
||||
require.Equal(t, gen.objSize, uint64(len(obj.Payload())))
|
||||
require.Equal(t, uint64(i), binary.LittleEndian.Uint64(id[:]))
|
||||
}
|
||||
require.True(t, slices.IsSorted(addrs))
|
||||
}
|
||||
|
||||
// randObjGenerator is an objectGenerator that generates entries with random IDs and payloads of size objSize.
|
||||
type randObjGenerator struct {
|
||||
objSize uint64
|
||||
}
|
||||
|
||||
func (g *randObjGenerator) Next() *objectSDK.Object {
|
||||
return genObject(oidtest.ID(), cidtest.ID(), g.objSize)
|
||||
}
|
||||
|
||||
func TestRandObjGenerator(t *testing.T) {
|
||||
gen := &randObjGenerator{objSize: 10}
|
||||
for i := 0; i < 10; i++ {
|
||||
obj := gen.Next()
|
||||
|
||||
require.Equal(t, gen.objSize, uint64(len(obj.Payload())))
|
||||
}
|
||||
}
|
||||
|
||||
// overwriteObjGenerator is an objectGenerator that generates entries with random payloads of size objSize and at most maxObjects distinct IDs.
|
||||
type overwriteObjGenerator struct {
|
||||
objSize uint64
|
||||
maxObjects uint64
|
||||
}
|
||||
|
||||
func (g *overwriteObjGenerator) Next() *objectSDK.Object {
|
||||
var id oid.ID
|
||||
binary.LittleEndian.PutUint64(id[:], uint64(1+rand.Int63n(int64(g.maxObjects))))
|
||||
return genObject(id, cid.ID{}, g.objSize)
|
||||
}
|
||||
|
||||
func TestOverwriteObjGenerator(t *testing.T) {
|
||||
gen := &overwriteObjGenerator{
|
||||
objSize: 10,
|
||||
maxObjects: 4,
|
||||
}
|
||||
for i := 0; i < 40; i++ {
|
||||
obj := gen.Next()
|
||||
id, isSet := obj.ID()
|
||||
i := binary.LittleEndian.Uint64(id[:])
|
||||
|
||||
require.True(t, isSet)
|
||||
require.Equal(t, gen.objSize, uint64(len(obj.Payload())))
|
||||
require.True(t, 1 <= i && i <= gen.maxObjects)
|
||||
}
|
||||
}
|
||||
|
||||
// Generates an object with random payload and the given address and size.
|
||||
// TODO(#86): there's some testing-related dupes in many places. Probably worth
|
||||
// spending some time cleaning up a bit.
|
||||
func genObject(id oid.ID, cid cid.ID, sz uint64) *objectSDK.Object {
|
||||
raw := objectSDK.New()
|
||||
|
||||
raw.SetID(id)
|
||||
raw.SetContainerID(cid)
|
||||
|
||||
payload := make([]byte, sz)
|
||||
rand.Read(payload)
|
||||
raw.SetPayload(payload)
|
||||
|
||||
return raw
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue