TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 5 Projects Releases 1 Activity Actions

[#86] node: Move testing utils to one package #150

Merged
fyrchik merged 1 commit from aarifullin/frostfs-node:refactor/86-move_test_utils into master 2023-03-23 08:19:16 +00:00
Conversation 0 Commits 1 Files changed 41 +495 -520
41 changed files with 495 additions and 520 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 54fb265598 - Show all commits

210
pkg/local_object_storage/blobstor/perf_test.go
View file

@ -1,7 +1,6 @@
package blobstor
import (
"encoding/binary"
"fmt"
"os"
"testing"
@ -10,15 +9,8 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"golang.org/x/exp/rand"
"golang.org/x/exp/slices"
)
// The storages to benchmark. Each storage has a description and a function which returns the actual
@ -83,20 +75,20 @@ func BenchmarkSubstorageReadPerf(b *testing.B) {
readTests := []struct {
desc string
size int
objGen func() objectGenerator
addrGen func() addressGenerator
objGen func() testutil.ObjectGenerator
addrGen func() testutil.AddressGenerator
}{
{
desc: "seq100",
size: 10000,
objGen: func() objectGenerator { return &seqObjGenerator{objSize: 100} },
addrGen: func() addressGenerator { return &seqAddrGenerator{maxID: 100} },
objGen: func() testutil.ObjectGenerator { return &testutil.SeqObjGenerator{ObjSize: 100} },
addrGen: func() testutil.AddressGenerator { return &testutil.SeqAddrGenerator{MaxID: 100} },
},
{
desc: "rand100",
size: 10000,
objGen: func() objectGenerator { return &seqObjGenerator{objSize: 100} },
addrGen: func() addressGenerator { return randAddrGenerator(10000) },
objGen: func() testutil.ObjectGenerator { return &testutil.SeqObjGenerator{ObjSize: 100} },
addrGen: func() testutil.AddressGenerator { return testutil.RandAddrGenerator(10000) },
},
}
for _, tt := range readTests {
@ -111,7 +103,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) {
// Fill database
for i := 0; i < tt.size; i++ {
obj := objGen.Next()
addr := addressFromObject(obj)
addr := testutil.AddressFromObject(obj)
raw, err := obj.Marshal()
require.NoError(b, err)
if _, err := st.Put(common.PutPrm{
@ -142,14 +134,16 @@ func BenchmarkSubstorageReadPerf(b *testing.B) {
func BenchmarkSubstorageWritePerf(b *testing.B) {
generators := []struct {
desc string
create func() objectGenerator
create func() testutil.ObjectGenerator
}{
{desc: "rand10", create: func() objectGenerator { return &randObjGenerator{objSize: 10} }},
{desc: "rand100", create: func() objectGenerator { return &randObjGenerator{objSize: 100} }},
{desc: "rand1000", create: func() objectGenerator { return &randObjGenerator{objSize: 1000} }},
{desc: "overwrite10", create: func() objectGenerator { return &overwriteObjGenerator{objSize: 10, maxObjects: 100} }},
{desc: "overwrite100", create: func() objectGenerator { return &overwriteObjGenerator{objSize: 100, maxObjects: 100} }},
{desc: "overwrite1000", create: func() objectGenerator { return &overwriteObjGenerator{objSize: 1000, maxObjects: 100} }},
{desc: "rand10", create: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 10} }},
{desc: "rand100", create: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 100} }},
{desc: "rand1000", create: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 1000} }},
{desc: "overwrite10", create: func() testutil.ObjectGenerator { return &testutil.OverwriteObjGenerator{ObjSize: 10, MaxObjects: 100} }},
{desc: "overwrite100", create: func() testutil.ObjectGenerator { return &testutil.OverwriteObjGenerator{ObjSize: 100, MaxObjects: 100} }},
{desc: "overwrite1000", create: func() testutil.ObjectGenerator {
return &testutil.OverwriteObjGenerator{ObjSize: 1000, MaxObjects: 100}
}},
}
for _, genEntry := range generators {
@ -165,7 +159,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
obj := gen.Next()
addr := addressFromObject(obj)
addr := testutil.AddressFromObject(obj)
raw, err := obj.Marshal()
require.NoError(b, err)
if _, err := st.Put(common.PutPrm{
@ -188,12 +182,12 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) {
iterateTests := []struct {
desc string
size int
objGen func() objectGenerator
objGen func() testutil.ObjectGenerator
}{
{
desc: "rand100",
size: 10000,
objGen: func() objectGenerator { return &randObjGenerator{objSize: 100} },
objGen: func() testutil.ObjectGenerator { return &testutil.RandObjGenerator{ObjSize: 100} },
},
}
for _, tt := range iterateTests {
@ -208,7 +202,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) {
// Fill database
for i := 0; i < tt.size; i++ {
obj := objGen.Next()
addr := addressFromObject(obj)
addr := testutil.AddressFromObject(obj)
raw, err := obj.Marshal()
require.NoError(b, err)
if _, err := st.Put(common.PutPrm{
@ -238,165 +232,3 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) {
}
}
}
func addressFromObject(obj *objectSDK.Object) oid.Address {
var addr oid.Address
if id, isSet := obj.ID(); isSet {
addr.SetObject(id)
} else {
panic("object ID is not set")
}
if cid, isSet := obj.ContainerID(); isSet {
addr.SetContainer(cid)
} else {
panic("container ID is not set")
}
return addr
}
// addressGenerator is the interface of types that generate object addresses.
type addressGenerator interface {
Next() oid.Address
}
// seqAddrGenerator is an addressGenerator that generates addresses sequentially and wraps around the given max ID.
type seqAddrGenerator struct {
cnt atomic.Uint64
maxID uint64
}
func (g *seqAddrGenerator) Next() oid.Address {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], ((g.cnt.Inc()-1)%g.maxID)+1)
var addr oid.Address
addr.SetContainer(cid.ID{})
addr.SetObject(id)
return addr
}
func TestSeqAddrGenerator(t *testing.T) {
gen := &seqAddrGenerator{maxID: 10}
for i := 1; i <= 20; i++ {
addr := gen.Next()
id := addr.Object()
require.Equal(t, uint64((i-1)%int(gen.maxID)+1), binary.LittleEndian.Uint64(id[:]))
}
}
// randAddrGenerator is an addressGenerator that generates random addresses in the given range.
type randAddrGenerator uint64
func (g randAddrGenerator) Next() oid.Address {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], uint64(1+int(rand.Int63n(int64(g)))))
var addr oid.Address
addr.SetContainer(cid.ID{})
addr.SetObject(id)
return addr
}
func TestRandAddrGenerator(t *testing.T) {
gen := randAddrGenerator(5)
for i := 0; i < 50; i++ {
addr := gen.Next()
id := addr.Object()
k := binary.LittleEndian.Uint64(id[:])
require.True(t, 1 <= k && k <= uint64(gen))
}
}
// objectGenerator is the interface of types that generate object entries.
type objectGenerator interface {
Next() *objectSDK.Object
}
// seqObjGenerator is an objectGenerator that generates entries with random payloads of size objSize and sequential IDs.
type seqObjGenerator struct {
cnt atomic.Uint64
objSize uint64
}
func (g *seqObjGenerator) Next() *objectSDK.Object {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], g.cnt.Inc())
return genObject(id, cid.ID{}, g.objSize)
}
func TestSeqObjGenerator(t *testing.T) {
gen := &seqObjGenerator{objSize: 10}
var addrs []string
for i := 1; i <= 10; i++ {
obj := gen.Next()
id, isSet := obj.ID()
addrs = append(addrs, addressFromObject(obj).EncodeToString())
require.True(t, isSet)
require.Equal(t, gen.objSize, uint64(len(obj.Payload())))
require.Equal(t, uint64(i), binary.LittleEndian.Uint64(id[:]))
}
require.True(t, slices.IsSorted(addrs))
}
// randObjGenerator is an objectGenerator that generates entries with random IDs and payloads of size objSize.
type randObjGenerator struct {
objSize uint64
}
func (g *randObjGenerator) Next() *objectSDK.Object {
return genObject(oidtest.ID(), cidtest.ID(), g.objSize)
}
func TestRandObjGenerator(t *testing.T) {
gen := &randObjGenerator{objSize: 10}
for i := 0; i < 10; i++ {
obj := gen.Next()
require.Equal(t, gen.objSize, uint64(len(obj.Payload())))
}
}
// overwriteObjGenerator is an objectGenerator that generates entries with random payloads of size objSize and at most maxObjects distinct IDs.
type overwriteObjGenerator struct {
objSize uint64
maxObjects uint64
}
func (g *overwriteObjGenerator) Next() *objectSDK.Object {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], uint64(1+rand.Int63n(int64(g.maxObjects))))
return genObject(id, cid.ID{}, g.objSize)
}
func TestOverwriteObjGenerator(t *testing.T) {
gen := &overwriteObjGenerator{
objSize: 10,
maxObjects: 4,
}
for i := 0; i < 40; i++ {
obj := gen.Next()
id, isSet := obj.ID()
i := binary.LittleEndian.Uint64(id[:])
require.True(t, isSet)
require.Equal(t, gen.objSize, uint64(len(obj.Payload())))
require.True(t, 1 <= i && i <= gen.maxObjects)
}
}
// Generates an object with random payload and the given address and size.
// TODO(#86): there's some testing-related dupes in many places. Probably worth
// spending some time cleaning up a bit.
func genObject(id oid.ID, cid cid.ID, sz uint64) *objectSDK.Object {
raw := objectSDK.New()
raw.SetID(id)
raw.SetContainerID(cid)
payload := make([]byte, sz)
rand.Read(payload)
raw.SetPayload(payload)
return raw
}

3
pkg/local_object_storage/engine/control_test.go
View file

@ -11,6 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
@ -156,7 +157,7 @@ func TestExecBlocks(t *testing.T) {
})
// put some object
obj := generateObjectWithCID(t, cidtest.ID())
obj := testutil.GenerateObjectWithCID(cidtest.ID())
addr := object.AddressOf(obj)

7
pkg/local_object_storage/engine/delete_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -22,7 +23,7 @@ func TestDeleteBigObject(t *testing.T) {
parentID := oidtest.ID()
splitID := objectSDK.NewSplitID()
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
parent.SetID(parentID)
parent.SetPayload(nil)
@ -30,7 +31,7 @@ func TestDeleteBigObject(t *testing.T) {
children := make([]*objectSDK.Object, childCount)
childIDs := make([]oid.ID, childCount)
for i := range children {
children[i] = generateObjectWithCID(t, cnr)
children[i] = testutil.GenerateObjectWithCID(cnr)
if i != 0 {
children[i].SetPreviousID(childIDs[i-1])
}
@ -42,7 +43,7 @@ func TestDeleteBigObject(t *testing.T) {
childIDs[i], _ = children[i].ID()
}
link := generateObjectWithCID(t, cnr)
link := testutil.GenerateObjectWithCID(cnr)
link.SetParent(parent)
link.SetParentID(parentID)
link.SetSplitID(splitID)

41
pkg/local_object_storage/engine/engine_test.go
View file

@ -9,20 +9,15 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"git.frostfs.info/TrueCloudLab/hrw"
"git.frostfs.info/TrueCloudLab/tzhash/tz"
"github.com/panjf2000/ants/v2"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
@ -61,7 +56,7 @@ func benchmarkExists(b *testing.B, shardNum int) {
addr := oidtest.Address()
for i := 0; i < 100; i++ {
obj := generateObjectWithCID(b, cidtest.ID())
obj := testutil.GenerateObjectWithCID(cidtest.ID())
err := Put(e, obj)
if err != nil {
b.Fatal(err)
@ -170,38 +165,6 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts []shard.Option) *S
return engine
}
func generateObjectWithCID(t testing.TB, cnr cid.ID) *object.Object {
var ver version.Version
ver.SetMajor(2)
ver.SetMinor(1)
csum := checksumtest.Checksum()
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(oidtest.ID())
obj.SetOwnerID(usertest.ID())
obj.SetContainerID(cnr)
obj.SetVersion(&ver)
obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ)
obj.SetPayload([]byte{1, 2, 3, 4, 5})
return obj
}
func addAttribute(obj *object.Object, key, val string) {
var attr object.Attribute
attr.SetKey(key)
attr.SetValue(val)
attrs := obj.Attributes()
attrs = append(attrs, attr)
obj.SetAttributes(attrs...)
}
func testNewEngineWithShardNum(t *testing.T, num int) *StorageEngine {
shards := make([]*shard.Shard, 0, num)

7
pkg/local_object_storage/engine/error_test.go
View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
@ -65,7 +66,7 @@ func TestErrorReporting(t *testing.T) {
t.Run("ignore errors by default", func(t *testing.T) {
e, dir, id := newEngineWithErrorThreshold(t, "", 0)
obj := generateObjectWithCID(t, cidtest.ID())
obj := testutil.GenerateObjectWithCID(cidtest.ID())
obj.SetPayload(make([]byte, errSmallSize))
var prm shard.PutPrm
@ -95,7 +96,7 @@ func TestErrorReporting(t *testing.T) {
e, dir, id := newEngineWithErrorThreshold(t, "", errThreshold)
obj := generateObjectWithCID(t, cidtest.ID())
obj := testutil.GenerateObjectWithCID(cidtest.ID())
obj.SetPayload(make([]byte, errSmallSize))
var prm shard.PutPrm
@ -145,7 +146,7 @@ func TestBlobstorFailback(t *testing.T) {
objs := make([]*objectSDK.Object, 0, 2)
for _, size := range []int{15, errSmallSize + 1} {
obj := generateObjectWithCID(t, cidtest.ID())
obj := testutil.GenerateObjectWithCID(cidtest.ID())
obj.SetPayload(make([]byte, size))
var prm shard.PutPrm

5
pkg/local_object_storage/engine/evacuate_test.go
View file

@ -11,6 +11,7 @@ import (
objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@ -55,7 +56,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng
objects := make([]*objectSDK.Object, 0, objPerShard*len(ids))
for _, sh := range ids {
obj := generateObjectWithCID(t, cidtest.ID())
obj := testutil.GenerateObjectWithCID(cidtest.ID())
objects = append(objects, obj)
var putPrm shard.PutPrm
@ -65,7 +66,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng
}
for i := 0; ; i++ {
objects = append(objects, generateObjectWithCID(t, cidtest.ID()))
objects = append(objects, testutil.GenerateObjectWithCID(cidtest.ID()))
var putPrm PutPrm
putPrm.WithObject(objects[len(objects)-1])

9
pkg/local_object_storage/engine/head_test.go
View file

@ -4,6 +4,7 @@ import (
"os"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -17,8 +18,8 @@ func TestHeadRaw(t *testing.T) {
cnr := cidtest.ID()
splitID := object.NewSplitID()
parent := generateObjectWithCID(t, cnr)
addAttribute(parent, "foo", "bar")
parent := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(parent, "foo", "bar")
var parentAddr oid.Address
parentAddr.SetContainer(cnr)
@ -26,12 +27,12 @@ func TestHeadRaw(t *testing.T) {
idParent, _ := parent.ID()
parentAddr.SetObject(idParent)
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
child.SetParentID(idParent)
child.SetSplitID(splitID)
link := generateObjectWithCID(t, cnr)
link := testutil.GenerateObjectWithCID(cnr)
link.SetParent(parent)
link.SetParentID(idParent)

9
pkg/local_object_storage/engine/inhume_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -20,16 +21,16 @@ func TestStorageEngine_Inhume(t *testing.T) {
fs := objectSDK.SearchFilters{}
fs.AddRootFilter()
tombstoneID := object.AddressOf(generateObjectWithCID(t, cnr))
parent := generateObjectWithCID(t, cnr)
tombstoneID := object.AddressOf(testutil.GenerateObjectWithCID(cnr))
parent := testutil.GenerateObjectWithCID(cnr)
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
child.SetSplitID(splitID)
link := generateObjectWithCID(t, cnr)
link := testutil.GenerateObjectWithCID(cnr)
link.SetParent(parent)
link.SetParentID(idParent)
idChild, _ := child.ID()

3
pkg/local_object_storage/engine/list_test.go
View file

@ -7,6 +7,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"github.com/stretchr/testify/require"
@ -29,7 +30,7 @@ func TestListWithCursor(t *testing.T) {
for i := 0; i < total; i++ {
containerID := cidtest.ID()
obj := generateObjectWithCID(t, containerID)
obj := testutil.GenerateObjectWithCID(containerID)
var prm PutPrm
prm.WithObject(obj)

15
pkg/local_object_storage/engine/lock_test.go
View file

@ -9,6 +9,7 @@ import (
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
@ -41,7 +42,7 @@ func TestLockUserScenario(t *testing.T) {
const lockerExpiresAfter = 13
cnr := cidtest.ID()
tombObj := generateObjectWithCID(t, cnr)
tombObj := testutil.GenerateObjectWithCID(cnr)
tombForLockID := oidtest.ID()
tombObj.SetID(tombForLockID)
@ -79,7 +80,7 @@ func TestLockUserScenario(t *testing.T) {
a.SetKey(objectV2.SysAttributeExpEpoch)
a.SetValue(strconv.Itoa(lockerExpiresAfter))
lockerObj := generateObjectWithCID(t, cnr)
lockerObj := testutil.GenerateObjectWithCID(cnr)
lockerObj.SetID(lockerID)
lockerObj.SetAttributes(a)
@ -88,7 +89,7 @@ func TestLockUserScenario(t *testing.T) {
tombForLockAddr.SetObject(tombForLockID)
// 1.
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
id, _ := obj.ID()
objAddr.SetObject(id)
@ -166,7 +167,7 @@ func TestLockExpiration(t *testing.T) {
var err error
// 1.
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
err = Put(e, obj)
require.NoError(t, err)
@ -176,7 +177,7 @@ func TestLockExpiration(t *testing.T) {
a.SetKey(objectV2.SysAttributeExpEpoch)
a.SetValue(strconv.Itoa(lockerExpiresAfter))
lock := generateObjectWithCID(t, cnr)
lock := testutil.GenerateObjectWithCID(cnr)
lock.SetType(object.TypeLock)
lock.SetAttributes(a)
@ -237,13 +238,13 @@ func TestLockForceRemoval(t *testing.T) {
var err error
// 1.
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
err = Put(e, obj)
require.NoError(t, err)
// 2.
lock := generateObjectWithCID(t, cnr)
lock := testutil.GenerateObjectWithCID(cnr)
lock.SetType(object.TypeLock)
err = Put(e, lock)

5
pkg/local_object_storage/engine/tree_test.go
View file

@ -4,6 +4,7 @@ import (
"strconv"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -28,8 +29,8 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) {
treeID := "someTree"
for i := 0; i < objCount; i++ {
obj := generateObjectWithCID(b, cid)
addAttribute(obj, pilorama.AttributeFilename, strconv.Itoa(i))
obj := testutil.GenerateObjectWithCID(cid)
testutil.AddAttribute(obj, pilorama.AttributeFilename, strconv.Itoa(i))
err := Put(e, obj)
if err != nil {
b.Fatal(err)

110
pkg/local_object_storage/internal/testutil/generators.go Normal file
View file

@ -0,0 +1,110 @@
package testutil
import (
"encoding/binary"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.uber.org/atomic"
"golang.org/x/exp/rand"
)
// AddressGenerator is the interface of types that generate object addresses.
type AddressGenerator interface {
Next() oid.Address
}
// SeqAddrGenerator is an AddressGenerator that generates addresses sequentially and wraps around the given max ID.
type SeqAddrGenerator struct {
cnt atomic.Uint64
MaxID uint64
}
var _ AddressGenerator = &SeqAddrGenerator{}
func (g *SeqAddrGenerator) Next() oid.Address {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], ((g.cnt.Inc()-1)%g.MaxID)+1)
var addr oid.Address
addr.SetContainer(cid.ID{})
addr.SetObject(id)
return addr
}
// RandAddrGenerator is an addressGenerator that generates random addresses in the given range.
type RandAddrGenerator uint64
func (g RandAddrGenerator) Next() oid.Address {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], uint64(1+int(rand.Int63n(int64(g)))))
var addr oid.Address
addr.SetContainer(cid.ID{})
addr.SetObject(id)
return addr
}
// ObjectGenerator is the interface of types that generate object entries.
type ObjectGenerator interface {
Next() *object.Object
}
// SeqObjGenerator is an ObjectGenerator that generates entries with random payloads of size objSize and sequential IDs.
type SeqObjGenerator struct {
cnt atomic.Uint64
ObjSize uint64
}
var _ ObjectGenerator = &SeqObjGenerator{}
func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *object.Object {
data := make([]byte, sz)
rand.Read(data)
obj := GenerateObjectWithCIDWithPayload(cid, data)
obj.SetID(oid)
return obj
}
func (g *SeqObjGenerator) Next() *object.Object {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], g.cnt.Inc())
return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize)
}
// RandObjGenerator is an ObjectGenerator that generates entries with random IDs and payloads of size objSize.
type RandObjGenerator struct {
ObjSize uint64
}
var _ ObjectGenerator = &RandObjGenerator{}
func (g *RandObjGenerator) Next() *object.Object {
return generateObjectWithOIDWithCIDWithSize(oid.ID{}, cid.ID{}, g.ObjSize)
}
// OverwriteObjGenerator is an ObjectGenerator that generates entries with random payloads of size objSize and at most maxObjects distinct IDs.
type OverwriteObjGenerator struct {
ObjSize uint64
MaxObjects uint64
}
func (g *OverwriteObjGenerator) Next() *object.Object {
var id oid.ID
binary.LittleEndian.PutUint64(id[:], uint64(1+rand.Int63n(int64(g.MaxObjects))))
return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize)
}
func AddressFromObject(obj *object.Object) oid.Address {
var addr oid.Address
if id, isSet := obj.ID(); isSet {
addr.SetObject(id)
} else {
panic("object ID is not set")
}
if cid, isSet := obj.ContainerID(); isSet {
addr.SetContainer(cid)
} else {
panic("container ID is not set")
}
return addr
}

70
pkg/local_object_storage/internal/testutil/generators_test.go Normal file
View file

@ -0,0 +1,70 @@
package testutil
import (
"encoding/binary"
"testing"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)
func TestOverwriteObjGenerator(t *testing.T) {
gen := &OverwriteObjGenerator{
ObjSize: 10,
MaxObjects: 4,
}
for i := 0; i < 40; i++ {
obj := gen.Next()
id, isSet := obj.ID()
i := binary.LittleEndian.Uint64(id[:])
require.True(t, isSet)
require.Equal(t, gen.ObjSize, uint64(len(obj.Payload())))
require.True(t, 1 <= i && i <= gen.MaxObjects)
}
}
func TestRandObjGenerator(t *testing.T) {
gen := &RandObjGenerator{ObjSize: 10}
for i := 0; i < 10; i++ {
obj := gen.Next()
require.Equal(t, gen.ObjSize, uint64(len(obj.Payload())))
}
}
func TestSeqObjGenerator(t *testing.T) {
gen := &SeqObjGenerator{ObjSize: 10}
var addrs []string
for i := 1; i <= 10; i++ {
obj := gen.Next()
id, isSet := obj.ID()
addrs = append(addrs, AddressFromObject(obj).EncodeToString())
require.True(t, isSet)
require.Equal(t, gen.ObjSize, uint64(len(obj.Payload())))
require.Equal(t, uint64(i), binary.LittleEndian.Uint64(id[:]))
}
require.True(t, slices.IsSorted(addrs))
}
func TestRandAddrGenerator(t *testing.T) {
gen := RandAddrGenerator(5)
for i := 0; i < 50; i++ {
addr := gen.Next()
id := addr.Object()
k := binary.LittleEndian.Uint64(id[:])
require.True(t, 1 <= k && k <= uint64(gen))
}
}
func TestSeqAddrGenerator(t *testing.T) {
gen := &SeqAddrGenerator{MaxID: 10}
for i := 1; i <= 20; i++ {
addr := gen.Next()
id := addr.Object()
require.Equal(t, uint64((i-1)%int(gen.MaxID)+1), binary.LittleEndian.Uint64(id[:]))
}
}

68
pkg/local_object_storage/internal/testutil/object.go Normal file
View file

@ -0,0 +1,68 @@
package testutil
import (
"crypto/sha256"
aarifullin marked this conversation as resolved Outdated
ale64bit commented 2023-03-21 07:52:34 +00:00
Outdated
Review
Copy link

crypto/rand seems like overkill here. Maybe use golang.org/x/exp/rand instead.

`crypto/rand` seems like overkill here. Maybe use `golang.org/x/exp/rand` instead.
👍 1
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
fyrchik marked this conversation as resolved Outdated
dstepanov-yadro commented 2023-03-21 14:48:21 +00:00
Outdated
Review
Copy link

Why are the import blocks separated?

Why are the import blocks separated?
aarifullin commented 2023-03-21 17:04:47 +00:00
Outdated
Review
Copy link

Sorry, I just did the same on my previous work :)
The reason for the separate group: these are third-party packages (neither std's nor frostfs')

If it's unnecessary, I can unite them with third group

Sorry, I just did the same on my previous work :) The reason for the separate group: these are third-party packages (neither std's nor frostfs') If it's unnecessary, I can unite them with third group
carpawell commented 2023-03-21 19:07:19 +00:00
Outdated
Review
Copy link

i think they should be either separated or be merged through the whole repo

i think they should be either separated or be merged through the whole repo
fyrchik commented 2023-03-22 07:16:11 +00:00
Outdated
Review
Copy link

Across the repo we use 2 groups: stdlib and everything else.

Across the repo we use 2 groups: stdlib and everything else.
aarifullin commented 2023-03-22 07:25:51 +00:00
Outdated
Review
Copy link

Okay. Fixed

Okay. Fixed
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"git.frostfs.info/TrueCloudLab/tzhash/tz"
"golang.org/x/exp/rand"
)
const defaultDataSize = 32
func GenerateObject() *object.Object {
return GenerateObjectWithCID(cidtest.ID())
}
ale64bit marked this conversation as resolved Outdated
ale64bit commented 2023-03-21 07:53:03 +00:00
Outdated
Review
Copy link

the data size should probably be a parameter.

the data size should probably be a parameter.
aarifullin commented 2023-03-21 11:14:20 +00:00
Outdated
Review
Copy link

Fow now the package contains two methods: GenerateObject that generates an object with default size 32 and GenerateObjectWithSize to create it with given size. IMHO, this will make the refactoring not so cumbersome

Fow now the package contains two methods: `GenerateObject` that generates an object with default size `32` and `GenerateObjectWithSize` to create it with given size. IMHO, this will make the refactoring not so cumbersome
👍 1
func GenerateObjectWithCID(cnr cid.ID) *object.Object {
data := make([]byte, defaultDataSize)
rand.Read(data)
return GenerateObjectWithCIDWithPayload(cnr, data)
}
func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object {
var ver version.Version
ver.SetMajor(2)
fyrchik marked this conversation as resolved Outdated
fyrchik commented 2023-03-21 14:44:17 +00:00
Outdated
Review
Copy link

It seems to be used only 3 times and only privately, do we need this function?

It seems to be used only 3 times and only privately, do we need this function?
fyrchik commented 2023-03-21 14:44:17 +00:00
Outdated
Review
Copy link

It seems to be used only 3 times and only privately, do we need this function?

It seems to be used only 3 times and only privately, do we need this function?
aarifullin commented 2023-03-21 17:06:17 +00:00
Outdated
Review
Copy link

Followed this suggestion: #150 (comment)

Anyway, let's keep it but I'll make it private?

Followed this suggestion: https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls/150#issuecomment-2679 Anyway, let's keep it but I'll make it private?
fyrchik commented 2023-03-22 07:27:29 +00:00
Outdated
Review
Copy link

To me it is an unnecessary abstraction, given that providing freshly generated CID is trivial.

To me it is an unnecessary abstraction, given that providing freshly generated CID is trivial.
aarifullin commented 2023-03-22 10:12:03 +00:00
Outdated
Review
Copy link

Okay. I removed GenerateObjectWithSize

Okay. I removed `GenerateObjectWithSize`
ver.SetMinor(1)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(data))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(oidtest.ID())
obj.SetOwnerID(usertest.ID())
obj.SetContainerID(cnr)
obj.SetVersion(&ver)
obj.SetPayload(data)
obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ)
return obj
}
func AddAttribute(obj *object.Object, key, val string) {
var attr object.Attribute
attr.SetKey(key)
attr.SetValue(val)
attrs := obj.Attributes()
attrs = append(attrs, attr)
obj.SetAttributes(attrs...)
}
func AddPayload(obj *object.Object, size int) {
buf := make([]byte, size)
_, _ = rand.Read(buf)
obj.SetPayload(buf)
ale64bit marked this conversation as resolved Outdated
ale64bit commented 2023-03-21 07:54:49 +00:00
Outdated
Review
Copy link

Consider moving the generators here and remove the TODO from https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/pkg/local_object_storage/blobstor/perf_test.go#L257-L402

Consider moving the generators here and remove the TODO from https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/pkg/local_object_storage/blobstor/perf_test.go#L257-L402
aarifullin commented 2023-03-21 11:16:46 +00:00
Outdated
Review
Copy link

I've moved the generators to testutil and reused GenerateObject... methods. Hope it looks good

I've moved the generators to `testutil` and reused `GenerateObject...` methods. Hope it looks good
👍 1
obj.SetPayloadSize(uint64(size))
}

13
pkg/local_object_storage/metabase/containers_test.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -21,7 +22,7 @@ func TestDB_Containers(t *testing.T) {
cids := make(map[string]int, N)
for i := 0; i < N; i++ {
obj := generateObject(t)
obj := testutil.GenerateObject()
cnr, _ := obj.ContainerID()
@ -53,7 +54,7 @@ func TestDB_Containers(t *testing.T) {
}
t.Run("Inhume", func(t *testing.T) {
obj := generateObject(t)
obj := testutil.GenerateObject()
require.NoError(t, putBig(db, obj))
@ -71,7 +72,7 @@ func TestDB_Containers(t *testing.T) {
})
t.Run("ToMoveIt", func(t *testing.T) {
obj := generateObject(t)
obj := testutil.GenerateObject()
require.NoError(t, putBig(db, obj))
@ -107,7 +108,7 @@ func TestDB_ContainersCount(t *testing.T) {
for _, upload := range uploadObjects {
for i := 0; i < upload.amount; i++ {
obj := generateObject(t)
obj := testutil.GenerateObject()
obj.SetType(upload.typ)
err := putBig(db, obj)
@ -150,10 +151,10 @@ func TestDB_ContainerSize(t *testing.T) {
for j := 0; j < N; j++ {
size := rand.Intn(1024)
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
parent.SetPayloadSize(uint64(size / 2))
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
obj.SetPayloadSize(uint64(size))
idParent, _ := parent.ID()
obj.SetParentID(idParent)

3
pkg/local_object_storage/metabase/control_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
@ -16,7 +17,7 @@ func TestReset(t *testing.T) {
err := db.Reset()
require.NoError(t, err)
obj := generateObject(t)
obj := testutil.GenerateObject()
addr := object.AddressOf(obj)
addrToInhume := oidtest.Address()

11
pkg/local_object_storage/metabase/counter_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -29,7 +30,7 @@ func TestCounters(t *testing.T) {
t.Run("put", func(t *testing.T) {
oo := make([]*object.Object, 0, objCount)
for i := 0; i < objCount; i++ {
oo = append(oo, generateObject(t))
oo = append(oo, testutil.GenerateObject())
}
var prm meta.PutPrm
@ -102,12 +103,12 @@ func TestCounters(t *testing.T) {
require.NoError(t, db.Reset())
t.Run("put_split", func(t *testing.T) {
parObj := generateObject(t)
parObj := testutil.GenerateObject()
// put objects and check that parent info
// does not affect the counter
for i := 0; i < objCount; i++ {
o := generateObject(t)
o := testutil.GenerateObject()
if i < objCount/2 { // half of the objs will have the parent
o.SetParent(parObj)
}
@ -271,11 +272,11 @@ func TestCounters_Expired(t *testing.T) {
func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*object.Object {
var prm meta.PutPrm
var err error
parent := generateObject(t)
parent := testutil.GenerateObject()
oo := make([]*object.Object, 0, count)
for i := 0; i < count; i++ {
o := generateObject(t)
o := testutil.GenerateObject()
if withParent {
o.SetParent(parent)
}

48
pkg/local_object_storage/metabase/db_test.go
View file

@ -6,18 +6,12 @@ import (
"testing"
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"git.frostfs.info/TrueCloudLab/tzhash/tz"
"github.com/stretchr/testify/require"
)
@ -68,49 +62,13 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB {
return bdb
}
func generateObject(t testing.TB) *object.Object {
return generateObjectWithCID(t, cidtest.ID())
}
func generateObjectWithCID(t testing.TB, cnr cid.ID) *object.Object {
var ver version.Version
ver.SetMajor(2)
ver.SetMinor(1)
csum := checksumtest.Checksum()
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(oidtest.ID())
obj.SetOwnerID(usertest.ID())
obj.SetContainerID(cnr)
obj.SetVersion(&ver)
obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ)
obj.SetPayload([]byte{1, 2, 3, 4, 5})
return obj
}
func addAttribute(obj *object.Object, key, val string) {
var attr object.Attribute
attr.SetKey(key)
attr.SetValue(val)
attrs := obj.Attributes()
attrs = append(attrs, attr)
obj.SetAttributes(attrs...)
}
func checkExpiredObjects(t *testing.T, db *meta.DB, f func(exp, nonExp *objectSDK.Object)) {
expObj := generateObject(t)
expObj := testutil.GenerateObject()
setExpiration(expObj, currEpoch-1)
require.NoError(t, metaPut(db, expObj, nil))
nonExpObj := generateObject(t)
nonExpObj := testutil.GenerateObject()
setExpiration(nonExpObj, currEpoch)
require.NoError(t, metaPut(db, nonExpObj, nil))

15
pkg/local_object_storage/metabase/delete_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -18,10 +19,10 @@ func TestDB_Delete(t *testing.T) {
db := newDB(t)
cnr := cidtest.ID()
parent := generateObjectWithCID(t, cnr)
addAttribute(parent, "foo", "bar")
parent := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(parent, "foo", "bar")
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
@ -44,7 +45,7 @@ func TestDB_Delete(t *testing.T) {
require.NoError(t, err)
// inhume parent and child so they will be on graveyard
ts := generateObjectWithCID(t, cnr)
ts := testutil.GenerateObjectWithCID(cnr)
err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts))
require.NoError(t, err)
@ -75,15 +76,15 @@ func TestDeleteAllChildren(t *testing.T) {
cnr := cidtest.ID()
// generate parent object
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
// generate 2 children
child1 := generateObjectWithCID(t, cnr)
child1 := testutil.GenerateObjectWithCID(cnr)
child1.SetParent(parent)
idParent, _ := parent.ID()
child1.SetParentID(idParent)
child2 := generateObjectWithCID(t, cnr)
child2 := testutil.GenerateObjectWithCID(cnr)
child2.SetParent(parent)
child2.SetParentID(idParent)

23
pkg/local_object_storage/metabase/exists_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -19,14 +20,14 @@ func TestDB_Exists(t *testing.T) {
db := newDB(t, meta.WithEpochState(epochState{currEpoch}))
t.Run("no object", func(t *testing.T) {
nonExist := generateObject(t)
nonExist := testutil.GenerateObject()
exists, err := metaExists(db, object.AddressOf(nonExist))
require.NoError(t, err)
require.False(t, exists)
})
t.Run("regular object", func(t *testing.T) {
regular := generateObject(t)
regular := testutil.GenerateObject()
err := putBig(db, regular)
require.NoError(t, err)
@ -45,7 +46,7 @@ func TestDB_Exists(t *testing.T) {
})
t.Run("tombstone object", func(t *testing.T) {
ts := generateObject(t)
ts := testutil.GenerateObject()
ts.SetType(objectSDK.TypeTombstone)
err := putBig(db, ts)
@ -57,7 +58,7 @@ func TestDB_Exists(t *testing.T) {
})
t.Run("storage group object", func(t *testing.T) {
sg := generateObject(t)
sg := testutil.GenerateObject()
sg.SetType(objectSDK.TypeStorageGroup)
err := putBig(db, sg)
@ -69,7 +70,7 @@ func TestDB_Exists(t *testing.T) {
})
t.Run("lock object", func(t *testing.T) {
lock := generateObject(t)
lock := testutil.GenerateObject()
lock.SetType(objectSDK.TypeLock)
err := putBig(db, lock)
@ -82,9 +83,9 @@ func TestDB_Exists(t *testing.T) {
t.Run("virtual object", func(t *testing.T) {
cnr := cidtest.ID()
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
@ -102,16 +103,16 @@ func TestDB_Exists(t *testing.T) {
cnr := cidtest.ID()
splitID := objectSDK.NewSplitID()
parent := generateObjectWithCID(t, cnr)
addAttribute(parent, "foo", "bar")
parent := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(parent, "foo", "bar")
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
child.SetSplitID(splitID)
link := generateObjectWithCID(t, cnr)
link := testutil.GenerateObjectWithCID(cnr)
link.SetParent(parent)
link.SetParentID(idParent)
idChild, _ := child.ID()

13
pkg/local_object_storage/metabase/get_test.go
View file

@ -9,6 +9,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -21,11 +22,11 @@ import (
func TestDB_Get(t *testing.T) {
db := newDB(t, meta.WithEpochState(epochState{currEpoch}))
raw := generateObject(t)
raw := testutil.GenerateObject()
// equal fails on diff of <nil> attributes and <{}> attributes,
/* so we make non empty attribute slice in parent*/
addAttribute(raw, "foo", "bar")
testutil.AddAttribute(raw, "foo", "bar")
t.Run("object not found", func(t *testing.T) {
_, err := metaGet(db, object.AddressOf(raw), false)
@ -81,10 +82,10 @@ func TestDB_Get(t *testing.T) {
cnr := cidtest.ID()
splitID := objectSDK.NewSplitID()
parent := generateObjectWithCID(t, cnr)
addAttribute(parent, "foo", "bar")
parent := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(parent, "foo", "bar")
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
@ -195,7 +196,7 @@ func benchmarkGet(b *testing.B, numOfObj int) {
addrs := make([]oid.Address, 0, numOfObj)
for i := 0; i < numOfObj; i++ {
raw := generateObject(b)
raw := testutil.GenerateObject()
addrs = append(addrs, object.AddressOf(raw))
err := putBig(db, raw)

33
pkg/local_object_storage/metabase/graveyard_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
@ -39,8 +40,8 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) {
func TestDB_Iterate_OffsetNotFound(t *testing.T) {
db := newDB(t)
obj1 := generateObject(t)
obj2 := generateObject(t)
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
var addr1 oid.Address
err := addr1.DecodeString("AUSF6rhReoAdPVKYUZWW9o2LbtTvekn54B3JXi7pdzmn/2daLhLB7yVXbjBaKkckkuvjX22BxRYuSHy9RPxuH9PZS")
@ -110,10 +111,10 @@ func TestDB_IterateDeletedObjects(t *testing.T) {
db := newDB(t)
// generate and put 4 objects
obj1 := generateObject(t)
obj2 := generateObject(t)
obj3 := generateObject(t)
obj4 := generateObject(t)
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
obj3 := testutil.GenerateObject()
obj4 := testutil.GenerateObject()
var err error
@ -196,10 +197,10 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) {
db := newDB(t)
// generate and put 4 objects
obj1 := generateObject(t)
obj2 := generateObject(t)
obj3 := generateObject(t)
obj4 := generateObject(t)
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
obj3 := testutil.GenerateObject()
obj4 := testutil.GenerateObject()
var err error
@ -294,10 +295,10 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) {
db := newDB(t)
// generate and put 4 objects
obj1 := generateObject(t)
obj2 := generateObject(t)
obj3 := generateObject(t)
obj4 := generateObject(t)
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
obj3 := testutil.GenerateObject()
obj4 := testutil.GenerateObject()
var err error
@ -385,8 +386,8 @@ func TestDB_DropGraves(t *testing.T) {
db := newDB(t)
// generate and put 2 objects
obj1 := generateObject(t)
obj2 := generateObject(t)
obj1 := testutil.GenerateObject()
obj2 := testutil.GenerateObject()
var err error

5
pkg/local_object_storage/metabase/inhume_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -14,8 +15,8 @@ import (
func TestDB_Inhume(t *testing.T) {
db := newDB(t)
raw := generateObject(t)
addAttribute(raw, "foo", "bar")
raw := testutil.GenerateObject()
testutil.AddAttribute(raw, "foo", "bar")
tombstoneID := oidtest.Address()

5
pkg/local_object_storage/metabase/iterators_test.go
View file

@ -6,6 +6,7 @@ import (
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -56,9 +57,9 @@ func TestDB_IterateExpired(t *testing.T) {
}
func putWithExpiration(t *testing.T, db *meta.DB, typ object.Type, expiresAt uint64) oid.Address {
obj := generateObject(t)
obj := testutil.GenerateObject()
obj.SetType(typ)
addAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(expiresAt, 10))
testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(expiresAt, 10))
require.NoError(t, putBig(db, obj))

25
pkg/local_object_storage/metabase/list_test.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -32,7 +33,7 @@ func listWithCursorPrepareDB(b *testing.B) *meta.DB {
NoSync: true,
})) // faster single-thread generation
obj := generateObject(b)
obj := testutil.GenerateObject()
for i := 0; i < 100_000; i++ { // should be a multiple of all batch sizes
obj.SetID(oidtest.ID())
if i%9 == 0 { // let's have 9 objects per container
@ -79,47 +80,47 @@ func TestLisObjectsWithCursor(t *testing.T) {
containerID := cidtest.ID()
// add one regular object
obj := generateObjectWithCID(t, containerID)
obj := testutil.GenerateObjectWithCID(containerID)
obj.SetType(objectSDK.TypeRegular)
err := putBig(db, obj)
require.NoError(t, err)
expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeRegular})
// add one tombstone
obj = generateObjectWithCID(t, containerID)
obj = testutil.GenerateObjectWithCID(containerID)
obj.SetType(objectSDK.TypeTombstone)
err = putBig(db, obj)
require.NoError(t, err)
expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeTombstone})
// add one storage group
obj = generateObjectWithCID(t, containerID)
obj = testutil.GenerateObjectWithCID(containerID)
obj.SetType(objectSDK.TypeStorageGroup)
err = putBig(db, obj)
require.NoError(t, err)
expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeStorageGroup})
// add one lock
obj = generateObjectWithCID(t, containerID)
obj = testutil.GenerateObjectWithCID(containerID)
obj.SetType(objectSDK.TypeLock)
err = putBig(db, obj)
require.NoError(t, err)
expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeLock})
// add one inhumed (do not include into expected)
obj = generateObjectWithCID(t, containerID)
obj = testutil.GenerateObjectWithCID(containerID)
obj.SetType(objectSDK.TypeRegular)
err = putBig(db, obj)
require.NoError(t, err)
ts := generateObjectWithCID(t, containerID)
ts := testutil.GenerateObjectWithCID(containerID)
err = metaInhume(db, object.AddressOf(obj), object.AddressOf(ts))
require.NoError(t, err)
// add one child object (do not include parent into expected)
splitID := objectSDK.NewSplitID()
parent := generateObjectWithCID(t, containerID)
addAttribute(parent, "foo", "bar")
child := generateObjectWithCID(t, containerID)
parent := testutil.GenerateObjectWithCID(containerID)
testutil.AddAttribute(parent, "foo", "bar")
child := testutil.GenerateObjectWithCID(containerID)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
@ -173,7 +174,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) {
// fill metabase with objects
for i := 0; i < total; i++ {
obj := generateObject(t)
obj := testutil.GenerateObject()
err := putBig(db, obj)
require.NoError(t, err)
expected[object.AddressOf(obj).EncodeToString()] = 0
@ -190,7 +191,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) {
// add new objects
for i := 0; i < total; i++ {
obj := generateObject(t)
obj := testutil.GenerateObject()
err = putBig(db, obj)
require.NoError(t, err)
}

5
pkg/local_object_storage/metabase/lock_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -242,7 +243,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob
lockedObjIDs := make([]oid.ID, 0, numOfLockedObjs)
for i := 0; i < numOfLockedObjs; i++ {
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
err := putBig(db, obj)
require.NoError(t, err)
@ -252,7 +253,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob
lockedObjIDs = append(lockedObjIDs, id)
}
lockObj := generateObjectWithCID(t, cnr)
lockObj := testutil.GenerateObjectWithCID(cnr)
lockID, _ := lockObj.ID()
lockObj.SetType(object.TypeLock)

5
pkg/local_object_storage/metabase/movable_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/stretchr/testify/require"
@ -12,8 +13,8 @@ import (
func TestDB_Movable(t *testing.T) {
db := newDB(t)
raw1 := generateObject(t)
raw2 := generateObject(t)
raw1 := testutil.GenerateObject()
raw2 := testutil.GenerateObject()
// put two objects in metabase
err := putBig(db, raw1)

7
pkg/local_object_storage/metabase/put_test.go
View file

@ -7,6 +7,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -21,7 +22,7 @@ func prepareObjects(t testing.TB, n int) []*objectSDK.Object {
parentID := objecttest.ID()
objs := make([]*objectSDK.Object, n)
for i := range objs {
objs[i] = generateObjectWithCID(t, cnr)
objs[i] = testutil.GenerateObjectWithCID(cnr)
// FKBT indices.
attrs := make([]objectSDK.Attribute, 20)
@ -78,7 +79,7 @@ func BenchmarkPut(b *testing.B) {
func TestDB_PutBlobovnicaUpdate(t *testing.T) {
db := newDB(t)
raw1 := generateObject(t)
raw1 := testutil.GenerateObject()
storageID := []byte{1, 2, 3, 4}
// put one object with storageID
@ -101,7 +102,7 @@ func TestDB_PutBlobovnicaUpdate(t *testing.T) {
})
t.Run("update storageID on bad object", func(t *testing.T) {
raw2 := generateObject(t)
raw2 := testutil.GenerateObject()
err := putBig(db, raw2)
require.NoError(t, err)

81
pkg/local_object_storage/metabase/select_test.go
View file

@ -7,6 +7,7 @@ import (
v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -22,40 +23,40 @@ func TestDB_SelectUserAttributes(t *testing.T) {
cnr := cidtest.ID()
raw1 := generateObjectWithCID(t, cnr)
addAttribute(raw1, "foo", "bar")
addAttribute(raw1, "x", "y")
raw1 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw1, "foo", "bar")
testutil.AddAttribute(raw1, "x", "y")
err := putBig(db, raw1)
require.NoError(t, err)
raw2 := generateObjectWithCID(t, cnr)
addAttribute(raw2, "foo", "bar")
addAttribute(raw2, "x", "z")
raw2 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw2, "foo", "bar")
testutil.AddAttribute(raw2, "x", "z")
err = putBig(db, raw2)
require.NoError(t, err)
raw3 := generateObjectWithCID(t, cnr)
addAttribute(raw3, "a", "b")
raw3 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw3, "a", "b")
err = putBig(db, raw3)
require.NoError(t, err)
raw4 := generateObjectWithCID(t, cnr)
addAttribute(raw4, "path", "test/1/2")
raw4 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw4, "path", "test/1/2")
err = putBig(db, raw4)
require.NoError(t, err)
raw5 := generateObjectWithCID(t, cnr)
addAttribute(raw5, "path", "test/1/3")
raw5 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw5, "path", "test/1/3")
err = putBig(db, raw5)
require.NoError(t, err)
raw6 := generateObjectWithCID(t, cnr)
addAttribute(raw6, "path", "test/2/3")
raw6 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw6, "path", "test/2/3")
err = putBig(db, raw6)
require.NoError(t, err)
@ -146,40 +147,40 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
// prepare
small := generateObjectWithCID(t, cnr)
small := testutil.GenerateObjectWithCID(cnr)
err := putBig(db, small)
require.NoError(t, err)
ts := generateObjectWithCID(t, cnr)
ts := testutil.GenerateObjectWithCID(cnr)
ts.SetType(objectSDK.TypeTombstone)
err = putBig(db, ts)
require.NoError(t, err)
sg := generateObjectWithCID(t, cnr)
sg := testutil.GenerateObjectWithCID(cnr)
sg.SetType(objectSDK.TypeStorageGroup)
err = putBig(db, sg)
require.NoError(t, err)
leftChild := generateObjectWithCID(t, cnr)
leftChild := testutil.GenerateObjectWithCID(cnr)
leftChild.InitRelations()
err = putBig(db, leftChild)
require.NoError(t, err)
lock := generateObjectWithCID(t, cnr)
lock := testutil.GenerateObjectWithCID(cnr)
lock.SetType(objectSDK.TypeLock)
err = putBig(db, lock)
require.NoError(t, err)
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
rightChild := generateObjectWithCID(t, cnr)
rightChild := testutil.GenerateObjectWithCID(cnr)
rightChild.SetParent(parent)
idParent, _ := parent.ID()
rightChild.SetParentID(idParent)
err = putBig(db, rightChild)
require.NoError(t, err)
link := generateObjectWithCID(t, cnr)
link := testutil.GenerateObjectWithCID(cnr)
link.SetParent(parent)
link.SetParentID(idParent)
idLeftChild, _ := leftChild.ID()
@ -326,11 +327,11 @@ func TestDB_SelectInhume(t *testing.T) {
cnr := cidtest.ID()
raw1 := generateObjectWithCID(t, cnr)
raw1 := testutil.GenerateObjectWithCID(cnr)
err := putBig(db, raw1)
require.NoError(t, err)
raw2 := generateObjectWithCID(t, cnr)
raw2 := testutil.GenerateObjectWithCID(cnr)
err = putBig(db, raw2)
require.NoError(t, err)
@ -358,11 +359,11 @@ func TestDB_SelectPayloadHash(t *testing.T) {
cnr := cidtest.ID()
raw1 := generateObjectWithCID(t, cnr)
raw1 := testutil.GenerateObjectWithCID(cnr)
err := putBig(db, raw1)
require.NoError(t, err)
raw2 := generateObjectWithCID(t, cnr)
raw2 := testutil.GenerateObjectWithCID(cnr)
err = putBig(db, raw2)
require.NoError(t, err)
@ -433,14 +434,14 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
v21.SetMajor(2)
v21.SetMinor(1)
raw1 := generateObjectWithCID(t, cnr)
raw1 := testutil.GenerateObjectWithCID(cnr)
raw1.SetPayloadSize(10)
raw1.SetCreationEpoch(11)
raw1.SetVersion(v20)
err := putBig(db, raw1)
require.NoError(t, err)
raw2 := generateObjectWithCID(t, cnr)
raw2 := testutil.GenerateObjectWithCID(cnr)
raw2.SetPayloadSize(20)
raw2.SetCreationEpoch(21)
raw2.SetVersion(&v21)
@ -533,9 +534,9 @@ func TestDB_SelectObjectID(t *testing.T) {
// prepare
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
regular := generateObjectWithCID(t, cnr)
regular := testutil.GenerateObjectWithCID(cnr)
idParent, _ := parent.ID()
regular.SetParentID(idParent)
regular.SetParent(parent)
@ -543,23 +544,23 @@ func TestDB_SelectObjectID(t *testing.T) {
err := putBig(db, regular)
require.NoError(t, err)
ts := generateObjectWithCID(t, cnr)
ts := testutil.GenerateObjectWithCID(cnr)
ts.SetType(objectSDK.TypeTombstone)
err = putBig(db, ts)
require.NoError(t, err)
sg := generateObjectWithCID(t, cnr)
sg := testutil.GenerateObjectWithCID(cnr)
sg.SetType(objectSDK.TypeStorageGroup)
err = putBig(db, sg)
require.NoError(t, err)
lock := generateObjectWithCID(t, cnr)
lock := testutil.GenerateObjectWithCID(cnr)
lock.SetType(objectSDK.TypeLock)
err = putBig(db, lock)
require.NoError(t, err)
t.Run("not found objects", func(t *testing.T) {
raw := generateObjectWithCID(t, cnr)
raw := testutil.GenerateObjectWithCID(cnr)
id, _ := raw.ID()
@ -671,9 +672,9 @@ func TestDB_SelectSplitID(t *testing.T) {
cnr := cidtest.ID()
child1 := generateObjectWithCID(t, cnr)
child2 := generateObjectWithCID(t, cnr)
child3 := generateObjectWithCID(t, cnr)
child1 := testutil.GenerateObjectWithCID(cnr)
child2 := testutil.GenerateObjectWithCID(cnr)
child3 := testutil.GenerateObjectWithCID(cnr)
split1 := objectSDK.NewSplitID()
split2 := objectSDK.NewSplitID()
@ -725,11 +726,11 @@ func TestDB_SelectContainerID(t *testing.T) {
cnr := cidtest.ID()
obj1 := generateObjectWithCID(t, cnr)
obj1 := testutil.GenerateObjectWithCID(cnr)
err := putBig(db, obj1)
require.NoError(t, err)
obj2 := generateObjectWithCID(t, cnr)
obj2 := testutil.GenerateObjectWithCID(cnr)
err = putBig(db, obj2)
require.NoError(t, err)
@ -775,7 +776,7 @@ func BenchmarkSelect(b *testing.B) {
var attr objectSDK.Attribute
attr.SetKey("myHeader")
attr.SetValue(strconv.Itoa(i))
obj := generateObjectWithCID(b, cid)
obj := testutil.GenerateObjectWithCID(cid)
obj.SetAttributes(attr)
require.NoError(b, metaPut(db, obj, nil))
}

5
pkg/local_object_storage/metabase/storage_id_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/stretchr/testify/require"
@ -12,8 +13,8 @@ import (
func TestDB_StorageID(t *testing.T) {
db := newDB(t)
raw1 := generateObject(t)
raw2 := generateObject(t)
raw1 := testutil.GenerateObject()
raw2 := testutil.GenerateObject()
storageID := []byte{1, 2, 3, 4}

13
pkg/local_object_storage/shard/delete_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -26,14 +27,14 @@ func testShardDelete(t *testing.T, hasWriteCache bool) {
cnr := cidtest.ID()
obj := generateObjectWithCID(t, cnr)
addAttribute(obj, "foo", "bar")
obj := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(obj, "foo", "bar")
var putPrm shard.PutPrm
var getPrm shard.GetPrm
t.Run("big object", func(t *testing.T) {
addPayload(obj, 1<<20)
testutil.AddPayload(obj, 1<<20)
putPrm.SetObject(obj)
getPrm.SetAddress(object.AddressOf(obj))
@ -55,9 +56,9 @@ func testShardDelete(t *testing.T, hasWriteCache bool) {
})
t.Run("small object", func(t *testing.T) {
obj := generateObjectWithCID(t, cnr)
addAttribute(obj, "foo", "bar")
addPayload(obj, 1<<5)
obj := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(obj, "foo", "bar")
testutil.AddPayload(obj, 1<<5)
putPrm.SetObject(obj)
getPrm.SetAddress(object.AddressOf(obj))

7
pkg/local_object_storage/shard/dump_test.go
View file

@ -14,6 +14,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
@ -97,7 +98,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) {
}
data := make([]byte, size)
rand.Read(data)
obj := generateObjectWithPayload(cnr, data)
obj := testutil.GenerateObjectWithCIDWithPayload(cnr, data)
objects[i] = obj
var prm shard.PutPrm
@ -227,7 +228,7 @@ func TestStream(t *testing.T) {
objects := make([]*objectSDK.Object, objCount)
for i := 0; i < objCount; i++ {
cnr := cidtest.ID()
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
objects[i] = obj
var prm shard.PutPrm
@ -326,7 +327,7 @@ func TestDumpIgnoreErrors(t *testing.T) {
objects := make([]*objectSDK.Object, objCount)
for i := 0; i < objCount; i++ {
size := (wcSmallObjectSize << (i % 4)) - headerSize
obj := generateObjectWithPayload(cidtest.ID(), make([]byte, size))
obj := testutil.GenerateObjectWithPayload(cidtest.ID(), make([]byte, size))
objects[i] = obj
var prm shard.PutPrm

25
pkg/local_object_storage/shard/get_test.go
View file

@ -7,6 +7,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -32,9 +33,9 @@ func testShardGet(t *testing.T, hasWriteCache bool) {
var getPrm shard.GetPrm
t.Run("small object", func(t *testing.T) {
obj := generateObject(t)
addAttribute(obj, "foo", "bar")
addPayload(obj, 1<<5)
obj := testutil.GenerateObject()
testutil.AddAttribute(obj, "foo", "bar")
testutil.AddPayload(obj, 1<<5)
putPrm.SetObject(obj)
@ -49,10 +50,10 @@ func testShardGet(t *testing.T, hasWriteCache bool) {
})
t.Run("big object", func(t *testing.T) {
obj := generateObject(t)
addAttribute(obj, "foo", "bar")
obj := testutil.GenerateObject()
testutil.AddAttribute(obj, "foo", "bar")
obj.SetID(oidtest.ID())
addPayload(obj, 1<<20) // big obj
testutil.AddPayload(obj, 1<<20) // big obj
putPrm.SetObject(obj)
@ -67,20 +68,20 @@ func testShardGet(t *testing.T, hasWriteCache bool) {
})
t.Run("parent object", func(t *testing.T) {
obj := generateObject(t)
addAttribute(obj, "foo", "bar")
obj := testutil.GenerateObject()
testutil.AddAttribute(obj, "foo", "bar")
cnr := cidtest.ID()
splitID := objectSDK.NewSplitID()
parent := generateObjectWithCID(t, cnr)
addAttribute(parent, "parent", "attribute")
parent := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(parent, "parent", "attribute")
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)
child.SetSplitID(splitID)
addPayload(child, 1<<5)
testutil.AddPayload(child, 1<<5)
putPrm.SetObject(child)

11
pkg/local_object_storage/shard/head_test.go
View file

@ -6,6 +6,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -30,8 +31,8 @@ func testShardHead(t *testing.T, hasWriteCache bool) {
var headPrm shard.HeadPrm
t.Run("regular object", func(t *testing.T) {
obj := generateObject(t)
addAttribute(obj, "foo", "bar")
obj := testutil.GenerateObject()
testutil.AddAttribute(obj, "foo", "bar")
putPrm.SetObject(obj)
@ -49,10 +50,10 @@ func testShardHead(t *testing.T, hasWriteCache bool) {
cnr := cidtest.ID()
splitID := objectSDK.NewSplitID()
parent := generateObjectWithCID(t, cnr)
addAttribute(parent, "foo", "bar")
parent := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(parent, "foo", "bar")
child := generateObjectWithCID(t, cnr)
child := testutil.GenerateObjectWithCID(cnr)
child.SetParent(parent)
idParent, _ := parent.ID()
child.SetParentID(idParent)

7
pkg/local_object_storage/shard/inhume_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -26,10 +27,10 @@ func testShardInhume(t *testing.T, hasWriteCache bool) {
cnr := cidtest.ID()
obj := generateObjectWithCID(t, cnr)
addAttribute(obj, "foo", "bar")
obj := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(obj, "foo", "bar")
ts := generateObjectWithCID(t, cnr)
ts := testutil.GenerateObjectWithCID(cnr)
var putPrm shard.PutPrm
putPrm.SetObject(obj)

7
pkg/local_object_storage/shard/list_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"github.com/stretchr/testify/require"
@ -38,11 +39,11 @@ func testShardList(t *testing.T, sh *shard.Shard) {
cnr := cidtest.ID()
for j := 0; j < N; j++ {
obj := generateObjectWithCID(t, cnr)
addPayload(obj, 1<<2)
obj := testutil.GenerateObjectWithCID(cnr)
testutil.AddPayload(obj, 1<<2)
// add parent as virtual object, it must be ignored in List()
parent := generateObjectWithCID(t, cnr)
parent := testutil.GenerateObjectWithCID(cnr)
idParent, _ := parent.ID()
obj.SetParentID(idParent)
obj.SetParent(parent)

11
pkg/local_object_storage/shard/lock_test.go
View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
@ -62,10 +63,10 @@ func TestShard_Lock(t *testing.T) {
})
cnr := cidtest.ID()
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
objID, _ := obj.ID()
lock := generateObjectWithCID(t, cnr)
lock := testutil.GenerateObjectWithCID(cnr)
lock.SetType(object.TypeLock)
lockID, _ := lock.ID()
@ -87,7 +88,7 @@ func TestShard_Lock(t *testing.T) {
require.NoError(t, err)
t.Run("inhuming locked objects", func(t *testing.T) {
ts := generateObjectWithCID(t, cnr)
ts := testutil.GenerateObjectWithCID(cnr)
var inhumePrm shard.InhumePrm
inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(obj))
@ -101,7 +102,7 @@ func TestShard_Lock(t *testing.T) {
})
t.Run("inhuming lock objects", func(t *testing.T) {
ts := generateObjectWithCID(t, cnr)
ts := testutil.GenerateObjectWithCID(cnr)
var inhumePrm shard.InhumePrm
inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(lock))
@ -145,7 +146,7 @@ func TestShard_IsLocked(t *testing.T) {
sh := newShard(t, false)
cnr := cidtest.ID()
obj := generateObjectWithCID(t, cnr)
obj := testutil.GenerateObjectWithCID(cnr)
cnrID, _ := obj.ContainerID()
objID, _ := obj.ID()

5
pkg/local_object_storage/shard/metrics_test.go
View file

@ -7,6 +7,7 @@ import (
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
@ -82,7 +83,7 @@ func TestCounters(t *testing.T) {
const objNumber = 10
oo := make([]*object.Object, objNumber)
for i := 0; i < objNumber; i++ {
oo[i] = generateObject(t)
oo[i] = testutil.GenerateObject()
}
t.Run("defaults", func(t *testing.T) {
@ -139,7 +140,7 @@ func TestCounters(t *testing.T) {
t.Run("inhume_TS", func(t *testing.T) {
var prm shard.InhumePrm
ts := objectcore.AddressOf(generateObject(t))
ts := objectcore.AddressOf(testutil.GenerateObject())
phy := mm.objCounters[physical]
logic := mm.objCounters[logical]

7
pkg/local_object_storage/shard/range_test.go
View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
@ -87,9 +88,9 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
obj := generateObject(t)
addAttribute(obj, "foo", "bar")
addPayload(obj, tc.payloadSize)
obj := testutil.GenerateObject()
testutil.AddAttribute(obj, "foo", "bar")
testutil.AddPayload(obj, tc.payloadSize)
addr := object.AddressOf(obj)
payload := slice.Copy(obj.Payload())

60
pkg/local_object_storage/shard/shard_test.go
View file

@ -1,8 +1,6 @@
package shard_test
import (
"crypto/sha256"
"math/rand"
"path/filepath"
"testing"
@ -14,14 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"git.frostfs.info/TrueCloudLab/tzhash/tz"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
@ -97,54 +88,3 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
func releaseShard(s *shard.Shard, t testing.TB) {
require.NoError(t, s.Close())
}
func generateObject(t *testing.T) *object.Object {
return generateObjectWithCID(t, cidtest.ID())
}
func generateObjectWithCID(t *testing.T, cnr cid.ID) *object.Object {
data := make([]byte, 32)
rand.Read(data)
return generateObjectWithPayload(cnr, data)
}
func generateObjectWithPayload(cnr cid.ID, data []byte) *object.Object {
var ver version.Version
ver.SetMajor(2)
ver.SetMinor(1)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(data))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(oidtest.ID())
obj.SetOwnerID(usertest.ID())
obj.SetContainerID(cnr)
obj.SetVersion(&ver)
obj.SetPayload(data)
obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ)
return obj
}
func addAttribute(obj *object.Object, key, val string) {
var attr object.Attribute
attr.SetKey(key)
attr.SetValue(val)
attrs := obj.Attributes()
attrs = append(attrs, attr)
obj.SetAttributes(attrs...)
}
func addPayload(obj *object.Object, size int) {
buf := make([]byte, size)
_, _ = rand.Read(buf)
obj.SetPayload(buf)
obj.SetPayloadSize(uint64(size))
}

3
pkg/local_object_storage/shard/shutdown_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
@ -27,7 +28,7 @@ func TestWriteCacheObjectLoss(t *testing.T) {
data := make([]byte, size)
rand.Read(data)
objects[i] = generateObjectWithPayload(cidtest.ID(), data)
objects[i] = testutil.GenerateObjectWithCIDWithPayload(cidtest.ID(), data)
}
dir := t.TempDir()