//go:build integration package meta import ( "context" "fmt" "io" "os" "strconv" "testing" "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) const upgradeFilePath = "/path/to/metabase.v2" func TestUpgradeV2ToV3(t *testing.T) { path := createTempCopy(t, upgradeFilePath) defer func() { require.NoError(t, os.Remove(path)) }() db := New(WithPath(path), WithEpochState(epochState{e: 1000}), WithLogger(test.NewLogger(t))) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.ErrorIs(t, db.Init(), ErrOutdatedVersion) require.NoError(t, db.Close()) require.NoError(t, Upgrade(context.Background(), path, true, &testContainerInfoProvider{}, t.Log)) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) require.NoError(t, db.Close()) fmt.Println() } type testContainerInfoProvider struct{} func (p *testContainerInfoProvider) Info(id cid.ID) (container.Info, error) { return container.Info{}, nil } func createTempCopy(t *testing.T, path string) string { src, err := os.Open(path) require.NoError(t, err) tmpPath := upgradeFilePath + time.Now().Format(time.RFC3339) dest, err := os.Create(tmpPath) require.NoError(t, err) _, err = io.Copy(dest, src) require.NoError(t, err) require.NoError(t, src.Close()) require.NoError(t, dest.Close()) return tmpPath } func TestGenerateMetabaseFile(t *testing.T) { t.Skip("for generating db") const ( containersCount = 10_000 simpleObjectsCount = 500_000 complexObjectsCount = 500_000 // x2 deletedByGCMarksCount = 100_000 deletedByTombstoneCount = 100_000 // x2 lockedCount = 100_000 // x2 allocSize = 128 << 20 generateWorkersCount = 1_000 minEpoch = 1_000 maxFilename = 1_000 maxStorageID = 10_000 ) db := New(WithPath(upgradeFilePath), WithEpochState(epochState{e: minEpoch}), WithLogger(test.NewLogger(t))) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) db.boltDB.AllocSize = allocSize db.boltDB.NoSync = true require.NoError(t, db.Init()) containers := make([]cid.ID, containersCount) for i := range containers { containers[i] = cidtest.ID() } oc, err := db.ObjectCounters() require.NoError(t, err) require.True(t, oc.IsZero()) eg, ctx := errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects for i := range simpleObjectsCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), }) require.NoError(t, err) return nil }) } require.NoError(t, eg.Wait()) db.log.Info("simple objects generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // complex objects for i := range complexObjectsCount { i := i eg.Go(func() error { parent := testutil.GenerateObjectWithCID(containers[i%len(containers)]) child := testutil.GenerateObjectWithCID(containers[i%len(containers)]) child.SetParent(parent) idParent, _ := parent.ID() child.SetParentID(idParent) testutil.AddAttribute(child, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) testutil.AddAttribute(parent, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) testutil.AddAttribute(child, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) testutil.AddAttribute(parent, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) _, err := db.Put(ctx, PutPrm{ obj: child, }) require.NoError(t, err) return nil }) } require.NoError(t, eg.Wait()) db.log.Info("complex objects generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects deleted by gc marks for i := range deletedByGCMarksCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), }) require.NoError(t, err) _, err = db.Inhume(ctx, InhumePrm{ target: []oid.Address{object.AddressOf(obj)}, }) require.NoError(t, err) return nil }) } require.NoError(t, eg.Wait()) db.log.Info("simple objects deleted by gc marks generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(10000) // simple objects deleted by tombstones for i := range deletedByTombstoneCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), }) tomb := testutil.GenerateObjectWithCID(containers[i%len(containers)]) tomb.SetType(objectSDK.TypeTombstone) _, err = db.Put(ctx, PutPrm{ obj: tomb, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), }) require.NoError(t, err) tombAddr := object.AddressOf(tomb) _, err = db.Inhume(ctx, InhumePrm{ target: []oid.Address{object.AddressOf(obj)}, tomb: &tombAddr, }) require.NoError(t, err) return nil }) } require.NoError(t, eg.Wait()) db.log.Info("simple objects deleted by tombstones generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects locked by locks for i := range lockedCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), }) lock := testutil.GenerateObjectWithCID(containers[i%len(containers)]) lock.SetType(objectSDK.TypeLock) testutil.AddAttribute(lock, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) _, err = db.Put(ctx, PutPrm{ obj: lock, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), }) require.NoError(t, err) err = db.Lock(ctx, containers[i%len(containers)], object.AddressOf(lock).Object(), []oid.ID{object.AddressOf(obj).Object()}) require.NoError(t, err) return nil }) } require.NoError(t, eg.Wait()) db.log.Info("simple objects locked by locks generated") require.NoError(t, db.boltDB.Sync()) require.NoError(t, db.Close()) }