frostfs-node/pkg/local_object_storage/shard/control_test.go

package shard

import (
	"context"
	"fmt"
	"io/fs"
	"math"
	"os"
	"path/filepath"
	"sync/atomic"
	"testing"

	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
	objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
	"github.com/stretchr/testify/require"
	"go.etcd.io/bbolt"
)

type objAddr struct {
	obj  *objectSDK.Object
	addr oid.Address
}

func TestShardOpen(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	metaPath := filepath.Join(dir, "meta")

	st := teststore.New(teststore.WithSubstorage(fstree.New(
		fstree.WithDirNameLen(2),
		fstree.WithPath(filepath.Join(dir, "blob")),
		fstree.WithDepth(1)),
	))

	var allowedMode atomic.Int64
	openFileMetabase := func(p string, f int, perm fs.FileMode) (*os.File, error) {
		const modeMask = os.O_RDONLY | os.O_RDWR | os.O_WRONLY
		if int64(f&modeMask) == allowedMode.Load() {
			return os.OpenFile(p, f, perm)
		}
		return nil, fs.ErrPermission
	}

	wcOpts := []writecache.Option{
		writecache.WithPath(filepath.Join(dir, "wc")),
	}

	newShard := func() *Shard {
		return New(
			WithID(NewIDFromBytes([]byte{})),
			WithLogger(test.NewLogger(t)),
			WithBlobStorOptions(
				blobstor.WithStorages([]blobstor.SubStorage{
					{Storage: st},
				})),
			WithMetaBaseOptions(
				meta.WithPath(metaPath),
				meta.WithEpochState(epochState{}),
				meta.WithBoltDBOptions(&bbolt.Options{OpenFile: openFileMetabase}),
			),
			WithPiloramaOptions(
				pilorama.WithPath(filepath.Join(dir, "pilorama"))),
			WithWriteCache(true),
			WithWriteCacheOptions(wcOpts))
	}

	allowedMode.Store(int64(os.O_RDWR))

	sh := newShard()
	require.NoError(t, sh.Open(context.Background()))
	require.NoError(t, sh.Init(context.Background()))
	require.Equal(t, mode.ReadWrite, sh.GetMode())
	require.NoError(t, sh.Close(context.Background()))

	// Metabase can be opened in read-only => start in ReadOnly mode.
	allowedMode.Store(int64(os.O_RDONLY))

	sh = newShard()
	require.NoError(t, sh.Open(context.Background()))
	require.NoError(t, sh.Init(context.Background()))
	require.Equal(t, mode.ReadOnly, sh.GetMode())
	require.Error(t, sh.SetMode(context.Background(), mode.ReadWrite))
	require.Equal(t, mode.ReadOnly, sh.GetMode())
	require.NoError(t, sh.Close(context.Background()))

	// Metabase is corrupted => start in DegradedReadOnly mode.
	allowedMode.Store(math.MaxInt64)

	sh = newShard()
	require.NoError(t, sh.Open(context.Background()))
	require.NoError(t, sh.Init(context.Background()))
	require.Equal(t, mode.DegradedReadOnly, sh.GetMode())
	require.NoError(t, sh.Close(context.Background()))
}

func TestRefillMetabaseCorrupted(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()

	fsTree := fstree.New(
		fstree.WithDirNameLen(2),
		fstree.WithPath(filepath.Join(dir, "blob")),
		fstree.WithDepth(1))
	blobOpts := []blobstor.Option{
		blobstor.WithStorages([]blobstor.SubStorage{
			{
				Storage: fsTree,
			},
		}),
	}

	mm := newMetricStore()

	sh := New(
		WithID(NewIDFromBytes([]byte{})),
		WithBlobStorOptions(blobOpts...),
		WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
		WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{})),
		WithMetricsWriter(mm),
	)
	require.NoError(t, sh.Open(context.Background()))
	require.NoError(t, sh.Init(context.Background()))

	obj := objecttest.Object()
	obj.SetType(objectSDK.TypeRegular)
	obj.SetPayload([]byte{0, 1, 2, 3, 4, 5})

	var putPrm PutPrm
	putPrm.SetObject(obj)
	_, err := sh.Put(context.Background(), putPrm)
	require.NoError(t, err)
	require.NoError(t, sh.Close(context.Background()))

	addr := object.AddressOf(obj)
	// This is copied from `fstree.treePath()` to avoid exporting function just for tests.
	{
		saddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString()
		p := fmt.Sprintf("%s/%s/%s", fsTree.RootPath, saddr[:2], saddr[2:])
		require.NoError(t, os.WriteFile(p, []byte("not an object"), fsTree.Permissions))
	}

	sh = New(
		WithID(NewIDFromBytes([]byte{})),
		WithBlobStorOptions(blobOpts...),
		WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
		WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new")), meta.WithEpochState(epochState{})),
		WithRefillMetabase(true),
		WithMetricsWriter(mm))
	require.NoError(t, sh.Open(context.Background()))
	require.NoError(t, sh.Init(context.Background()))

	var getPrm GetPrm
	getPrm.SetAddress(addr)
	_, err = sh.Get(context.Background(), getPrm)
	require.True(t, client.IsErrObjectNotFound(err))
	require.NoError(t, sh.Close(context.Background()))
}

func TestRefillMetabase(t *testing.T) {
	t.Parallel()

	p := t.Name()

	defer os.RemoveAll(p)

	blobOpts := []blobstor.Option{
		blobstor.WithStorages([]blobstor.SubStorage{
			{
				Storage: fstree.New(
					fstree.WithPath(filepath.Join(p, "blob")),
					fstree.WithDepth(1)),
			},
		}),
	}

	mm := newMetricStore()

	sh := New(
		WithID(NewIDFromBytes([]byte{})),
		WithBlobStorOptions(blobOpts...),
		WithMetaBaseOptions(
			meta.WithPath(filepath.Join(p, "meta")),
			meta.WithEpochState(epochState{}),
		),
		WithPiloramaOptions(
			pilorama.WithPath(filepath.Join(p, "pilorama"))),
		WithMetricsWriter(mm),
	)

	// open Blobstor
	require.NoError(t, sh.Open(context.Background()))

	// initialize Blobstor
	require.NoError(t, sh.Init(context.Background()))

	const objNum = 5

	mObjs := make(map[string]objAddr)
	locked := make([]oid.ID, 1, 2)
	locked[0] = oidtest.ID()
	cnrLocked := cidtest.ID()
	for range objNum {
		obj := objecttest.Object()
		obj.SetType(objectSDK.TypeRegular)

		if len(locked) < 2 {
			obj.SetContainerID(cnrLocked)
			id, _ := obj.ID()
			locked = append(locked, id)
		}

		addr := object.AddressOf(obj)

		mObjs[addr.EncodeToString()] = objAddr{
			obj:  obj,
			addr: addr,
		}
	}

	tombObj := objecttest.Object()
	tombObj.SetType(objectSDK.TypeTombstone)

	tombstone := objecttest.Tombstone()

	tombData, err := tombstone.Marshal()
	require.NoError(t, err)

	tombObj.SetPayload(tombData)

	tombMembers := make([]oid.Address, 0, len(tombstone.Members()))

	members := tombstone.Members()
	for i := range tombstone.Members() {
		var a oid.Address
		a.SetObject(members[i])
		cnr, _ := tombObj.ContainerID()
		a.SetContainer(cnr)

		tombMembers = append(tombMembers, a)
	}

	var putPrm PutPrm

	for _, v := range mObjs {
		putPrm.SetObject(v.obj)

		_, err := sh.Put(context.Background(), putPrm)
		require.NoError(t, err)
	}

	putPrm.SetObject(tombObj)

	_, err = sh.Put(context.Background(), putPrm)
	require.NoError(t, err)

	// LOCK object handling
	var lock objectSDK.Lock
	lock.WriteMembers(locked)

	lockObj := objecttest.Object()
	lockObj.SetContainerID(cnrLocked)
	objectSDK.WriteLock(lockObj, lock)

	putPrm.SetObject(lockObj)
	_, err = sh.Put(context.Background(), putPrm)
	require.NoError(t, err)

	lockID, _ := lockObj.ID()
	require.NoError(t, sh.Lock(context.Background(), cnrLocked, lockID, locked))

	var inhumePrm InhumePrm
	inhumePrm.SetTarget(object.AddressOf(tombObj), tombMembers...)

	_, err = sh.Inhume(context.Background(), inhumePrm)
	require.NoError(t, err)

	var headPrm HeadPrm

	checkObj := func(addr oid.Address, expObj *objectSDK.Object) {
		headPrm.SetAddress(addr)

		res, err := sh.Head(context.Background(), headPrm)

		if expObj == nil {
			require.True(t, client.IsErrObjectNotFound(err))
			return
		}

		require.NoError(t, err)
		require.Equal(t, expObj.CutPayload(), res.Object())
	}

	checkAllObjs := func(exists bool) {
		for _, v := range mObjs {
			if exists {
				checkObj(v.addr, v.obj)
			} else {
				checkObj(v.addr, nil)
			}
		}
	}

	checkTombMembers := func(exists bool) {
		for _, member := range tombMembers {
			headPrm.SetAddress(member)

			_, err := sh.Head(context.Background(), headPrm)

			if exists {
				require.True(t, client.IsErrObjectAlreadyRemoved(err))
			} else {
				require.True(t, client.IsErrObjectNotFound(err))
			}
		}
	}

	checkLocked := func(t *testing.T, cnr cid.ID, locked []oid.ID) {
		var addr oid.Address
		addr.SetContainer(cnr)

		for i := range locked {
			addr.SetObject(locked[i])

			var prm InhumePrm
			prm.MarkAsGarbage(addr)

			var target *apistatus.ObjectLocked
			_, err := sh.Inhume(context.Background(), prm)
			require.ErrorAs(t, err, &target, "object %s should be locked", locked[i])
		}
	}

	checkAllObjs(true)
	checkObj(object.AddressOf(tombObj), tombObj)
	checkTombMembers(true)
	checkLocked(t, cnrLocked, locked)

	c, err := sh.metaBase.ObjectCounters()
	require.NoError(t, err)

	phyBefore := c.Phy
	logicalBefore := c.Logic

	err = sh.Close(context.Background())
	require.NoError(t, err)

	sh = New(
		WithID(NewIDFromBytes([]byte{})),
		WithBlobStorOptions(blobOpts...),
		WithMetaBaseOptions(
			meta.WithPath(filepath.Join(p, "meta_restored")),
			meta.WithEpochState(epochState{}),
		),
		WithPiloramaOptions(
			pilorama.WithPath(filepath.Join(p, "pilorama_another"))),
		WithMetricsWriter(mm),
	)

	// open Blobstor
	require.NoError(t, sh.Open(context.Background()))

	// initialize Blobstor
	require.NoError(t, sh.Init(context.Background()))

	defer sh.Close(context.Background())

	checkAllObjs(false)
	checkObj(object.AddressOf(tombObj), nil)
	checkTombMembers(false)

	err = sh.refillMetabase(context.Background())
	require.NoError(t, err)

	c, err = sh.metaBase.ObjectCounters()
	require.NoError(t, err)

	require.Equal(t, phyBefore, c.Phy)
	require.Equal(t, logicalBefore, c.Logic)

	checkAllObjs(true)
	checkObj(object.AddressOf(tombObj), tombObj)
	checkTombMembers(true)
	checkLocked(t, cnrLocked, locked)
	require.Equal(t, int64(len(mObjs)+2), mm.refillCount) // 1 lock + 1 tomb
	require.Equal(t, "completed", mm.refillStatus)
	require.Equal(t, uint32(100), mm.refillPercent)
}