frostfs-node/pkg/local_object_storage/engine/container_stat_test.go

package engine

import (
	"context"
	"testing"

	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
	"github.com/stretchr/testify/require"
)

func TestContainerList(t *testing.T) {
	t.Parallel()

	s1 := testNewShard(t)
	s2 := testNewShard(t)
	s3 := testNewShard(t)

	e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine
	e.log = test.NewLogger(t)
	defer e.Close(context.Background())

	const containerCount = 10
	expStat := testPutComplexObject(t, []*shard.Shard{s1, s2, s3}, containerCount, nil)
	expStat = testPutSimpleObject(t, []*shard.Shard{s1, s2, s3}, containerCount, expStat)
	expStat = testPutSimpleObject(t, []*shard.Shard{s1, s2}, containerCount, expStat)
	expStat = testPutSimpleObject(t, []*shard.Shard{s2, s3}, containerCount, expStat)
	expStat = testPutSimpleObject(t, []*shard.Shard{s1}, containerCount, expStat)
	expStat = testPutSimpleObject(t, []*shard.Shard{s2}, containerCount, expStat)
	expStat = testPutSimpleObject(t, []*shard.Shard{s3}, containerCount, expStat)

	t.Run("with default limit", func(t *testing.T) {
		var prm ContainerStatPrm
		prm.Limit = 10_000
		res, err := e.ContainerStat(context.Background(), prm)
		require.NoError(t, err)
		require.NotNil(t, res)
		require.ElementsMatch(t, expStat, res.ContainerStats)
		require.False(t, res.Partial)
	})

	t.Run("with limit, batched", func(t *testing.T) {
		var prm ContainerStatPrm
		prm.Limit = 1
		var stats []ContainerStat

		for {
			res, err := e.ContainerStat(context.Background(), prm)
			require.NoError(t, err)
			require.NotNil(t, res)
			require.False(t, res.Partial)
			if len(res.ContainerStats) == 0 {
				break
			}
			stats = append(stats, res.ContainerStats...)
			last := res.ContainerStats[len(res.ContainerStats)-1].ContainerID
			prm.StartFromContainerID = &last
			prm.Limit += 1
		}

		require.ElementsMatch(t, expStat, stats)
	})

	t.Run("by container id", func(t *testing.T) {
		for _, cc := range []int{1, 2, 3, 4, 5} {
			var prm ContainerStatPrm
			for idx := 0; idx+cc < len(expStat); idx += cc {
				prm.ContainerID = nil
				for i := 0; i < cc; i++ {
					prm.ContainerID = append(prm.ContainerID, expStat[idx+i].ContainerID)
				}
				prm.Limit = uint32(len(prm.ContainerID))
				res, err := e.ContainerStat(context.Background(), prm)
				require.NoError(t, err)
				require.NotNil(t, res)
				require.False(t, res.Partial)
				require.ElementsMatch(t, expStat[idx:idx+cc], res.ContainerStats)
			}
		}
	})

	t.Run("unknown container id", func(t *testing.T) {
		var prm ContainerStatPrm
		prm.ContainerID = append(prm.ContainerID, cidtest.ID())
		prm.Limit = uint32(len(prm.ContainerID))
		res, err := e.ContainerStat(context.Background(), prm)
		require.NoError(t, err)
		require.NotNil(t, res)
		require.False(t, res.Partial)
		require.ElementsMatch(t, []ContainerStat{{ContainerID: prm.ContainerID[0]}}, res.ContainerStats)
	})

	t.Run("degraded shard", func(t *testing.T) {
		s1.SetMode(mode.Degraded)
		var prm ContainerStatPrm
		prm.Limit = 10_000
		res, err := e.ContainerStat(context.Background(), prm)
		require.NoError(t, err)
		require.NotNil(t, res)
		require.True(t, res.Partial)
	})
}

func testPutComplexObject(t *testing.T, shards []*shard.Shard, count int, stats []ContainerStat) []ContainerStat {
	const payloadSize = 10 * 1024
	for count > 0 {
		var cnr cid.ID
		var stat *ContainerStat
		var newStat bool
		if len(stats) == 0 || count%2 == 0 {
			cnr = cidtest.ID()
			stat = &ContainerStat{ContainerID: cnr}
			newStat = true
		} else {
			cnr = stats[count%len(stats)].ContainerID
			stat = &stats[count%len(stats)]
		}

		parentID := oidtest.ID()
		splitID := objectSDK.NewSplitID()

		parent := testutil.GenerateObjectWithCID(cnr)
		parent.SetID(parentID)
		parent.SetPayload(nil)

		const childCount = 10
		children := make([]*objectSDK.Object, childCount)
		childIDs := make([]oid.ID, childCount)
		for i := range children {
			children[i] = testutil.GenerateObjectWithCID(cnr)
			if i != 0 {
				children[i].SetPreviousID(childIDs[i-1])
			}
			if i == len(children)-1 {
				children[i].SetParent(parent)
			}
			children[i].SetSplitID(splitID)
			children[i].SetPayload(make([]byte, payloadSize))
			children[i].SetPayloadSize(payloadSize)
			childIDs[i], _ = children[i].ID()

			stat.SizeLogic += payloadSize
			stat.CountLogic += 1
			stat.CountPhy += 1
		}
		stat.CountUser += 1

		link := testutil.GenerateObjectWithCID(cnr)
		link.SetParent(parent)
		link.SetParentID(parentID)
		link.SetSplitID(splitID)
		link.SetChildren(childIDs...)

		stat.CountLogic += 1
		stat.CountPhy += 1
		stat.SizeLogic += link.PayloadSize()

		for i := range children {
			sh := shards[i%len(shards)]
			var putPrm shard.PutPrm
			putPrm.SetObject(children[i])
			_, err := sh.Put(context.Background(), putPrm)
			require.NoError(t, err)
		}
		sh := shards[count%len(shards)]
		var putPrm shard.PutPrm
		putPrm.SetObject(link)
		_, err := sh.Put(context.Background(), putPrm)
		require.NoError(t, err)

		if newStat {
			stats = append(stats, *stat)
		}
		count--
	}
	return stats
}

func testPutSimpleObject(t *testing.T, shards []*shard.Shard, count int, stats []ContainerStat) []ContainerStat {
	const payloadSize = 7 * 1024
	for count > 0 {
		var cnr cid.ID
		var stat *ContainerStat
		var newStat bool
		if len(stats) == 0 || count%2 == 0 {
			cnr = cidtest.ID()
			stat = &ContainerStat{ContainerID: cnr}
			newStat = true
		} else {
			cnr = stats[count%len(stats)].ContainerID
			stat = &stats[count%len(stats)]
		}

		obj := testutil.GenerateObjectWithCID(cnr)
		obj.SetPayload(make([]byte, payloadSize))
		obj.SetPayloadSize(payloadSize)
		stat.SizeLogic += payloadSize
		stat.CountLogic += 1
		stat.CountPhy += 1
		stat.CountUser += 1

		sh := shards[count%len(shards)]
		var putPrm shard.PutPrm
		putPrm.SetObject(obj)
		_, err := sh.Put(context.Background(), putPrm)
		require.NoError(t, err)

		if newStat {
			stats = append(stats, *stat)
		}
		count--
	}
	return stats
}