frostfs-node/pkg/local_object_storage/shard/metrics_test.go

package shard

import (
	"context"
	"path/filepath"
	"sync"
	"testing"
	"time"

	objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
	"github.com/stretchr/testify/require"
)

type metricsStore struct {
	mtx           sync.Mutex
	objCounters   map[string]uint64
	cnrSize       map[string]int64
	cnrCount      map[string]uint64
	pldSize       int64
	mode          mode.Mode
	errCounter    int64
	refillCount   int64
	refillSize    int64
	refillPercent uint32
	refillStatus  string
}

func NewMetricStore() *metricsStore {
	return &metricsStore{
		objCounters: map[string]uint64{
			"phy":   0,
			"logic": 0,
		},
		cnrSize:  make(map[string]int64),
		cnrCount: make(map[string]uint64),
	}
}

func (m *metricsStore) SetShardID(_ string) {}

func (m *metricsStore) SetObjectCounter(objectType string, v uint64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.objCounters[objectType] = v
}

func (m *metricsStore) getObjectCounter(objectType string) uint64 {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	return m.objCounters[objectType]
}

func (m *metricsStore) containerSizes() map[string]int64 {
	m.mtx.Lock()
	defer m.mtx.Unlock()

	r := make(map[string]int64, len(m.cnrSize))
	for c, s := range m.cnrSize {
		r[c] = s
	}
	return r
}

func (m *metricsStore) payloadSize() int64 {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	return m.pldSize
}

func (m *metricsStore) AddToObjectCounter(objectType string, delta int) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	switch {
	case delta > 0:
		m.objCounters[objectType] += uint64(delta)
	case delta < 0:
		uDelta := uint64(-delta)

		if m.objCounters[objectType] >= uDelta {
			m.objCounters[objectType] -= uDelta
		} else {
			m.objCounters[objectType] = 0
		}
	case delta == 0:
		return
	}
}

func (m *metricsStore) IncObjectCounter(objectType string) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.objCounters[objectType] += 1
}

func (m *metricsStore) SetMode(mode mode.Mode) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.mode = mode
}

func (m *metricsStore) AddToContainerSize(cnr string, size int64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.cnrSize[cnr] += size
}

func (m *metricsStore) AddToPayloadSize(size int64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.pldSize += size
}

func (m *metricsStore) IncErrorCounter() {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.errCounter += 1
}

func (m *metricsStore) ClearErrorCounter() {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.errCounter = 0
}

func (m *metricsStore) DeleteShardMetrics() {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.errCounter = 0
}

func (m *metricsStore) SetContainerObjectsCount(cnrID string, objectType string, value uint64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.cnrCount[cnrID+objectType] = value
}

func (m *metricsStore) IncContainerObjectsCount(cnrID string, objectType string) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.cnrCount[cnrID+objectType]++
}

func (m *metricsStore) SubContainerObjectsCount(cnrID string, objectType string, value uint64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	existed := m.cnrCount[cnrID+objectType]
	if existed < value {
		panic("existed value smaller than value to sustract")
	}
	if existed == value {
		delete(m.cnrCount, cnrID+objectType)
	} else {
		m.cnrCount[cnrID+objectType] -= value
	}
}

func (m *metricsStore) getContainerCount(cnrID, objectType string) (uint64, bool) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	v, ok := m.cnrCount[cnrID+objectType]
	return v, ok
}

func (m *metricsStore) IncRefillObjectsCount(_ string, size int, success bool) {
	m.mtx.Lock()
	defer m.mtx.Unlock()

	m.refillCount++
	m.refillSize += int64(size)
}

func (m *metricsStore) SetRefillPercent(_ string, percent uint32) {
	m.mtx.Lock()
	defer m.mtx.Unlock()

	m.refillPercent = percent
}

func (m *metricsStore) SetRefillStatus(_ string, status string) {
	m.mtx.Lock()
	defer m.mtx.Unlock()

	m.refillStatus = status
}

func (m *metricsStore) SetEvacuationInProgress(bool) {
}

func TestCounters(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	sh, mm := shardWithMetrics(t, dir)
	defer func() { require.NoError(t, sh.Close()) }()

	sh.SetMode(mode.ReadOnly)
	require.Equal(t, mode.ReadOnly, mm.mode)
	sh.SetMode(mode.ReadWrite)
	require.Equal(t, mode.ReadWrite, mm.mode)

	const objNumber = 10
	oo := make([]*objectSDK.Object, objNumber)
	for i := range objNumber {
		oo[i] = testutil.GenerateObject()
	}

	t.Run("defaults", func(t *testing.T) {
		require.Zero(t, mm.getObjectCounter(physical))
		require.Zero(t, mm.getObjectCounter(logical))
		require.Empty(t, mm.containerSizes())
		require.Zero(t, mm.payloadSize())

		for _, obj := range oo {
			contID, _ := obj.ContainerID()
			v, ok := mm.getContainerCount(contID.EncodeToString(), physical)
			require.Zero(t, v)
			require.False(t, ok)
			v, ok = mm.getContainerCount(contID.EncodeToString(), logical)
			require.Zero(t, v)
			require.False(t, ok)
			v, ok = mm.getContainerCount(contID.EncodeToString(), user)
			require.Zero(t, v)
			require.False(t, ok)
		}
	})

	var totalPayload int64

	expectedLogicalSizes := make(map[string]int64)
	expected := make(map[cid.ID]meta.ObjectCounters)
	for i := range oo {
		cnr, _ := oo[i].ContainerID()
		oSize := int64(oo[i].PayloadSize())
		expectedLogicalSizes[cnr.EncodeToString()] += oSize
		totalPayload += oSize
		expected[cnr] = meta.ObjectCounters{
			Logic: 1,
			Phy:   1,
			User:  1,
		}
	}

	var prm PutPrm

	for i := range objNumber {
		prm.SetObject(oo[i])

		_, err := sh.Put(context.Background(), prm)
		require.NoError(t, err)
	}

	require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical))
	require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical))
	require.Equal(t, uint64(objNumber), mm.getObjectCounter(user))
	require.Equal(t, expectedLogicalSizes, mm.containerSizes())
	require.Equal(t, totalPayload, mm.payloadSize())

	cc, err := sh.metaBase.ContainerCounters(context.Background())
	require.NoError(t, err)
	require.Equal(t, meta.ContainerCounters{Counts: expected}, cc)

	t.Run("inhume_GC", func(t *testing.T) {
		var prm InhumePrm
		inhumedNumber := objNumber / 4

		for i := range inhumedNumber {
			prm.MarkAsGarbage(objectcore.AddressOf(oo[i]))

			_, err := sh.Inhume(context.Background(), prm)
			require.NoError(t, err)

			cid, ok := oo[i].ContainerID()
			require.True(t, ok)
			expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize())

			if v, ok := expected[cid]; ok {
				v.Logic--
				v.User--
				if v.IsZero() {
					delete(expected, cid)
				} else {
					expected[cid] = v
				}
			}
		}

		require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical))
		require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(logical))
		require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(user))
		require.Equal(t, expectedLogicalSizes, mm.containerSizes())
		require.Equal(t, totalPayload, mm.payloadSize())

		cc, err := sh.metaBase.ContainerCounters(context.Background())
		require.NoError(t, err)
		require.Equal(t, meta.ContainerCounters{Counts: expected}, cc)

		oo = oo[inhumedNumber:]
	})

	t.Run("inhume_TS", func(t *testing.T) {
		var prm InhumePrm

		phy := mm.getObjectCounter(physical)
		logic := mm.getObjectCounter(logical)
		custom := mm.getObjectCounter(user)

		inhumedNumber := int(phy / 4)
		for _, o := range addrFromObjs(oo[:inhumedNumber]) {
			ts := oidtest.Address()
			ts.SetContainer(o.Container())
			prm.SetTarget(ts, o)
			_, err := sh.Inhume(context.Background(), prm)
			require.NoError(t, err)
		}

		for i := range inhumedNumber {
			cid, ok := oo[i].ContainerID()
			require.True(t, ok)
			expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize())

			if v, ok := expected[cid]; ok {
				v.Logic--
				v.User--
				if v.IsZero() {
					delete(expected, cid)
				} else {
					expected[cid] = v
				}
			}
		}

		require.Equal(t, phy, mm.getObjectCounter(physical))
		require.Equal(t, logic-uint64(inhumedNumber), mm.getObjectCounter(logical))
		require.Equal(t, custom-uint64(inhumedNumber), mm.getObjectCounter(user))
		require.Equal(t, expectedLogicalSizes, mm.containerSizes())
		require.Equal(t, totalPayload, mm.payloadSize())

		cc, err = sh.metaBase.ContainerCounters(context.Background())
		require.NoError(t, err)
		require.Equal(t, meta.ContainerCounters{Counts: expected}, cc)

		oo = oo[inhumedNumber:]
	})

	t.Run("Delete", func(t *testing.T) {
		var prm DeletePrm

		phy := mm.getObjectCounter(physical)
		logic := mm.getObjectCounter(logical)
		custom := mm.getObjectCounter(user)

		deletedNumber := int(phy / 4)
		prm.SetAddresses(addrFromObjs(oo[:deletedNumber])...)

		_, err := sh.Delete(context.Background(), prm)
		require.NoError(t, err)

		require.Equal(t, phy-uint64(deletedNumber), mm.getObjectCounter(physical))
		require.Equal(t, logic-uint64(deletedNumber), mm.getObjectCounter(logical))
		require.Equal(t, custom-uint64(deletedNumber), mm.getObjectCounter(user))
		var totalRemovedpayload uint64
		for i := range oo[:deletedNumber] {
			removedPayload := oo[i].PayloadSize()
			totalRemovedpayload += removedPayload

			cnr, _ := oo[i].ContainerID()
			expectedLogicalSizes[cnr.EncodeToString()] -= int64(removedPayload)

			if v, ok := expected[cnr]; ok {
				v.Logic--
				v.Phy--
				v.User--
				expected[cnr] = v
			}
		}
		require.Equal(t, expectedLogicalSizes, mm.containerSizes())
		require.Equal(t, totalPayload-int64(totalRemovedpayload), mm.payloadSize())

		cc, err = sh.metaBase.ContainerCounters(context.Background())
		require.NoError(t, err)
		require.Equal(t, meta.ContainerCounters{Counts: expected}, cc)
	})
}

func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) {
	blobOpts := []blobstor.Option{
		blobstor.WithStorages([]blobstor.SubStorage{
			{
				Storage: fstree.New(
					fstree.WithDirNameLen(2),
					fstree.WithPath(filepath.Join(path, "blob")),
					fstree.WithDepth(1)),
			},
		}),
	}

	mm := NewMetricStore()

	sh := New(
		WithID(NewIDFromBytes([]byte{})),
		WithBlobStorOptions(blobOpts...),
		WithPiloramaOptions(pilorama.WithPath(filepath.Join(path, "pilorama"))),
		WithMetaBaseOptions(
			meta.WithPath(filepath.Join(path, "meta")),
			meta.WithEpochState(epochState{})),
		WithMetricsWriter(mm),
		WithGCRemoverSleepInterval(time.Hour),
	)
	require.NoError(t, sh.Open(context.Background()))
	require.NoError(t, sh.Init(context.Background()))

	return sh, mm
}

func addrFromObjs(oo []*objectSDK.Object) []oid.Address {
	aa := make([]oid.Address, len(oo))

	for i := range len(oo) {
		aa[i] = objectcore.AddressOf(oo[i])
	}

	return aa
}