package shard import ( "context" "path/filepath" "sync" "testing" "time" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) type metricsStore struct { mtx sync.Mutex objCounters map[string]uint64 cnrSize map[string]int64 cnrCount map[string]uint64 pldSize int64 mode mode.Mode errCounter int64 refillCount int64 refillSize int64 refillPercent uint32 refillStatus string } func newMetricStore() *metricsStore { return &metricsStore{ objCounters: map[string]uint64{ "phy": 0, "logic": 0, }, cnrSize: make(map[string]int64), cnrCount: make(map[string]uint64), } } func (m *metricsStore) SetShardID(_ string) {} func (m *metricsStore) SetObjectCounter(objectType string, v uint64) { m.mtx.Lock() defer m.mtx.Unlock() m.objCounters[objectType] = v } func (m *metricsStore) getObjectCounter(objectType string) uint64 { m.mtx.Lock() defer m.mtx.Unlock() return m.objCounters[objectType] } func (m *metricsStore) containerSizes() map[string]int64 { m.mtx.Lock() defer m.mtx.Unlock() r := make(map[string]int64, len(m.cnrSize)) for c, s := range m.cnrSize { r[c] = s } return r } func (m *metricsStore) payloadSize() int64 { m.mtx.Lock() defer m.mtx.Unlock() return m.pldSize } func (m *metricsStore) AddToObjectCounter(objectType string, delta int) { m.mtx.Lock() defer m.mtx.Unlock() switch { case delta > 0: m.objCounters[objectType] += uint64(delta) case delta < 0: uDelta := uint64(-delta) if m.objCounters[objectType] >= uDelta { m.objCounters[objectType] -= uDelta } else { m.objCounters[objectType] = 0 } case delta == 0: return } } func (m *metricsStore) IncObjectCounter(objectType string) { m.mtx.Lock() defer m.mtx.Unlock() m.objCounters[objectType] += 1 } func (m *metricsStore) SetMode(mode mode.Mode) { m.mtx.Lock() defer m.mtx.Unlock() m.mode = mode } func (m *metricsStore) AddToContainerSize(cnr string, size int64) { m.mtx.Lock() defer m.mtx.Unlock() m.cnrSize[cnr] += size } func (m *metricsStore) AddToPayloadSize(size int64) { m.mtx.Lock() defer m.mtx.Unlock() m.pldSize += size } func (m *metricsStore) IncErrorCounter() { m.mtx.Lock() defer m.mtx.Unlock() m.errCounter += 1 } func (m *metricsStore) ClearErrorCounter() { m.mtx.Lock() defer m.mtx.Unlock() m.errCounter = 0 } func (m *metricsStore) DeleteShardMetrics() { m.mtx.Lock() defer m.mtx.Unlock() m.errCounter = 0 } func (m *metricsStore) SetContainerObjectsCount(cnrID string, objectType string, value uint64) { m.mtx.Lock() defer m.mtx.Unlock() m.cnrCount[cnrID+objectType] = value } func (m *metricsStore) IncContainerObjectsCount(cnrID string, objectType string) { m.mtx.Lock() defer m.mtx.Unlock() m.cnrCount[cnrID+objectType]++ } func (m *metricsStore) SubContainerObjectsCount(cnrID string, objectType string, value uint64) { m.mtx.Lock() defer m.mtx.Unlock() existed := m.cnrCount[cnrID+objectType] if existed < value { panic("existed value smaller than value to sustract") } if existed == value { delete(m.cnrCount, cnrID+objectType) } else { m.cnrCount[cnrID+objectType] -= value } } func (m *metricsStore) getContainerCount(cnrID, objectType string) (uint64, bool) { m.mtx.Lock() defer m.mtx.Unlock() v, ok := m.cnrCount[cnrID+objectType] return v, ok } func (m *metricsStore) IncRefillObjectsCount(_ string, size int, success bool) { m.mtx.Lock() defer m.mtx.Unlock() m.refillCount++ m.refillSize += int64(size) } func (m *metricsStore) SetRefillPercent(_ string, percent uint32) { m.mtx.Lock() defer m.mtx.Unlock() m.refillPercent = percent } func (m *metricsStore) SetRefillStatus(_ string, status string) { m.mtx.Lock() defer m.mtx.Unlock() m.refillStatus = status } func (m *metricsStore) SetEvacuationInProgress(bool) { } func TestCounters(t *testing.T) { t.Parallel() dir := t.TempDir() sh, mm := shardWithMetrics(t, dir) defer func() { require.NoError(t, sh.Close(context.Background())) }() sh.SetMode(context.Background(), mode.ReadOnly) require.Equal(t, mode.ReadOnly, mm.mode) sh.SetMode(context.Background(), mode.ReadWrite) require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 oo := make([]*objectSDK.Object, objNumber) for i := range objNumber { oo[i] = testutil.GenerateObject() } t.Run("defaults", func(t *testing.T) { require.Zero(t, mm.getObjectCounter(physical)) require.Zero(t, mm.getObjectCounter(logical)) require.Empty(t, mm.containerSizes()) require.Zero(t, mm.payloadSize()) for _, obj := range oo { contID, _ := obj.ContainerID() v, ok := mm.getContainerCount(contID.EncodeToString(), physical) require.Zero(t, v) require.False(t, ok) v, ok = mm.getContainerCount(contID.EncodeToString(), logical) require.Zero(t, v) require.False(t, ok) v, ok = mm.getContainerCount(contID.EncodeToString(), user) require.Zero(t, v) require.False(t, ok) } }) var totalPayload int64 expectedLogicalSizes := make(map[string]int64) expected := make(map[cid.ID]meta.ObjectCounters) for i := range oo { cnr, _ := oo[i].ContainerID() oSize := int64(oo[i].PayloadSize()) expectedLogicalSizes[cnr.EncodeToString()] += oSize totalPayload += oSize expected[cnr] = meta.ObjectCounters{ Logic: 1, Phy: 1, User: 1, } } var prm PutPrm for i := range objNumber { prm.SetObject(oo[i]) _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical)) require.Equal(t, uint64(objNumber), mm.getObjectCounter(user)) require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) cc, err := sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) t.Run("inhume_GC", func(t *testing.T) { var prm InhumePrm inhumedNumber := objNumber / 4 for i := range inhumedNumber { prm.MarkAsGarbage(objectcore.AddressOf(oo[i])) _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) cid, ok := oo[i].ContainerID() require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) if v, ok := expected[cid]; ok { v.Logic-- v.User-- if v.IsZero() { delete(expected, cid) } else { expected[cid] = v } } } require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(logical)) require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(user)) require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) cc, err := sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) oo = oo[inhumedNumber:] }) t.Run("inhume_TS", func(t *testing.T) { var prm InhumePrm phy := mm.getObjectCounter(physical) logic := mm.getObjectCounter(logical) custom := mm.getObjectCounter(user) inhumedNumber := int(phy / 4) for _, o := range addrFromObjs(oo[:inhumedNumber]) { ts := oidtest.Address() ts.SetContainer(o.Container()) prm.SetTarget(ts, o) _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) } for i := range inhumedNumber { cid, ok := oo[i].ContainerID() require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) if v, ok := expected[cid]; ok { v.Logic-- v.User-- if v.IsZero() { delete(expected, cid) } else { expected[cid] = v } } } require.Equal(t, phy, mm.getObjectCounter(physical)) require.Equal(t, logic-uint64(inhumedNumber), mm.getObjectCounter(logical)) require.Equal(t, custom-uint64(inhumedNumber), mm.getObjectCounter(user)) require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) cc, err = sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) oo = oo[inhumedNumber:] }) t.Run("Delete", func(t *testing.T) { var prm DeletePrm phy := mm.getObjectCounter(physical) logic := mm.getObjectCounter(logical) custom := mm.getObjectCounter(user) deletedNumber := int(phy / 4) prm.SetAddresses(addrFromObjs(oo[:deletedNumber])...) _, err := sh.Delete(context.Background(), prm) require.NoError(t, err) require.Equal(t, phy-uint64(deletedNumber), mm.getObjectCounter(physical)) require.Equal(t, logic-uint64(deletedNumber), mm.getObjectCounter(logical)) require.Equal(t, custom-uint64(deletedNumber), mm.getObjectCounter(user)) var totalRemovedpayload uint64 for i := range oo[:deletedNumber] { removedPayload := oo[i].PayloadSize() totalRemovedpayload += removedPayload cnr, _ := oo[i].ContainerID() expectedLogicalSizes[cnr.EncodeToString()] -= int64(removedPayload) if v, ok := expected[cnr]; ok { v.Logic-- v.Phy-- v.User-- expected[cnr] = v } } require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload-int64(totalRemovedpayload), mm.payloadSize()) cc, err = sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) }) } func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { blobOpts := []blobstor.Option{ blobstor.WithStorages([]blobstor.SubStorage{ { Storage: fstree.New( fstree.WithDirNameLen(2), fstree.WithPath(filepath.Join(path, "blob")), fstree.WithDepth(1)), }, }), } mm := newMetricStore() sh := New( WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(path, "pilorama"))), WithMetaBaseOptions( meta.WithPath(filepath.Join(path, "meta")), meta.WithEpochState(epochState{})), WithMetricsWriter(mm), WithGCRemoverSleepInterval(time.Hour), ) require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) return sh, mm } func addrFromObjs(oo []*objectSDK.Object) []oid.Address { aa := make([]oid.Address, len(oo)) for i := range len(oo) { aa[i] = objectcore.AddressOf(oo[i]) } return aa }