forked from TrueCloudLab/frostfs-node
[#838] metabase: Add user object type counter
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
29550fe600
commit
f314da4af3
8 changed files with 247 additions and 227 deletions
|
|
@ -24,13 +24,13 @@ func TestCounters(t *testing.T) {
|
|||
db := newDB(t)
|
||||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, c.Phy())
|
||||
require.Zero(t, c.Logic())
|
||||
require.Zero(t, c.Phy)
|
||||
require.Zero(t, c.Logic)
|
||||
require.Zero(t, c.User)
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, len(cc.Physical))
|
||||
require.Zero(t, len(cc.Logical))
|
||||
require.Zero(t, len(cc.Counts))
|
||||
})
|
||||
|
||||
t.Run("put", func(t *testing.T) {
|
||||
|
|
@ -42,29 +42,30 @@ func TestCounters(t *testing.T) {
|
|||
}
|
||||
|
||||
var prm meta.PutPrm
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
|
||||
for i := 0; i < objCount; i++ {
|
||||
prm.SetObject(oo[i])
|
||||
cnrID, _ := oo[i].ContainerID()
|
||||
expPhy[cnrID]++
|
||||
expLog[cnrID]++
|
||||
c := meta.ObjectCounters{}
|
||||
exp[cnrID] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
|
||||
_, err := db.Put(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
|
||||
c, err := db.ObjectCounters()
|
||||
c, err = db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, uint64(i+1), c.Phy())
|
||||
require.Equal(t, uint64(i+1), c.Logic())
|
||||
require.Equal(t, uint64(i+1), c.Phy)
|
||||
require.Equal(t, uint64(i+1), c.Logic)
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -73,12 +74,13 @@ func TestCounters(t *testing.T) {
|
|||
db := newDB(t)
|
||||
oo := putObjs(t, db, objCount, false)
|
||||
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
for _, obj := range oo {
|
||||
cnrID, _ := obj.ContainerID()
|
||||
expPhy[cnrID]++
|
||||
expLog[cnrID]++
|
||||
exp[cnrID] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
}
|
||||
|
||||
var prm meta.DeletePrm
|
||||
|
|
@ -92,29 +94,23 @@ func TestCounters(t *testing.T) {
|
|||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, uint64(i), c.Phy())
|
||||
require.Equal(t, uint64(i), c.Logic())
|
||||
require.Equal(t, uint64(i), c.Phy)
|
||||
require.Equal(t, uint64(i), c.Logic)
|
||||
|
||||
cnrID, _ := oo[i].ContainerID()
|
||||
if v, ok := expPhy[cnrID]; ok {
|
||||
if v == 1 {
|
||||
delete(expPhy, cnrID)
|
||||
if v, ok := exp[cnrID]; ok {
|
||||
v.Phy--
|
||||
v.Logic--
|
||||
if v.IsZero() {
|
||||
delete(exp, cnrID)
|
||||
} else {
|
||||
expPhy[cnrID]--
|
||||
}
|
||||
}
|
||||
if v, ok := expLog[cnrID]; ok {
|
||||
if v == 1 {
|
||||
delete(expLog, cnrID)
|
||||
} else {
|
||||
expLog[cnrID]--
|
||||
exp[cnrID] = v
|
||||
}
|
||||
}
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -123,12 +119,13 @@ func TestCounters(t *testing.T) {
|
|||
db := newDB(t)
|
||||
oo := putObjs(t, db, objCount, false)
|
||||
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
for _, obj := range oo {
|
||||
cnrID, _ := obj.ContainerID()
|
||||
expPhy[cnrID]++
|
||||
expLog[cnrID]++
|
||||
exp[cnrID] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
}
|
||||
|
||||
inhumedObjs := make([]oid.Address, objCount/2)
|
||||
|
|
@ -142,11 +139,12 @@ func TestCounters(t *testing.T) {
|
|||
}
|
||||
|
||||
for _, addr := range inhumedObjs {
|
||||
if v, ok := expLog[addr.Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expLog, addr.Container())
|
||||
if v, ok := exp[addr.Container()]; ok {
|
||||
v.Logic--
|
||||
if v.IsZero() {
|
||||
delete(exp, addr.Container())
|
||||
} else {
|
||||
expLog[addr.Container()]--
|
||||
exp[addr.Container()] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -162,14 +160,13 @@ func TestCounters(t *testing.T) {
|
|||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, uint64(objCount), c.Phy())
|
||||
require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic())
|
||||
require.Equal(t, uint64(objCount), c.Phy)
|
||||
require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic)
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
})
|
||||
|
||||
t.Run("put_split", func(t *testing.T) {
|
||||
|
|
@ -177,8 +174,7 @@ func TestCounters(t *testing.T) {
|
|||
db := newDB(t)
|
||||
parObj := testutil.GenerateObject()
|
||||
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
|
||||
// put objects and check that parent info
|
||||
// does not affect the counter
|
||||
|
|
@ -189,20 +185,21 @@ func TestCounters(t *testing.T) {
|
|||
}
|
||||
|
||||
cnrID, _ := o.ContainerID()
|
||||
expLog[cnrID]++
|
||||
expPhy[cnrID]++
|
||||
exp[cnrID] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
|
||||
require.NoError(t, putBig(db, o))
|
||||
|
||||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(i+1), c.Phy())
|
||||
require.Equal(t, uint64(i+1), c.Logic())
|
||||
require.Equal(t, uint64(i+1), c.Phy)
|
||||
require.Equal(t, uint64(i+1), c.Logic)
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -211,12 +208,13 @@ func TestCounters(t *testing.T) {
|
|||
db := newDB(t)
|
||||
oo := putObjs(t, db, objCount, true)
|
||||
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
for _, obj := range oo {
|
||||
cnrID, _ := obj.ContainerID()
|
||||
expPhy[cnrID]++
|
||||
expLog[cnrID]++
|
||||
exp[cnrID] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// delete objects that have parent info
|
||||
|
|
@ -228,21 +226,15 @@ func TestCounters(t *testing.T) {
|
|||
|
||||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(objCount-i-1), c.Phy())
|
||||
require.Equal(t, uint64(objCount-i-1), c.Logic())
|
||||
require.Equal(t, uint64(objCount-i-1), c.Phy)
|
||||
|
||||
if v, ok := expPhy[addr.Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expPhy, addr.Container())
|
||||
if v, ok := exp[addr.Container()]; ok {
|
||||
v.Logic--
|
||||
v.Phy--
|
||||
if v.IsZero() {
|
||||
delete(exp, addr.Container())
|
||||
} else {
|
||||
expPhy[addr.Container()]--
|
||||
}
|
||||
}
|
||||
if v, ok := expLog[addr.Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expLog, addr.Container())
|
||||
} else {
|
||||
expLog[addr.Container()]--
|
||||
exp[addr.Container()] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -253,12 +245,13 @@ func TestCounters(t *testing.T) {
|
|||
db := newDB(t)
|
||||
oo := putObjs(t, db, objCount, true)
|
||||
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
for _, obj := range oo {
|
||||
cnrID, _ := obj.ContainerID()
|
||||
expPhy[cnrID]++
|
||||
expLog[cnrID]++
|
||||
exp[cnrID] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
}
|
||||
|
||||
inhumedObjs := make([]oid.Address, objCount/2)
|
||||
|
|
@ -272,11 +265,12 @@ func TestCounters(t *testing.T) {
|
|||
}
|
||||
|
||||
for _, addr := range inhumedObjs {
|
||||
if v, ok := expLog[addr.Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expLog, addr.Container())
|
||||
if v, ok := exp[addr.Container()]; ok {
|
||||
v.Logic--
|
||||
if v.IsZero() {
|
||||
delete(exp, addr.Container())
|
||||
} else {
|
||||
expLog[addr.Container()]--
|
||||
exp[addr.Container()] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -291,14 +285,13 @@ func TestCounters(t *testing.T) {
|
|||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, uint64(objCount), c.Phy())
|
||||
require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic())
|
||||
require.Equal(t, uint64(objCount), c.Phy)
|
||||
require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic)
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -320,25 +313,25 @@ func TestCounters_Expired(t *testing.T) {
|
|||
oo[i] = putWithExpiration(t, db, objectSDK.TypeRegular, epoch+1)
|
||||
}
|
||||
|
||||
expPhy := make(map[cid.ID]uint64)
|
||||
expLog := make(map[cid.ID]uint64)
|
||||
exp := make(map[cid.ID]meta.ObjectCounters)
|
||||
for _, addr := range oo {
|
||||
expPhy[addr.Container()]++
|
||||
expLog[addr.Container()]++
|
||||
exp[addr.Container()] = meta.ObjectCounters{
|
||||
Logic: 1,
|
||||
Phy: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// 1. objects are available and counters are correct
|
||||
|
||||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(objCount), c.Phy())
|
||||
require.Equal(t, uint64(objCount), c.Logic())
|
||||
require.Equal(t, uint64(objCount), c.Phy)
|
||||
require.Equal(t, uint64(objCount), c.Logic)
|
||||
|
||||
cc, err := db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
|
||||
for _, o := range oo {
|
||||
_, err := metaGet(db, o, true)
|
||||
|
|
@ -352,14 +345,13 @@ func TestCounters_Expired(t *testing.T) {
|
|||
|
||||
c, err = db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(objCount), c.Phy())
|
||||
require.Equal(t, uint64(objCount), c.Logic())
|
||||
require.Equal(t, uint64(objCount), c.Phy)
|
||||
require.Equal(t, uint64(objCount), c.Logic)
|
||||
|
||||
cc, err = db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
|
||||
for _, o := range oo {
|
||||
_, err := metaGet(db, o, true)
|
||||
|
|
@ -381,22 +373,22 @@ func TestCounters_Expired(t *testing.T) {
|
|||
c, err = db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, uint64(len(oo)), c.Phy())
|
||||
require.Equal(t, uint64(len(oo)-1), c.Logic())
|
||||
require.Equal(t, uint64(len(oo)), c.Phy)
|
||||
require.Equal(t, uint64(len(oo)-1), c.Logic)
|
||||
|
||||
if v, ok := expLog[oo[0].Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expLog, oo[0].Container())
|
||||
if v, ok := exp[oo[0].Container()]; ok {
|
||||
v.Logic--
|
||||
if v.IsZero() {
|
||||
delete(exp, oo[0].Container())
|
||||
} else {
|
||||
expLog[oo[0].Container()]--
|
||||
exp[oo[0].Container()] = v
|
||||
}
|
||||
}
|
||||
|
||||
cc, err = db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
|
||||
// 4. `Delete` an object with GCMark should decrease the
|
||||
// phy counter but does not affect the logic counter (after
|
||||
|
|
@ -409,11 +401,12 @@ func TestCounters_Expired(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Zero(t, deleteRes.AvailableObjectsRemoved())
|
||||
|
||||
if v, ok := expPhy[oo[0].Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expPhy, oo[0].Container())
|
||||
if v, ok := exp[oo[0].Container()]; ok {
|
||||
v.Phy--
|
||||
if v.IsZero() {
|
||||
delete(exp, oo[0].Container())
|
||||
} else {
|
||||
expPhy[oo[0].Container()]--
|
||||
exp[oo[0].Container()] = v
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -421,14 +414,13 @@ func TestCounters_Expired(t *testing.T) {
|
|||
|
||||
c, err = db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(len(oo)), c.Phy())
|
||||
require.Equal(t, uint64(len(oo)), c.Logic())
|
||||
require.Equal(t, uint64(len(oo)), c.Phy)
|
||||
require.Equal(t, uint64(len(oo)), c.Logic)
|
||||
|
||||
cc, err = db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
|
||||
// 5 `Delete` an expired object (like it would the control
|
||||
// service do) should decrease both counters despite the
|
||||
|
|
@ -440,19 +432,13 @@ func TestCounters_Expired(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved())
|
||||
|
||||
if v, ok := expLog[oo[0].Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expLog, oo[0].Container())
|
||||
if v, ok := exp[oo[0].Container()]; ok {
|
||||
v.Phy--
|
||||
v.Logic--
|
||||
if v.IsZero() {
|
||||
delete(exp, oo[0].Container())
|
||||
} else {
|
||||
expLog[oo[0].Container()]--
|
||||
}
|
||||
}
|
||||
|
||||
if v, ok := expPhy[oo[0].Container()]; ok {
|
||||
if v == 1 {
|
||||
delete(expPhy, oo[0].Container())
|
||||
} else {
|
||||
expPhy[oo[0].Container()]--
|
||||
exp[oo[0].Container()] = v
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -460,14 +446,13 @@ func TestCounters_Expired(t *testing.T) {
|
|||
|
||||
c, err = db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(len(oo)), c.Phy())
|
||||
require.Equal(t, uint64(len(oo)), c.Logic())
|
||||
require.Equal(t, uint64(len(oo)), c.Phy)
|
||||
require.Equal(t, uint64(len(oo)), c.Logic)
|
||||
|
||||
cc, err = db.ContainerCounters(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, expPhy, cc.Physical)
|
||||
require.Equal(t, expLog, cc.Logical)
|
||||
require.Equal(t, meta.ContainerCounters{Counts: exp}, cc)
|
||||
}
|
||||
|
||||
func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK.Object {
|
||||
|
|
@ -491,8 +476,8 @@ func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK
|
|||
c, err := db.ObjectCounters()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, uint64(i+1), c.Phy())
|
||||
require.Equal(t, uint64(i+1), c.Logic())
|
||||
require.Equal(t, uint64(i+1), c.Phy)
|
||||
require.Equal(t, uint64(i+1), c.Logic)
|
||||
}
|
||||
|
||||
return oo
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue