[#215] container: remove old estimations when adding new ones

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-01-18 10:08:13 +03:00 committed by Alex Vanin
parent 7bca6bf782
commit 676daa1782
2 changed files with 67 additions and 6 deletions

View file

@ -3,6 +3,7 @@ package container
import ( import (
"github.com/nspcc-dev/neo-go/pkg/interop" "github.com/nspcc-dev/neo-go/pkg/interop"
"github.com/nspcc-dev/neo-go/pkg/interop/contract" "github.com/nspcc-dev/neo-go/pkg/interop/contract"
"github.com/nspcc-dev/neo-go/pkg/interop/convert"
"github.com/nspcc-dev/neo-go/pkg/interop/iterator" "github.com/nspcc-dev/neo-go/pkg/interop/iterator"
"github.com/nspcc-dev/neo-go/pkg/interop/native/crypto" "github.com/nspcc-dev/neo-go/pkg/interop/native/crypto"
"github.com/nspcc-dev/neo-go/pkg/interop/native/management" "github.com/nspcc-dev/neo-go/pkg/interop/native/management"
@ -59,10 +60,15 @@ const (
// V2 format // V2 format
containerIDSize = 32 // SHA256 size containerIDSize = 32 // SHA256 size
estimateKeyPrefix = "cnr" singleEstimatePrefix = "est"
estimatePostfixSize = 10 estimateKeyPrefix = "cnr"
// CleanupDelta contains number last epochs for which container estimations are present. estimatePostfixSize = 10
// CleanupDelta contains number of last epochs for which container estimations are present.
CleanupDelta = 3 CleanupDelta = 3
// TotalCleanupDelta contains number of epochs after which estimation
// will be removed by epoch tick cleanup if any node didn't updated
// container size and/or container was removed. Must be greater than CleanupDelta.
TotalCleanupDelta = CleanupDelta + 1
// NotFoundError is returned if container is missing. // NotFoundError is returned if container is missing.
NotFoundError = "container does not exist" NotFoundError = "container does not exist"
@ -88,6 +94,20 @@ func _deploy(data interface{}, isUpdate bool) {
args := data.([]interface{}) args := data.([]interface{})
common.CheckVersion(args[len(args)-1].(int)) common.CheckVersion(args[len(args)-1].(int))
storage.Delete(ctx, common.LegacyOwnerKey) storage.Delete(ctx, common.LegacyOwnerKey)
// Migrate container estimation keys.
it := storage.Find(ctx, []byte(estimateKeyPrefix), storage.DeserializeValues)
for iterator.Next(it) {
kv := iterator.Value(it).(struct {
key []byte
value estimation
})
end := len(kv.key) - containerIDSize - estimatePostfixSize
rawEpoch := kv.key[len(estimateKeyPrefix):end]
cid := kv.key[end : len(kv.key)-estimatePostfixSize]
updateEstimations(ctx, convert.ToInteger(rawEpoch), cid, kv.value.from, true)
}
return return
} }
@ -503,6 +523,7 @@ func PutContainerSize(epoch int, cid []byte, usedSize int, pubKey interop.Public
} }
storage.Put(ctx, key, std.Serialize(s)) storage.Put(ctx, key, std.Serialize(s))
updateEstimations(ctx, epoch, cid, pubKey, false)
runtime.Log("saved container size estimation") runtime.Log("saved container size estimation")
} }
@ -772,6 +793,32 @@ func isStorageNode(ctx storage.Context, key interop.PublicKey) bool {
return false return false
} }
func updateEstimations(ctx storage.Context, epoch int, cid []byte, pub interop.PublicKey, isUpdate bool) {
h := crypto.Ripemd160(pub)
estKey := append([]byte(singleEstimatePrefix), cid...)
estKey = append(estKey, h...)
var newEpochs []int
rawList := storage.Get(ctx, estKey).([]byte)
if rawList != nil {
epochs := std.Deserialize(rawList).([]int)
for _, oldEpoch := range epochs {
if !isUpdate && epoch-oldEpoch > CleanupDelta {
key := append([]byte(estimateKeyPrefix), convert.ToBytes(oldEpoch)...)
key = append(key, cid...)
key = append(key, h[:estimatePostfixSize]...)
storage.Delete(ctx, key)
} else {
newEpochs = append(newEpochs, oldEpoch)
}
}
}
newEpochs = append(newEpochs, epoch)
common.SetSerialized(ctx, estKey, newEpochs)
}
func cleanupContainers(ctx storage.Context, epoch int) { func cleanupContainers(ctx storage.Context, epoch int) {
it := storage.Find(ctx, []byte(estimateKeyPrefix), storage.KeysOnly) it := storage.Find(ctx, []byte(estimateKeyPrefix), storage.KeysOnly)
for iterator.Next(it) { for iterator.Next(it) {
@ -781,7 +828,7 @@ func cleanupContainers(ctx storage.Context, epoch int) {
var n interface{} = nbytes var n interface{} = nbytes
if epoch-n.(int) > CleanupDelta { if epoch-n.(int) > TotalCleanupDelta {
storage.Delete(ctx, k) storage.Delete(ctx, k)
} }
} }

View file

@ -308,9 +308,23 @@ func TestContainerSizeEstimation(t *testing.T) {
checkEstimations(t, c, 3, cnt, estimation{nodes[2].pub, 888}) checkEstimations(t, c, 3, cnt, estimation{nodes[2].pub, 888})
} }
cNm.Invoke(t, stackitem.Null{}, "newEpoch", int64(2+container.CleanupDelta+1)) epoch := int64(2 + container.CleanupDelta + 1)
checkEstimations(t, c, 2, cnt) cNm.Invoke(t, stackitem.Null{}, "newEpoch", epoch)
checkEstimations(t, c, 2, cnt, estimations...) // not yet removed
checkEstimations(t, c, 3, cnt, estimation{nodes[2].pub, 888}) checkEstimations(t, c, 3, cnt, estimation{nodes[2].pub, 888})
c.WithSigners(nodes[1].signer).Invoke(t, stackitem.Null{}, "putContainerSize",
epoch, cnt.id[:], int64(999), nodes[1].pub)
checkEstimations(t, c, 2, cnt, estimations[:1]...)
checkEstimations(t, c, epoch, cnt, estimation{nodes[1].pub, int64(999)})
// Estimation from node 0 should be cleaned during epoch tick.
for i := int64(1); i <= container.TotalCleanupDelta-container.CleanupDelta; i++ {
cNm.Invoke(t, stackitem.Null{}, "newEpoch", epoch+i)
}
checkEstimations(t, c, 2, cnt)
checkEstimations(t, c, epoch, cnt, estimation{nodes[1].pub, int64(999)})
} }
type estimation struct { type estimation struct {