forked from TrueCloudLab/frostfs-node
[#92] Refactor policer and add some unit tests
Signed-off-by: Alejandro Lopez <a.lopez@yadro.com>
This commit is contained in:
parent
0c5b025788
commit
f9730f090d
13 changed files with 640 additions and 310 deletions
28
cmd/frostfs-node/keyspaceiterator.go
Normal file
28
cmd/frostfs-node/keyspaceiterator.go
Normal file
|
@ -0,0 +1,28 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
|
||||
)
|
||||
|
||||
type keySpaceIterator struct {
|
||||
ng *engine.StorageEngine
|
||||
cur *engine.Cursor
|
||||
}
|
||||
|
||||
func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objectcore.AddressWithType, error) {
|
||||
var prm engine.ListWithCursorPrm
|
||||
prm.WithCursor(it.cur)
|
||||
prm.WithCount(batchSize)
|
||||
|
||||
res, err := it.ng.ListWithCursor(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot list objects in engine: %w", err)
|
||||
}
|
||||
|
||||
it.cur = res.Cursor()
|
||||
return res.AddressList(), nil
|
||||
}
|
|
@ -38,6 +38,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
|
@ -211,15 +212,30 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl
|
|||
|
||||
ls := c.cfgObject.cfgLocalStorage.localStorage
|
||||
|
||||
buryFn := func(ctx context.Context, addr oid.Address) error {
|
||||
var prm engine.InhumePrm
|
||||
prm.MarkAsGarbage(addr)
|
||||
prm.WithForceRemoval()
|
||||
|
||||
_, err := ls.Inhume(ctx, prm)
|
||||
return err
|
||||
}
|
||||
|
||||
remoteHeader := headsvc.NewRemoteHeader(keyStorage, clientConstructor)
|
||||
|
||||
pol := policer.New(
|
||||
policer.WithLogger(c.log),
|
||||
policer.WithLocalStorage(ls),
|
||||
policer.WithKeySpaceIterator(&keySpaceIterator{ng: ls}),
|
||||
policer.WithBuryFunc(buryFn),
|
||||
policer.WithContainerSource(c.cfgObject.cnrSource),
|
||||
policer.WithPlacementBuilder(
|
||||
placement.NewNetworkMapSourceBuilder(c.netMapSource),
|
||||
),
|
||||
policer.WithRemoteHeader(
|
||||
headsvc.NewRemoteHeader(keyStorage, clientConstructor),
|
||||
policer.WithRemoteObjectHeaderFunc(
|
||||
func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) {
|
||||
prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a)
|
||||
return remoteHeader.Head(ctx, prm)
|
||||
},
|
||||
),
|
||||
policer.WithNetmapKeys(c),
|
||||
policer.WithHeadTimeout(
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue