forked from TrueCloudLab/frostfs-node
[#729] containersvc: Remove load announcement
IR code was removed in 8879c6ea.
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
parent
d07afd803c
commit
4caa934eea
17 changed files with 2 additions and 1845 deletions
|
|
@ -3,44 +3,22 @@ package main
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
containerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||
containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client"
|
||||
containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
|
||||
cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
|
||||
containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container"
|
||||
containerTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/container/grpc"
|
||||
containerService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container"
|
||||
loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller"
|
||||
loadroute "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route"
|
||||
placementrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route/placement"
|
||||
loadstorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/storage"
|
||||
containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
apiClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
startEstimationNotifyEvent = "StartEstimation"
|
||||
stopEstimationNotifyEvent = "StopEstimation"
|
||||
)
|
||||
|
||||
func initContainerService(ctx context.Context, c *cfg) {
|
||||
func initContainerService(_ context.Context, c *cfg) {
|
||||
// container wrapper that tries to invoke notary
|
||||
// requests if chain is configured so
|
||||
wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, cntClient.TryNotary())
|
||||
|
|
@ -52,44 +30,10 @@ func initContainerService(ctx context.Context, c *cfg) {
|
|||
|
||||
cnrRdr, cnrWrt := configureEACLAndContainerSources(c, wrap, cnrSrc)
|
||||
|
||||
loadAccumulator := loadstorage.New(loadstorage.Prm{})
|
||||
|
||||
loadPlacementBuilder := &loadPlacementBuilder{
|
||||
log: c.log,
|
||||
nmSrc: c.netMapSource,
|
||||
cnrSrc: cnrSrc,
|
||||
}
|
||||
|
||||
routeBuilder := placementrouter.New(placementrouter.Prm{
|
||||
PlacementBuilder: loadPlacementBuilder,
|
||||
})
|
||||
|
||||
loadRouter := loadroute.New(
|
||||
loadroute.Prm{
|
||||
LocalServerInfo: c,
|
||||
RemoteWriterProvider: &remoteLoadAnnounceProvider{
|
||||
key: &c.key.PrivateKey,
|
||||
netmapKeys: c,
|
||||
clientCache: c.bgClientCache,
|
||||
deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator),
|
||||
},
|
||||
Builder: routeBuilder,
|
||||
},
|
||||
loadroute.WithLogger(c.log),
|
||||
)
|
||||
|
||||
setLoadController(ctx, c, loadRouter, loadAccumulator)
|
||||
|
||||
server := containerTransportGRPC.New(
|
||||
containerService.NewSignService(
|
||||
&c.key.PrivateKey,
|
||||
&usedSpaceService{
|
||||
Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc),
|
||||
loadWriterProvider: loadRouter,
|
||||
loadPlacementBuilder: loadPlacementBuilder,
|
||||
routeBuilder: routeBuilder,
|
||||
cfg: c,
|
||||
},
|
||||
containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -178,50 +122,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
|
|||
return cnrRdr, cnrWrt
|
||||
}
|
||||
|
||||
func setLoadController(ctx context.Context, c *cfg, loadRouter *loadroute.Router, loadAccumulator *loadstorage.Storage) {
|
||||
pubKey := c.key.PublicKey().Bytes()
|
||||
|
||||
// container wrapper that always sends non-notary
|
||||
// requests
|
||||
wrapperNoNotary, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0)
|
||||
fatalOnErr(err)
|
||||
|
||||
resultWriter := &morphLoadWriter{
|
||||
log: c.log,
|
||||
cnrMorphClient: wrapperNoNotary,
|
||||
key: pubKey,
|
||||
}
|
||||
|
||||
localMetrics := &localStorageLoad{
|
||||
log: c.log,
|
||||
engine: c.cfgObject.cfgLocalStorage.localStorage,
|
||||
}
|
||||
|
||||
ctrl := loadcontroller.New(
|
||||
loadcontroller.Prm{
|
||||
LocalMetrics: loadcontroller.SimpleIteratorProvider(localMetrics),
|
||||
AnnouncementAccumulator: loadcontroller.SimpleIteratorProvider(loadAccumulator),
|
||||
LocalAnnouncementTarget: loadRouter,
|
||||
ResultReceiver: loadcontroller.SimpleWriterProvider(resultWriter),
|
||||
},
|
||||
loadcontroller.WithLogger(c.log),
|
||||
)
|
||||
|
||||
setContainerNotificationParser(c, startEstimationNotifyEvent, containerEvent.ParseStartEstimation)
|
||||
addContainerAsyncNotificationHandler(c, startEstimationNotifyEvent, func(ev event.Event) {
|
||||
ctrl.Start(ctx, loadcontroller.StartPrm{
|
||||
Epoch: ev.(containerEvent.StartEstimation).Epoch(),
|
||||
})
|
||||
})
|
||||
|
||||
setContainerNotificationParser(c, stopEstimationNotifyEvent, containerEvent.ParseStopEstimation)
|
||||
addContainerAsyncNotificationHandler(c, stopEstimationNotifyEvent, func(ev event.Event) {
|
||||
ctrl.Stop(ctx, loadcontroller.StopPrm{
|
||||
Epoch: ev.(containerEvent.StopEstimation).Epoch(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// addContainerNotificationHandler adds handler that will be executed synchronously.
|
||||
func addContainerNotificationHandler(c *cfg, sTyp string, h event.Handler) {
|
||||
typ := event.TypeFromString(sTyp)
|
||||
|
|
@ -284,219 +184,6 @@ func setContainerNotificationParser(c *cfg, sTyp string, p event.NotificationPar
|
|||
c.cfgContainer.parsers[typ] = p
|
||||
}
|
||||
|
||||
type morphLoadWriter struct {
|
||||
log *logger.Logger
|
||||
|
||||
cnrMorphClient *cntClient.Client
|
||||
|
||||
key []byte
|
||||
}
|
||||
|
||||
func (w *morphLoadWriter) Put(a containerSDK.SizeEstimation) error {
|
||||
w.log.Debug(logs.FrostFSNodeSaveUsedSpaceAnnouncementInContract,
|
||||
zap.Uint64("epoch", a.Epoch()),
|
||||
zap.Stringer("cid", a.Container()),
|
||||
zap.Uint64("size", a.Value()),
|
||||
)
|
||||
|
||||
prm := cntClient.AnnounceLoadPrm{}
|
||||
|
||||
prm.SetAnnouncement(a)
|
||||
prm.SetReporter(w.key)
|
||||
|
||||
return w.cnrMorphClient.AnnounceLoad(prm)
|
||||
}
|
||||
|
||||
func (*morphLoadWriter) Close(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopLoadWriter struct{}
|
||||
|
||||
func (nopLoadWriter) Put(containerSDK.SizeEstimation) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nopLoadWriter) Close(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type remoteLoadAnnounceProvider struct {
|
||||
key *ecdsa.PrivateKey
|
||||
|
||||
netmapKeys netmapCore.AnnouncedKeys
|
||||
|
||||
clientCache interface {
|
||||
Get(client.NodeInfo) (client.MultiAddressClient, error)
|
||||
}
|
||||
|
||||
deadEndProvider loadcontroller.WriterProvider
|
||||
}
|
||||
|
||||
func (r *remoteLoadAnnounceProvider) InitRemote(srv loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error) {
|
||||
if srv == nil {
|
||||
return r.deadEndProvider, nil
|
||||
}
|
||||
|
||||
if r.netmapKeys.IsLocalKey(srv.PublicKey()) {
|
||||
// if local => return no-op writer
|
||||
return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil
|
||||
}
|
||||
|
||||
var info client.NodeInfo
|
||||
|
||||
err := client.NodeInfoFromRawNetmapElement(&info, srv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse client node info: %w", err)
|
||||
}
|
||||
|
||||
c, err := r.clientCache.Get(info)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not initialize API client: %w", err)
|
||||
}
|
||||
|
||||
return &remoteLoadAnnounceWriterProvider{
|
||||
client: c,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type remoteLoadAnnounceWriterProvider struct {
|
||||
client client.Client
|
||||
}
|
||||
|
||||
func (p *remoteLoadAnnounceWriterProvider) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) {
|
||||
return &remoteLoadAnnounceWriter{
|
||||
client: p.client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type remoteLoadAnnounceWriter struct {
|
||||
client client.Client
|
||||
|
||||
buf []containerSDK.SizeEstimation
|
||||
}
|
||||
|
||||
func (r *remoteLoadAnnounceWriter) Put(a containerSDK.SizeEstimation) error {
|
||||
r.buf = append(r.buf, a)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *remoteLoadAnnounceWriter) Close(ctx context.Context) error {
|
||||
cliPrm := apiClient.PrmAnnounceSpace{
|
||||
Announcements: r.buf,
|
||||
}
|
||||
|
||||
_, err := r.client.ContainerAnnounceUsedSpace(ctx, cliPrm)
|
||||
return err
|
||||
}
|
||||
|
||||
type loadPlacementBuilder struct {
|
||||
log *logger.Logger
|
||||
|
||||
nmSrc netmapCore.Source
|
||||
|
||||
cnrSrc containerCore.Source
|
||||
}
|
||||
|
||||
func (l *loadPlacementBuilder) BuildPlacement(epoch uint64, cnr cid.ID) ([][]netmap.NodeInfo, error) {
|
||||
cnrNodes, nm, err := l.buildPlacement(epoch, cnr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
const pivotPrefix = "load_announcement_"
|
||||
|
||||
pivot := []byte(
|
||||
pivotPrefix + strconv.FormatUint(epoch, 10),
|
||||
)
|
||||
|
||||
placement, err := nm.PlacementVectors(cnrNodes, pivot)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not build placement vectors: %w", err)
|
||||
}
|
||||
|
||||
return placement, nil
|
||||
}
|
||||
|
||||
func (l *loadPlacementBuilder) buildPlacement(epoch uint64, idCnr cid.ID) ([][]netmap.NodeInfo, *netmap.NetMap, error) {
|
||||
cnr, err := l.cnrSrc.Get(idCnr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
nm, err := l.nmSrc.GetNetMapByEpoch(epoch)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("could not get network map: %w", err)
|
||||
}
|
||||
|
||||
binCnr := make([]byte, sha256.Size)
|
||||
idCnr.Encode(binCnr)
|
||||
|
||||
cnrNodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), binCnr)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("could not build container nodes: %w", err)
|
||||
}
|
||||
|
||||
return cnrNodes, nm, nil
|
||||
}
|
||||
|
||||
type localStorageLoad struct {
|
||||
log *logger.Logger
|
||||
|
||||
engine *engine.StorageEngine
|
||||
}
|
||||
|
||||
func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error {
|
||||
idList, err := engine.ListContainers(context.TODO(), d.engine)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list containers on engine failure: %w", err)
|
||||
}
|
||||
|
||||
for i := range idList {
|
||||
sz, err := engine.ContainerSize(d.engine, idList[i])
|
||||
if err != nil {
|
||||
d.log.Debug(logs.FrostFSNodeFailedToCalculateContainerSizeInStorageEngine,
|
||||
zap.Stringer("cid", idList[i]),
|
||||
zap.String("error", err.Error()),
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
d.log.Debug(logs.FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully,
|
||||
zap.Uint64("size", sz),
|
||||
zap.Stringer("cid", idList[i]),
|
||||
)
|
||||
|
||||
var a containerSDK.SizeEstimation
|
||||
a.SetContainer(idList[i])
|
||||
a.SetValue(sz)
|
||||
|
||||
if f != nil && !f(a) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := h(a); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type usedSpaceService struct {
|
||||
containerService.Server
|
||||
|
||||
loadWriterProvider loadcontroller.WriterProvider
|
||||
|
||||
loadPlacementBuilder *loadPlacementBuilder
|
||||
|
||||
routeBuilder loadroute.Builder
|
||||
|
||||
cfg *cfg
|
||||
}
|
||||
|
||||
func (c *cfg) PublicKey() []byte {
|
||||
return nodeKeyFromNetmap(c)
|
||||
}
|
||||
|
|
@ -517,125 +204,6 @@ func (c *cfg) ExternalAddresses() []string {
|
|||
return c.cfgNodeInfo.localInfo.ExternalAddresses()
|
||||
}
|
||||
|
||||
func (c *usedSpaceService) PublicKey() []byte {
|
||||
return nodeKeyFromNetmap(c.cfg)
|
||||
}
|
||||
|
||||
func (c *usedSpaceService) IterateAddresses(f func(string) bool) {
|
||||
c.cfg.iterateNetworkAddresses(f)
|
||||
}
|
||||
|
||||
func (c *usedSpaceService) NumberOfAddresses() int {
|
||||
return c.cfg.addressNum()
|
||||
}
|
||||
|
||||
func (c *usedSpaceService) ExternalAddresses() []string {
|
||||
return c.cfg.ExternalAddresses()
|
||||
}
|
||||
|
||||
func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *containerV2.AnnounceUsedSpaceRequest) (*containerV2.AnnounceUsedSpaceResponse, error) {
|
||||
var passedRoute []loadcontroller.ServerInfo
|
||||
|
||||
for hdr := req.GetVerificationHeader(); hdr != nil; hdr = hdr.GetOrigin() {
|
||||
passedRoute = append(passedRoute, &containerOnlyKeyRemoteServerInfo{
|
||||
key: hdr.GetBodySignature().GetKey(),
|
||||
})
|
||||
}
|
||||
|
||||
for left, right := 0, len(passedRoute)-1; left < right; left, right = left+1, right-1 {
|
||||
passedRoute[left], passedRoute[right] = passedRoute[right], passedRoute[left]
|
||||
}
|
||||
|
||||
passedRoute = append(passedRoute, c)
|
||||
|
||||
w, err := c.loadWriterProvider.InitWriter(passedRoute)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not initialize container's used space writer: %w", err)
|
||||
}
|
||||
|
||||
var est containerSDK.SizeEstimation
|
||||
|
||||
for _, aV2 := range req.GetBody().GetAnnouncements() {
|
||||
err = est.ReadFromV2(aV2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid size announcement: %w", err)
|
||||
}
|
||||
|
||||
if err := c.processLoadValue(ctx, est, passedRoute, w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
respBody := new(containerV2.AnnounceUsedSpaceResponseBody)
|
||||
|
||||
resp := new(containerV2.AnnounceUsedSpaceResponse)
|
||||
resp.SetBody(respBody)
|
||||
|
||||
c.cfg.respSvc.SetMeta(resp)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
var errNodeOutsideContainer = errors.New("node outside the container")
|
||||
|
||||
type containerOnlyKeyRemoteServerInfo struct {
|
||||
key []byte
|
||||
}
|
||||
|
||||
func (i *containerOnlyKeyRemoteServerInfo) PublicKey() []byte {
|
||||
return i.key
|
||||
}
|
||||
|
||||
func (*containerOnlyKeyRemoteServerInfo) IterateAddresses(func(string) bool) {
|
||||
}
|
||||
|
||||
func (*containerOnlyKeyRemoteServerInfo) NumberOfAddresses() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (*containerOnlyKeyRemoteServerInfo) ExternalAddresses() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *loadPlacementBuilder) isNodeFromContainerKey(epoch uint64, cnr cid.ID, key []byte) (bool, error) {
|
||||
cnrNodes, _, err := l.buildPlacement(epoch, cnr)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for i := range cnrNodes {
|
||||
for j := range cnrNodes[i] {
|
||||
if bytes.Equal(cnrNodes[i][j].PublicKey(), key) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.SizeEstimation,
|
||||
route []loadcontroller.ServerInfo, w loadcontroller.Writer) error {
|
||||
fromCnr, err := c.loadPlacementBuilder.isNodeFromContainerKey(a.Epoch(), a.Container(), route[0].PublicKey())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not verify that the sender belongs to the container: %w", err)
|
||||
} else if !fromCnr {
|
||||
return errNodeOutsideContainer
|
||||
}
|
||||
|
||||
err = loadroute.CheckRoute(c.routeBuilder, a, route)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wrong route of container's used space value: %w", err)
|
||||
}
|
||||
|
||||
err = w.Put(a)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not write container's used space value: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// implements interface required by container service provided by morph executor.
|
||||
type morphContainerReader struct {
|
||||
eacl containerCore.EACLSource
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue