TrueCloudLab/frostfs-node
21
1
Fork 27
Code Issues 108 Pull requests 3 Projects Releases 2 Activity Actions

[#645] *: Change the locality condition of the node from the placement

Browse source 
Some software components regulate the way of working with placement arrays
when a local node enters it. In the previous implementation, the locality
criterion was the correspondence between the announced network address
(group) and the address with which the node was configured. However, by
design, network addresses are not unique identifiers of storage nodes in the
system.

Change comparisons by network addresses to comparisons by keys in all
packages with the logic described above. Implement `netmap.AnnouncedKeys`
interface on `cfg` type in the storage node application.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-09-06 15:17:14 +03:00 committed by Alex Vanin
parent f14987c06d
commit 358e3ed8c4
10 changed files with 62 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
cmd/neofs-node/container.go
View file

@ -97,7 +97,7 @@ func initContainerService(c *cfg) {
LocalServerInfo: c,
RemoteWriterProvider: &remoteLoadAnnounceProvider{
key: &c.key.PrivateKey,
loadAddrSrc: c,
netmapKeys: c,
clientCache: c.clientCache,
deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator),
},
@ -220,7 +220,7 @@ func (nopLoadWriter) Close() error {
type remoteLoadAnnounceProvider struct {
key *ecdsa.PrivateKey
loadAddrSrc network.LocalAddressSource
netmapKeys netmapCore.AnnouncedKeys
clientCache interface {
Get(network.AddressGroup) (apiClient.Client, error)
@ -234,6 +234,11 @@ func (r *remoteLoadAnnounceProvider) InitRemote(srv loadroute.ServerInfo) (loadc
return r.deadEndProvider, nil
}
if r.netmapKeys.IsLocalKey(srv.PublicKey()) {
// if local => return no-op writer
return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil
}
var netAddr network.AddressGroup
err := netAddr.FromIterator(srv)
@ -241,11 +246,6 @@ func (r *remoteLoadAnnounceProvider) InitRemote(srv loadroute.ServerInfo) (loadc
return nil, fmt.Errorf("could not convert address to IP format: %w", err)
}
if network.IsLocalAddress(r.loadAddrSrc, netAddr) {
// if local => return no-op writer
return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil
}
c, err := r.clientCache.Get(netAddr)
if err != nil {
return nil, fmt.Errorf("could not initialize API client: %w", err)
@ -385,6 +385,10 @@ func (c *cfg) PublicKey() []byte {
return nodeKeyFromNetmap(c)
}
func (c *cfg) IsLocalKey(key []byte) bool {
return bytes.Equal(key, c.PublicKey())
}
func (c *cfg) IterateAddresses(f func(string) bool) {
c.iterateNetworkAddresses(f)
}

4
cmd/neofs-node/object.go
View file

@ -240,7 +240,7 @@ func initObjectService(c *cfg) {
policer.WithRemoteHeader(
headsvc.NewRemoteHeader(keyStorage, clientConstructor),
),
policer.WithLocalAddressSource(c),
policer.WithNetmapKeys(c),
policer.WithHeadTimeout(
policerconfig.HeadTimeout(c.appCfg),
),
@ -276,7 +276,7 @@ func initObjectService(c *cfg) {
putsvc.WithLocalStorage(ls),
putsvc.WithContainerSource(c.cfgObject.cnrSource),
putsvc.WithNetworkMapSource(c.cfgObject.netMapSource),
putsvc.WithLocalAddressSource(c),
putsvc.WithNetmapKeys(c),
putsvc.WithFormatValidatorOpts(
objectCore.WithDeleteHandler(objInhumer),
),

4
cmd/neofs-node/reputation.go
View file

@ -87,7 +87,7 @@ func initReputationService(c *cfg) {
remoteLocalTrustProvider := common.NewRemoteTrustProvider(
common.RemoteProviderPrm{
LocalAddrSrc: c,
NetmapKeys: c,
DeadEndProvider: daughterStorageWriterProvider,
ClientCache: c.clientCache,
WriterProvider: localreputation.NewRemoteProvider(
@ -100,7 +100,7 @@ func initReputationService(c *cfg) {
remoteIntermediateTrustProvider := common.NewRemoteTrustProvider(
common.RemoteProviderPrm{
LocalAddrSrc: c,
NetmapKeys: c,
DeadEndProvider: consumerStorageWriterProvider,
ClientCache: c.clientCache,
WriterProvider: intermediatereputation.NewRemoteProvider(

21
cmd/neofs-node/reputation/common/remote.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
apiClient "github.com/nspcc-dev/neofs-api-go/pkg/client"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/network"
reputationcommon "github.com/nspcc-dev/neofs-node/pkg/services/reputation/common"
reputationrouter "github.com/nspcc-dev/neofs-node/pkg/services/reputation/common/router"
@ -27,7 +28,7 @@ type clientKeyRemoteProvider interface {
//
// remoteTrustProvider requires to be provided with clientKeyRemoteProvider.
type remoteTrustProvider struct {
localAddrSrc network.LocalAddressSource
netmapKeys netmap.AnnouncedKeys
deadEndProvider reputationcommon.WriterProvider
clientCache clientCache
remoteProvider clientKeyRemoteProvider
@ -39,7 +40,7 @@ type remoteTrustProvider struct {
// Passing incorrect parameter values will result in constructor
// failure (error or panic depending on the implementation).
type RemoteProviderPrm struct {
LocalAddrSrc network.LocalAddressSource
NetmapKeys netmap.AnnouncedKeys
DeadEndProvider reputationcommon.WriterProvider
ClientCache clientCache
WriterProvider clientKeyRemoteProvider
@ -47,8 +48,8 @@ type RemoteProviderPrm struct {
func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriterProvider {
switch {
case prm.LocalAddrSrc == nil:
PanicOnPrmValue("LocalAddrSrc", prm.LocalAddrSrc)
case prm.NetmapKeys == nil:
PanicOnPrmValue("NetmapKeys", prm.NetmapKeys)
case prm.DeadEndProvider == nil:
PanicOnPrmValue("DeadEndProvider", prm.DeadEndProvider)
case prm.ClientCache == nil:
@ -58,7 +59,7 @@ func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriter
}
return &remoteTrustProvider{
localAddrSrc: prm.LocalAddrSrc,
netmapKeys: prm.NetmapKeys,
deadEndProvider: prm.DeadEndProvider,
clientCache: prm.ClientCache,
remoteProvider: prm.WriterProvider,
@ -70,6 +71,11 @@ func (rtp *remoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (rep
return rtp.deadEndProvider, nil
}
if rtp.netmapKeys.IsLocalKey(srv.PublicKey()) {
// if local => return no-op writer
return trustcontroller.SimpleWriterProvider(new(NopReputationWriter)), nil
}
var netAddr network.AddressGroup
err := netAddr.FromIterator(srv)
@ -77,11 +83,6 @@ func (rtp *remoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (rep
return nil, fmt.Errorf("could not convert address to IP format: %w", err)
}
if network.IsLocalAddress(rtp.localAddrSrc, netAddr) {
// if local => return no-op writer
return trustcontroller.SimpleWriterProvider(new(NopReputationWriter)), nil
}
c, err := rtp.clientCache.Get(netAddr)
if err != nil {
return nil, fmt.Errorf("could not initialize API client: %w", err)

5
pkg/services/object/put/distributed.go
View file

@ -6,7 +6,6 @@ import (
"sync"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/network"
svcutil "github.com/nspcc-dev/neofs-node/pkg/services/object/util"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer"
@ -23,7 +22,7 @@ type distributedTarget struct {
chunks [][]byte
nodeTargetInitializer func(network.AddressGroup) transformer.ObjectTarget
nodeTargetInitializer func(placement.Node) transformer.ObjectTarget
relay func(placement.Node) error
@ -76,7 +75,7 @@ func (t *distributedTarget) sendObject(node placement.Node) error {
}
}
target := t.nodeTargetInitializer(node.Addresses())
target := t.nodeTargetInitializer(node)
if err := target.WriteHeader(t.obj); err != nil {
return fmt.Errorf("could not write header: %w", err)

6
pkg/services/object/put/service.go
View file

@ -46,7 +46,7 @@ type cfg struct {
workerPool util.WorkerPool
localAddrSrc network.LocalAddressSource
netmapKeys netmap.AnnouncedKeys
fmtValidator *object.FormatValidator
@ -123,9 +123,9 @@ func WithWorkerPool(v util.WorkerPool) Option {
}
}
func WithLocalAddressSource(v network.LocalAddressSource) Option {
func WithNetmapKeys(v netmap.AnnouncedKeys) Option {
return func(c *cfg) {
c.localAddrSrc = v
c.netmapKeys = v
}
}

10
pkg/services/object/put/streamer.go
View file

@ -135,7 +135,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error {
prm.traverseOpts = append(prm.traverseOpts, placement.SuccessAfter(1))
// use local-only placement builder
builder = util.NewLocalPlacement(builder, p.localAddrSrc)
builder = util.NewLocalPlacement(builder, p.netmapKeys)
}
// set placement builder
@ -152,7 +152,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget {
relay = func(node placement.Node) error {
addr := node.Addresses()
if network.IsLocalAddress(p.localAddrSrc, addr) {
if p.netmapKeys.IsLocalKey(node.Key()) {
return errLocalAddress
}
@ -168,8 +168,8 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget {
return &distributedTarget{
traverseOpts: prm.traverseOpts,
workerPool: p.workerPool,
nodeTargetInitializer: func(addr network.AddressGroup) transformer.ObjectTarget {
if network.IsLocalAddress(p.localAddrSrc, addr) {
nodeTargetInitializer: func(node placement.Node) transformer.ObjectTarget {
if p.netmapKeys.IsLocalKey(node.Key()) {
return &localTarget{
storage: p.localStore,
}
@ -179,7 +179,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget {
ctx: p.ctx,
keyStorage: p.keyStorage,
commonPrm: prm.common,
addr: addr,
addr: node.Addresses(),
clientConstructor: p.clientConstructor,
}
},

40
pkg/services/object/util/placement.go
View file

@ -14,13 +14,13 @@ import (
type localPlacement struct {
builder placement.Builder
localAddrSrc network.LocalAddressSource
netmapKeys netmap.AnnouncedKeys
}
type remotePlacement struct {
builder placement.Builder
localAddrSrc network.LocalAddressSource
netmapKeys netmap.AnnouncedKeys
}
// TraverserGenerator represents tool that generates
@ -30,15 +30,15 @@ type TraverserGenerator struct {
cnrSrc container.Source
localAddrSrc network.LocalAddressSource
netmapKeys netmap.AnnouncedKeys
customOpts []placement.Option
}
func NewLocalPlacement(b placement.Builder, s network.LocalAddressSource) placement.Builder {
func NewLocalPlacement(b placement.Builder, s netmap.AnnouncedKeys) placement.Builder {
return &localPlacement{
builder: b,
localAddrSrc: s,
builder: b,
netmapKeys: s,
}
}
@ -58,7 +58,7 @@ func (p *localPlacement) BuildPlacement(addr *object.Address, policy *netmapSDK.
continue
}
if network.IsLocalAddress(p.localAddrSrc, addr) {
if p.netmapKeys.IsLocalKey(vs[i][j].PublicKey()) {
return []netmapSDK.Nodes{{vs[i][j]}}, nil
}
}
@ -69,10 +69,10 @@ func (p *localPlacement) BuildPlacement(addr *object.Address, policy *netmapSDK.
// NewRemotePlacementBuilder creates, initializes and returns placement builder that
// excludes local node from any placement vector.
func NewRemotePlacementBuilder(b placement.Builder, s network.LocalAddressSource) placement.Builder {
func NewRemotePlacementBuilder(b placement.Builder, s netmap.AnnouncedKeys) placement.Builder {
return &remotePlacement{
builder: b,
localAddrSrc: s,
builder: b,
netmapKeys: s,
}
}
@ -92,7 +92,7 @@ func (p *remotePlacement) BuildPlacement(addr *object.Address, policy *netmapSDK
continue
}
if network.IsLocalAddress(p.localAddrSrc, addr) {
if p.netmapKeys.IsLocalKey(vs[i][j].PublicKey()) {
vs[i] = append(vs[i][:j], vs[i][j+1:]...)
j--
}
@ -103,21 +103,21 @@ func (p *remotePlacement) BuildPlacement(addr *object.Address, policy *netmapSDK
}
// NewTraverserGenerator creates, initializes and returns new TraverserGenerator instance.
func NewTraverserGenerator(nmSrc netmap.Source, cnrSrc container.Source, localAddrSrc network.LocalAddressSource) *TraverserGenerator {
func NewTraverserGenerator(nmSrc netmap.Source, cnrSrc container.Source, netmapKeys netmap.AnnouncedKeys) *TraverserGenerator {
return &TraverserGenerator{
netMapSrc: nmSrc,
cnrSrc: cnrSrc,
localAddrSrc: localAddrSrc,
netMapSrc: nmSrc,
cnrSrc: cnrSrc,
netmapKeys: netmapKeys,
}
}
// WithTraverseOptions returns TraverseGenerator that additionally applies provided options.
func (g *TraverserGenerator) WithTraverseOptions(opts ...placement.Option) *TraverserGenerator {
return &TraverserGenerator{
netMapSrc: g.netMapSrc,
cnrSrc: g.cnrSrc,
localAddrSrc: g.localAddrSrc,
customOpts: opts,
netMapSrc: g.netMapSrc,
cnrSrc: g.cnrSrc,
netmapKeys: g.netmapKeys,
customOpts: opts,
}
}
@ -143,7 +143,7 @@ func (g *TraverserGenerator) GenerateTraverser(addr *object.Address, epoch uint6
// create builder of the remote nodes from network map
builder := NewRemotePlacementBuilder(
placement.NewNetworkMapBuilder(nm),
g.localAddrSrc,
g.netmapKeys,
)
traverseOpts = append(traverseOpts,

2
pkg/services/policer/check.go
View file

@ -68,7 +68,7 @@ func (p *Policer) processNodes(ctx context.Context, addr *object.Address, nodes
continue
}
if network.IsLocalAddress(p.localAddrSrc, node) {
if p.netmapKeys.IsLocalKey(nodes[i].PublicKey()) {
if shortage == 0 {
// we can call the redundant copy callback
// here to slightly improve the performance

10
pkg/services/policer/policer.go
View file

@ -6,8 +6,8 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/container"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
"github.com/nspcc-dev/neofs-node/pkg/network"
headsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/head"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/services/replicator"
@ -47,7 +47,7 @@ type cfg struct {
remoteHeader *headsvc.RemoteHeader
localAddrSrc network.LocalAddressSource
netmapKeys netmap.AnnouncedKeys
replicator *replicator.Replicator
@ -142,10 +142,10 @@ func WithRemoteHeader(v *headsvc.RemoteHeader) Option {
}
}
// WithLocalAddressSource returns option to set local address source of Policer.
func WithLocalAddressSource(v network.LocalAddressSource) Option {
// WithNetmapKeys returns option to set tool to work with announced public keys.
func WithNetmapKeys(v netmap.AnnouncedKeys) Option {
return func(c *cfg) {
c.localAddrSrc = v
c.netmapKeys = v
}
}