[#645] *: Change the locality condition of the node from the placement

Some software components regulate the way of working with placement arrays
when a local node enters it. In the previous implementation, the locality
criterion was the correspondence between the announced network address
(group) and the address with which the node was configured. However, by
design, network addresses are not unique identifiers of storage nodes in the
system.

Change comparisons by network addresses to comparisons by keys in all
packages with the logic described above. Implement `netmap.AnnouncedKeys`
interface on `cfg` type in the storage node application.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-09-06 15:17:14 +03:00 committed by Alex Vanin
parent f14987c06d
commit 358e3ed8c4
10 changed files with 62 additions and 58 deletions

View file

@ -97,7 +97,7 @@ func initContainerService(c *cfg) {
LocalServerInfo: c, LocalServerInfo: c,
RemoteWriterProvider: &remoteLoadAnnounceProvider{ RemoteWriterProvider: &remoteLoadAnnounceProvider{
key: &c.key.PrivateKey, key: &c.key.PrivateKey,
loadAddrSrc: c, netmapKeys: c,
clientCache: c.clientCache, clientCache: c.clientCache,
deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator), deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator),
}, },
@ -220,7 +220,7 @@ func (nopLoadWriter) Close() error {
type remoteLoadAnnounceProvider struct { type remoteLoadAnnounceProvider struct {
key *ecdsa.PrivateKey key *ecdsa.PrivateKey
loadAddrSrc network.LocalAddressSource netmapKeys netmapCore.AnnouncedKeys
clientCache interface { clientCache interface {
Get(network.AddressGroup) (apiClient.Client, error) Get(network.AddressGroup) (apiClient.Client, error)
@ -234,6 +234,11 @@ func (r *remoteLoadAnnounceProvider) InitRemote(srv loadroute.ServerInfo) (loadc
return r.deadEndProvider, nil return r.deadEndProvider, nil
} }
if r.netmapKeys.IsLocalKey(srv.PublicKey()) {
// if local => return no-op writer
return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil
}
var netAddr network.AddressGroup var netAddr network.AddressGroup
err := netAddr.FromIterator(srv) err := netAddr.FromIterator(srv)
@ -241,11 +246,6 @@ func (r *remoteLoadAnnounceProvider) InitRemote(srv loadroute.ServerInfo) (loadc
return nil, fmt.Errorf("could not convert address to IP format: %w", err) return nil, fmt.Errorf("could not convert address to IP format: %w", err)
} }
if network.IsLocalAddress(r.loadAddrSrc, netAddr) {
// if local => return no-op writer
return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil
}
c, err := r.clientCache.Get(netAddr) c, err := r.clientCache.Get(netAddr)
if err != nil { if err != nil {
return nil, fmt.Errorf("could not initialize API client: %w", err) return nil, fmt.Errorf("could not initialize API client: %w", err)
@ -385,6 +385,10 @@ func (c *cfg) PublicKey() []byte {
return nodeKeyFromNetmap(c) return nodeKeyFromNetmap(c)
} }
func (c *cfg) IsLocalKey(key []byte) bool {
return bytes.Equal(key, c.PublicKey())
}
func (c *cfg) IterateAddresses(f func(string) bool) { func (c *cfg) IterateAddresses(f func(string) bool) {
c.iterateNetworkAddresses(f) c.iterateNetworkAddresses(f)
} }

View file

@ -240,7 +240,7 @@ func initObjectService(c *cfg) {
policer.WithRemoteHeader( policer.WithRemoteHeader(
headsvc.NewRemoteHeader(keyStorage, clientConstructor), headsvc.NewRemoteHeader(keyStorage, clientConstructor),
), ),
policer.WithLocalAddressSource(c), policer.WithNetmapKeys(c),
policer.WithHeadTimeout( policer.WithHeadTimeout(
policerconfig.HeadTimeout(c.appCfg), policerconfig.HeadTimeout(c.appCfg),
), ),
@ -276,7 +276,7 @@ func initObjectService(c *cfg) {
putsvc.WithLocalStorage(ls), putsvc.WithLocalStorage(ls),
putsvc.WithContainerSource(c.cfgObject.cnrSource), putsvc.WithContainerSource(c.cfgObject.cnrSource),
putsvc.WithNetworkMapSource(c.cfgObject.netMapSource), putsvc.WithNetworkMapSource(c.cfgObject.netMapSource),
putsvc.WithLocalAddressSource(c), putsvc.WithNetmapKeys(c),
putsvc.WithFormatValidatorOpts( putsvc.WithFormatValidatorOpts(
objectCore.WithDeleteHandler(objInhumer), objectCore.WithDeleteHandler(objInhumer),
), ),

View file

@ -87,7 +87,7 @@ func initReputationService(c *cfg) {
remoteLocalTrustProvider := common.NewRemoteTrustProvider( remoteLocalTrustProvider := common.NewRemoteTrustProvider(
common.RemoteProviderPrm{ common.RemoteProviderPrm{
LocalAddrSrc: c, NetmapKeys: c,
DeadEndProvider: daughterStorageWriterProvider, DeadEndProvider: daughterStorageWriterProvider,
ClientCache: c.clientCache, ClientCache: c.clientCache,
WriterProvider: localreputation.NewRemoteProvider( WriterProvider: localreputation.NewRemoteProvider(
@ -100,7 +100,7 @@ func initReputationService(c *cfg) {
remoteIntermediateTrustProvider := common.NewRemoteTrustProvider( remoteIntermediateTrustProvider := common.NewRemoteTrustProvider(
common.RemoteProviderPrm{ common.RemoteProviderPrm{
LocalAddrSrc: c, NetmapKeys: c,
DeadEndProvider: consumerStorageWriterProvider, DeadEndProvider: consumerStorageWriterProvider,
ClientCache: c.clientCache, ClientCache: c.clientCache,
WriterProvider: intermediatereputation.NewRemoteProvider( WriterProvider: intermediatereputation.NewRemoteProvider(

View file

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
apiClient "github.com/nspcc-dev/neofs-api-go/pkg/client" apiClient "github.com/nspcc-dev/neofs-api-go/pkg/client"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/network" "github.com/nspcc-dev/neofs-node/pkg/network"
reputationcommon "github.com/nspcc-dev/neofs-node/pkg/services/reputation/common" reputationcommon "github.com/nspcc-dev/neofs-node/pkg/services/reputation/common"
reputationrouter "github.com/nspcc-dev/neofs-node/pkg/services/reputation/common/router" reputationrouter "github.com/nspcc-dev/neofs-node/pkg/services/reputation/common/router"
@ -27,7 +28,7 @@ type clientKeyRemoteProvider interface {
// //
// remoteTrustProvider requires to be provided with clientKeyRemoteProvider. // remoteTrustProvider requires to be provided with clientKeyRemoteProvider.
type remoteTrustProvider struct { type remoteTrustProvider struct {
localAddrSrc network.LocalAddressSource netmapKeys netmap.AnnouncedKeys
deadEndProvider reputationcommon.WriterProvider deadEndProvider reputationcommon.WriterProvider
clientCache clientCache clientCache clientCache
remoteProvider clientKeyRemoteProvider remoteProvider clientKeyRemoteProvider
@ -39,7 +40,7 @@ type remoteTrustProvider struct {
// Passing incorrect parameter values will result in constructor // Passing incorrect parameter values will result in constructor
// failure (error or panic depending on the implementation). // failure (error or panic depending on the implementation).
type RemoteProviderPrm struct { type RemoteProviderPrm struct {
LocalAddrSrc network.LocalAddressSource NetmapKeys netmap.AnnouncedKeys
DeadEndProvider reputationcommon.WriterProvider DeadEndProvider reputationcommon.WriterProvider
ClientCache clientCache ClientCache clientCache
WriterProvider clientKeyRemoteProvider WriterProvider clientKeyRemoteProvider
@ -47,8 +48,8 @@ type RemoteProviderPrm struct {
func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriterProvider { func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriterProvider {
switch { switch {
case prm.LocalAddrSrc == nil: case prm.NetmapKeys == nil:
PanicOnPrmValue("LocalAddrSrc", prm.LocalAddrSrc) PanicOnPrmValue("NetmapKeys", prm.NetmapKeys)
case prm.DeadEndProvider == nil: case prm.DeadEndProvider == nil:
PanicOnPrmValue("DeadEndProvider", prm.DeadEndProvider) PanicOnPrmValue("DeadEndProvider", prm.DeadEndProvider)
case prm.ClientCache == nil: case prm.ClientCache == nil:
@ -58,7 +59,7 @@ func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriter
} }
return &remoteTrustProvider{ return &remoteTrustProvider{
localAddrSrc: prm.LocalAddrSrc, netmapKeys: prm.NetmapKeys,
deadEndProvider: prm.DeadEndProvider, deadEndProvider: prm.DeadEndProvider,
clientCache: prm.ClientCache, clientCache: prm.ClientCache,
remoteProvider: prm.WriterProvider, remoteProvider: prm.WriterProvider,
@ -70,6 +71,11 @@ func (rtp *remoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (rep
return rtp.deadEndProvider, nil return rtp.deadEndProvider, nil
} }
if rtp.netmapKeys.IsLocalKey(srv.PublicKey()) {
// if local => return no-op writer
return trustcontroller.SimpleWriterProvider(new(NopReputationWriter)), nil
}
var netAddr network.AddressGroup var netAddr network.AddressGroup
err := netAddr.FromIterator(srv) err := netAddr.FromIterator(srv)
@ -77,11 +83,6 @@ func (rtp *remoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (rep
return nil, fmt.Errorf("could not convert address to IP format: %w", err) return nil, fmt.Errorf("could not convert address to IP format: %w", err)
} }
if network.IsLocalAddress(rtp.localAddrSrc, netAddr) {
// if local => return no-op writer
return trustcontroller.SimpleWriterProvider(new(NopReputationWriter)), nil
}
c, err := rtp.clientCache.Get(netAddr) c, err := rtp.clientCache.Get(netAddr)
if err != nil { if err != nil {
return nil, fmt.Errorf("could not initialize API client: %w", err) return nil, fmt.Errorf("could not initialize API client: %w", err)

View file

@ -6,7 +6,6 @@ import (
"sync" "sync"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/network"
svcutil "github.com/nspcc-dev/neofs-node/pkg/services/object/util" svcutil "github.com/nspcc-dev/neofs-node/pkg/services/object/util"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer"
@ -23,7 +22,7 @@ type distributedTarget struct {
chunks [][]byte chunks [][]byte
nodeTargetInitializer func(network.AddressGroup) transformer.ObjectTarget nodeTargetInitializer func(placement.Node) transformer.ObjectTarget
relay func(placement.Node) error relay func(placement.Node) error
@ -76,7 +75,7 @@ func (t *distributedTarget) sendObject(node placement.Node) error {
} }
} }
target := t.nodeTargetInitializer(node.Addresses()) target := t.nodeTargetInitializer(node)
if err := target.WriteHeader(t.obj); err != nil { if err := target.WriteHeader(t.obj); err != nil {
return fmt.Errorf("could not write header: %w", err) return fmt.Errorf("could not write header: %w", err)

View file

@ -46,7 +46,7 @@ type cfg struct {
workerPool util.WorkerPool workerPool util.WorkerPool
localAddrSrc network.LocalAddressSource netmapKeys netmap.AnnouncedKeys
fmtValidator *object.FormatValidator fmtValidator *object.FormatValidator
@ -123,9 +123,9 @@ func WithWorkerPool(v util.WorkerPool) Option {
} }
} }
func WithLocalAddressSource(v network.LocalAddressSource) Option { func WithNetmapKeys(v netmap.AnnouncedKeys) Option {
return func(c *cfg) { return func(c *cfg) {
c.localAddrSrc = v c.netmapKeys = v
} }
} }

View file

@ -135,7 +135,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error {
prm.traverseOpts = append(prm.traverseOpts, placement.SuccessAfter(1)) prm.traverseOpts = append(prm.traverseOpts, placement.SuccessAfter(1))
// use local-only placement builder // use local-only placement builder
builder = util.NewLocalPlacement(builder, p.localAddrSrc) builder = util.NewLocalPlacement(builder, p.netmapKeys)
} }
// set placement builder // set placement builder
@ -152,7 +152,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget {
relay = func(node placement.Node) error { relay = func(node placement.Node) error {
addr := node.Addresses() addr := node.Addresses()
if network.IsLocalAddress(p.localAddrSrc, addr) { if p.netmapKeys.IsLocalKey(node.Key()) {
return errLocalAddress return errLocalAddress
} }
@ -168,8 +168,8 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget {
return &distributedTarget{ return &distributedTarget{
traverseOpts: prm.traverseOpts, traverseOpts: prm.traverseOpts,
workerPool: p.workerPool, workerPool: p.workerPool,
nodeTargetInitializer: func(addr network.AddressGroup) transformer.ObjectTarget { nodeTargetInitializer: func(node placement.Node) transformer.ObjectTarget {
if network.IsLocalAddress(p.localAddrSrc, addr) { if p.netmapKeys.IsLocalKey(node.Key()) {
return &localTarget{ return &localTarget{
storage: p.localStore, storage: p.localStore,
} }
@ -179,7 +179,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget {
ctx: p.ctx, ctx: p.ctx,
keyStorage: p.keyStorage, keyStorage: p.keyStorage,
commonPrm: prm.common, commonPrm: prm.common,
addr: addr, addr: node.Addresses(),
clientConstructor: p.clientConstructor, clientConstructor: p.clientConstructor,
} }
}, },

View file

@ -14,13 +14,13 @@ import (
type localPlacement struct { type localPlacement struct {
builder placement.Builder builder placement.Builder
localAddrSrc network.LocalAddressSource netmapKeys netmap.AnnouncedKeys
} }
type remotePlacement struct { type remotePlacement struct {
builder placement.Builder builder placement.Builder
localAddrSrc network.LocalAddressSource netmapKeys netmap.AnnouncedKeys
} }
// TraverserGenerator represents tool that generates // TraverserGenerator represents tool that generates
@ -30,15 +30,15 @@ type TraverserGenerator struct {
cnrSrc container.Source cnrSrc container.Source
localAddrSrc network.LocalAddressSource netmapKeys netmap.AnnouncedKeys
customOpts []placement.Option customOpts []placement.Option
} }
func NewLocalPlacement(b placement.Builder, s network.LocalAddressSource) placement.Builder { func NewLocalPlacement(b placement.Builder, s netmap.AnnouncedKeys) placement.Builder {
return &localPlacement{ return &localPlacement{
builder: b, builder: b,
localAddrSrc: s, netmapKeys: s,
} }
} }
@ -58,7 +58,7 @@ func (p *localPlacement) BuildPlacement(addr *object.Address, policy *netmapSDK.
continue continue
} }
if network.IsLocalAddress(p.localAddrSrc, addr) { if p.netmapKeys.IsLocalKey(vs[i][j].PublicKey()) {
return []netmapSDK.Nodes{{vs[i][j]}}, nil return []netmapSDK.Nodes{{vs[i][j]}}, nil
} }
} }
@ -69,10 +69,10 @@ func (p *localPlacement) BuildPlacement(addr *object.Address, policy *netmapSDK.
// NewRemotePlacementBuilder creates, initializes and returns placement builder that // NewRemotePlacementBuilder creates, initializes and returns placement builder that
// excludes local node from any placement vector. // excludes local node from any placement vector.
func NewRemotePlacementBuilder(b placement.Builder, s network.LocalAddressSource) placement.Builder { func NewRemotePlacementBuilder(b placement.Builder, s netmap.AnnouncedKeys) placement.Builder {
return &remotePlacement{ return &remotePlacement{
builder: b, builder: b,
localAddrSrc: s, netmapKeys: s,
} }
} }
@ -92,7 +92,7 @@ func (p *remotePlacement) BuildPlacement(addr *object.Address, policy *netmapSDK
continue continue
} }
if network.IsLocalAddress(p.localAddrSrc, addr) { if p.netmapKeys.IsLocalKey(vs[i][j].PublicKey()) {
vs[i] = append(vs[i][:j], vs[i][j+1:]...) vs[i] = append(vs[i][:j], vs[i][j+1:]...)
j-- j--
} }
@ -103,11 +103,11 @@ func (p *remotePlacement) BuildPlacement(addr *object.Address, policy *netmapSDK
} }
// NewTraverserGenerator creates, initializes and returns new TraverserGenerator instance. // NewTraverserGenerator creates, initializes and returns new TraverserGenerator instance.
func NewTraverserGenerator(nmSrc netmap.Source, cnrSrc container.Source, localAddrSrc network.LocalAddressSource) *TraverserGenerator { func NewTraverserGenerator(nmSrc netmap.Source, cnrSrc container.Source, netmapKeys netmap.AnnouncedKeys) *TraverserGenerator {
return &TraverserGenerator{ return &TraverserGenerator{
netMapSrc: nmSrc, netMapSrc: nmSrc,
cnrSrc: cnrSrc, cnrSrc: cnrSrc,
localAddrSrc: localAddrSrc, netmapKeys: netmapKeys,
} }
} }
@ -116,7 +116,7 @@ func (g *TraverserGenerator) WithTraverseOptions(opts ...placement.Option) *Trav
return &TraverserGenerator{ return &TraverserGenerator{
netMapSrc: g.netMapSrc, netMapSrc: g.netMapSrc,
cnrSrc: g.cnrSrc, cnrSrc: g.cnrSrc,
localAddrSrc: g.localAddrSrc, netmapKeys: g.netmapKeys,
customOpts: opts, customOpts: opts,
} }
} }
@ -143,7 +143,7 @@ func (g *TraverserGenerator) GenerateTraverser(addr *object.Address, epoch uint6
// create builder of the remote nodes from network map // create builder of the remote nodes from network map
builder := NewRemotePlacementBuilder( builder := NewRemotePlacementBuilder(
placement.NewNetworkMapBuilder(nm), placement.NewNetworkMapBuilder(nm),
g.localAddrSrc, g.netmapKeys,
) )
traverseOpts = append(traverseOpts, traverseOpts = append(traverseOpts,

View file

@ -68,7 +68,7 @@ func (p *Policer) processNodes(ctx context.Context, addr *object.Address, nodes
continue continue
} }
if network.IsLocalAddress(p.localAddrSrc, node) { if p.netmapKeys.IsLocalKey(nodes[i].PublicKey()) {
if shortage == 0 { if shortage == 0 {
// we can call the redundant copy callback // we can call the redundant copy callback
// here to slightly improve the performance // here to slightly improve the performance

View file

@ -6,8 +6,8 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/container" "github.com/nspcc-dev/neofs-node/pkg/core/container"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
"github.com/nspcc-dev/neofs-node/pkg/network"
headsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/head" headsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/head"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/services/replicator" "github.com/nspcc-dev/neofs-node/pkg/services/replicator"
@ -47,7 +47,7 @@ type cfg struct {
remoteHeader *headsvc.RemoteHeader remoteHeader *headsvc.RemoteHeader
localAddrSrc network.LocalAddressSource netmapKeys netmap.AnnouncedKeys
replicator *replicator.Replicator replicator *replicator.Replicator
@ -142,10 +142,10 @@ func WithRemoteHeader(v *headsvc.RemoteHeader) Option {
} }
} }
// WithLocalAddressSource returns option to set local address source of Policer. // WithNetmapKeys returns option to set tool to work with announced public keys.
func WithLocalAddressSource(v network.LocalAddressSource) Option { func WithNetmapKeys(v netmap.AnnouncedKeys) Option {
return func(c *cfg) { return func(c *cfg) {
c.localAddrSrc = v c.netmapKeys = v
} }
} }