From 236414df498032417426726e408e0d2947b2b9c9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Sep 2022 15:34:01 +0300 Subject: [PATCH] [#1817] network: Allow to use network addresses from the iterator Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 4 +- cmd/neofs-node/config.go | 1 + cmd/neofs-node/config/apiclient/config.go | 8 +++ .../config/apiclient/config_test.go | 2 + cmd/neofs-node/container.go | 12 ++++ cmd/neofs-node/reputation/common/util.go | 4 ++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + pkg/core/client/client.go | 12 ++++ pkg/core/client/util.go | 16 ++++- pkg/core/netmap/nodes.go | 10 +++ pkg/innerring/innerring.go | 11 +-- pkg/innerring/rpc.go | 4 +- pkg/network/cache/client.go | 18 +++-- pkg/network/group.go | 21 ++++++ pkg/network/group_test.go | 69 +++++++++++++++++++ .../container/announcement/load/route/deps.go | 3 + .../object_manager/placement/traverser.go | 13 ++++ pkg/services/reputation/common/deps.go | 3 + pkg/services/reputation/common/managers.go | 4 ++ 21 files changed, 203 insertions(+), 17 deletions(-) create mode 100644 pkg/network/group_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a983b57..4ac3b7c6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Changelog for NeoFS Node ### Added - Serving `NetmapService.NetmapSnapshot` RPC (#1793) - `netmap snapshot` command of NeoFS CLI (#1793) - +- `apiclient.allow_external` config flag to fallback to node external addresses (#1817) - Changelog updates CI step (#1808) - Validate storage node configuration before node startup (#1805) - `neofs-node -check` command to check the configuration file (#1805) @@ -31,6 +31,8 @@ Changelog for NeoFS Node ### Updating from v0.32.0 Replace using the `control netmap-snapshot` command with `netmap snapshot` one in NeoFS CLI. +Node can now specify additional addresses in `ExternalAddr` attribute. To allow a node to dial +other nodes external address, use `apiclient.allow_external` config setting. ## [0.32.0] - 2022-09-14 - Pungdo (풍도, 楓島) diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index 096f8ec8c..e753e07ff 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -537,6 +537,7 @@ func initCfg(appCfg *config.Config) *cfg { DialTimeout: apiclientconfig.DialTimeout(appCfg), StreamTimeout: apiclientconfig.StreamTimeout(appCfg), Key: &key.PrivateKey, + AllowExternal: apiclientconfig.AllowExternal(appCfg), }), persistate: persistate, } diff --git a/cmd/neofs-node/config/apiclient/config.go b/cmd/neofs-node/config/apiclient/config.go index eedad9430..12f78af39 100644 --- a/cmd/neofs-node/config/apiclient/config.go +++ b/cmd/neofs-node/config/apiclient/config.go @@ -41,3 +41,11 @@ func StreamTimeout(c *config.Config) time.Duration { return StreamTimeoutDefault } + +// AllowExternal returns the value of "allow_external" config parameter +// from "apiclient" section. +// +// Returns false if the value is missing or invalid. +func AllowExternal(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "allow_external") +} diff --git a/cmd/neofs-node/config/apiclient/config_test.go b/cmd/neofs-node/config/apiclient/config_test.go index ca77447ed..a7da7d553 100644 --- a/cmd/neofs-node/config/apiclient/config_test.go +++ b/cmd/neofs-node/config/apiclient/config_test.go @@ -16,6 +16,7 @@ func TestApiclientSection(t *testing.T) { require.Equal(t, apiclientconfig.DialTimeoutDefault, apiclientconfig.DialTimeout(empty)) require.Equal(t, apiclientconfig.StreamTimeoutDefault, apiclientconfig.StreamTimeout(empty)) + require.False(t, apiclientconfig.AllowExternal(empty)) }) const path = "../../../../config/example/node" @@ -23,6 +24,7 @@ func TestApiclientSection(t *testing.T) { var fileConfigTest = func(c *config.Config) { require.Equal(t, 15*time.Second, apiclientconfig.DialTimeout(c)) require.Equal(t, 20*time.Second, apiclientconfig.StreamTimeout(c)) + require.True(t, apiclientconfig.AllowExternal(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/neofs-node/container.go b/cmd/neofs-node/container.go index d271a11bf..2cbe30938 100644 --- a/cmd/neofs-node/container.go +++ b/cmd/neofs-node/container.go @@ -507,6 +507,10 @@ func (c *cfg) NumberOfAddresses() int { return c.addressNum() } +func (c *cfg) ExternalAddresses() []string { + return c.cfgNodeInfo.localInfo.ExternalAddresses() +} + func (c *usedSpaceService) PublicKey() []byte { return nodeKeyFromNetmap(c.cfg) } @@ -519,6 +523,10 @@ func (c *usedSpaceService) NumberOfAddresses() int { return c.cfg.addressNum() } +func (c *usedSpaceService) ExternalAddresses() []string { + return c.cfg.ExternalAddresses() +} + func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *containerV2.AnnounceUsedSpaceRequest) (*containerV2.AnnounceUsedSpaceResponse, error) { var passedRoute []loadroute.ServerInfo @@ -577,6 +585,10 @@ func (*containerOnlyKeyRemoteServerInfo) NumberOfAddresses() int { return 0 } +func (*containerOnlyKeyRemoteServerInfo) ExternalAddresses() []string { + return nil +} + func (l *loadPlacementBuilder) isNodeFromContainerKey(epoch uint64, cnr cid.ID, key []byte) (bool, error) { cnrNodes, _, err := l.buildPlacement(epoch, cnr) if err != nil { diff --git a/cmd/neofs-node/reputation/common/util.go b/cmd/neofs-node/reputation/common/util.go index 0e34f34f5..b1acf0539 100644 --- a/cmd/neofs-node/reputation/common/util.go +++ b/cmd/neofs-node/reputation/common/util.go @@ -44,6 +44,10 @@ func (*OnlyKeyRemoteServerInfo) NumberOfAddresses() int { return 0 } +func (*OnlyKeyRemoteServerInfo) ExternalAddresses() []string { + return nil +} + const invalidPrmValFmt = "invalid parameter %s (%T):%v" func PanicOnPrmValue(n string, v interface{}) { diff --git a/config/example/node.env b/config/example/node.env index 89116388d..0b259365f 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -70,6 +70,7 @@ NEOFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2 # API Client section NEOFS_APICLIENT_DIAL_TIMEOUT=15s NEOFS_APICLIENT_STREAM_TIMEOUT=20s +NEOFS_APICLIENT_ALLOW_EXTERNAL=true # Policer section NEOFS_POLICER_HEAD_TIMEOUT=15s diff --git a/config/example/node.json b/config/example/node.json index 34a683650..c098e8f6d 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -113,7 +113,8 @@ }, "apiclient": { "dial_timeout": "15s", - "stream_timeout": "20s" + "stream_timeout": "20s", + "allow_external": true }, "policer": { "head_timeout": "15s" diff --git a/config/example/node.yaml b/config/example/node.yaml index f531687d6..38112e7fd 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -92,6 +92,7 @@ morph: apiclient: dial_timeout: 15s # timeout for NEOFS API client connection stream_timeout: 20s # timeout for individual operations in a streaming RPC + allow_external: true # allow to fallback to addresses in `ExternalAddr` attribute policer: head_timeout: 15s # timeout for the Policer HEAD remote operation diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 1a85749b2..6ed337460 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -39,6 +39,8 @@ type MultiAddressClient interface { type NodeInfo struct { addrGroup network.AddressGroup + externalAddrGroup network.AddressGroup + key []byte } @@ -52,6 +54,16 @@ func (x NodeInfo) AddressGroup() network.AddressGroup { return x.addrGroup } +// SetExternalAddressGroup sets an external group of network addresses. +func (x *NodeInfo) SetExternalAddressGroup(v network.AddressGroup) { + x.externalAddrGroup = v +} + +// ExternalAddressGroup returns a group of network addresses. +func (x NodeInfo) ExternalAddressGroup() network.AddressGroup { + return x.externalAddrGroup +} + // SetPublicKey sets a public key in a binary format. // // Argument must not be mutated. diff --git a/pkg/core/client/util.go b/pkg/core/client/util.go index fa0c0225e..fb3e9e008 100644 --- a/pkg/core/client/util.go +++ b/pkg/core/client/util.go @@ -8,9 +8,10 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/client" ) -func nodeInfoFromKeyAddr(dst *NodeInfo, k []byte, a network.AddressGroup) { +func nodeInfoFromKeyAddr(dst *NodeInfo, k []byte, a, external network.AddressGroup) { dst.SetPublicKey(k) dst.SetAddressGroup(a) + dst.SetExternalAddressGroup(external) } // NodeInfoFromRawNetmapElement fills NodeInfo structure from the interface of raw netmap member's descriptor. @@ -20,6 +21,7 @@ func NodeInfoFromRawNetmapElement(dst *NodeInfo, info interface { PublicKey() []byte IterateAddresses(func(string) bool) NumberOfAddresses() int + ExternalAddresses() []string }) error { var a network.AddressGroup @@ -28,7 +30,14 @@ func NodeInfoFromRawNetmapElement(dst *NodeInfo, info interface { return fmt.Errorf("parse network address: %w", err) } - nodeInfoFromKeyAddr(dst, info.PublicKey(), a) + var external network.AddressGroup + + ext := info.ExternalAddresses() + if len(ext) > 0 { + _ = external.FromStringSlice(ext) + } + + nodeInfoFromKeyAddr(dst, info.PublicKey(), a, external) return nil } @@ -39,8 +48,9 @@ func NodeInfoFromRawNetmapElement(dst *NodeInfo, info interface { func NodeInfoFromNetmapElement(dst *NodeInfo, info interface { PublicKey() []byte Addresses() network.AddressGroup + ExternalAddresses() network.AddressGroup }) { - nodeInfoFromKeyAddr(dst, info.PublicKey(), info.Addresses()) + nodeInfoFromKeyAddr(dst, info.PublicKey(), info.Addresses(), info.ExternalAddresses()) } // AssertKeyResponseCallback returns client response callback which checks if the response was signed by the expected key. diff --git a/pkg/core/netmap/nodes.go b/pkg/core/netmap/nodes.go index 3183a2cce..5e6640216 100644 --- a/pkg/core/netmap/nodes.go +++ b/pkg/core/netmap/nodes.go @@ -17,6 +17,11 @@ func (x Node) PublicKey() []byte { // IterateAddresses iterates over all announced network addresses // and passes them into f. Handler MUST NOT be nil. func (x Node) IterateAddresses(f func(string) bool) { + for _, addr := range (netmap.NodeInfo)(x).ExternalAddresses() { + if f(addr) { + return + } + } (netmap.NodeInfo)(x).IterateNetworkEndpoints(f) } @@ -25,6 +30,11 @@ func (x Node) NumberOfAddresses() int { return (netmap.NodeInfo)(x).NumberOfNetworkEndpoints() } +// ExternalAddresses returns external addresses of a node. +func (x Node) ExternalAddresses() []string { + return (netmap.NodeInfo)(x).ExternalAddresses() +} + // Nodes is a named type of []netmap.NodeInfo which provides interface needed // in the current repository. Nodes is expected to be used everywhere instead // of direct usage of []netmap.NodeInfo, so it represents a type mediator. diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 4806fd7c6..4289fafb7 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -553,11 +553,12 @@ func New(ctx context.Context, log *zap.Logger, cfg *viper.Viper, errChan chan<- ) clientCache := newClientCache(&clientCacheParams{ - Log: log, - Key: &server.key.PrivateKey, - SGTimeout: cfg.GetDuration("audit.timeout.get"), - HeadTimeout: cfg.GetDuration("audit.timeout.head"), - RangeTimeout: cfg.GetDuration("audit.timeout.rangehash"), + Log: log, + Key: &server.key.PrivateKey, + SGTimeout: cfg.GetDuration("audit.timeout.get"), + HeadTimeout: cfg.GetDuration("audit.timeout.head"), + RangeTimeout: cfg.GetDuration("audit.timeout.rangehash"), + AllowExternal: cfg.GetBool("audit.allow_external"), }) server.registerNoErrCloser(clientCache.cache.CloseAll) diff --git a/pkg/innerring/rpc.go b/pkg/innerring/rpc.go index 4e3aa1774..4f229e26b 100644 --- a/pkg/innerring/rpc.go +++ b/pkg/innerring/rpc.go @@ -37,6 +37,8 @@ type ( Log *zap.Logger Key *ecdsa.PrivateKey + AllowExternal bool + SGTimeout, HeadTimeout, RangeTimeout time.Duration } ) @@ -44,7 +46,7 @@ type ( func newClientCache(p *clientCacheParams) *ClientCache { return &ClientCache{ log: p.Log, - cache: cache.NewSDKClientCache(cache.ClientCacheOpts{}), + cache: cache.NewSDKClientCache(cache.ClientCacheOpts{AllowExternal: p.AllowExternal}), key: p.Key, sgTimeout: p.SGTimeout, headTimeout: p.HeadTimeout, diff --git a/pkg/network/cache/client.go b/pkg/network/cache/client.go index 027e90a4b..c77078eb7 100644 --- a/pkg/network/cache/client.go +++ b/pkg/network/cache/client.go @@ -13,9 +13,10 @@ type ( // ClientCache is a structure around neofs-sdk-go/client to reuse // already created clients. ClientCache struct { - mu *sync.RWMutex - clients map[string]*multiClient - opts ClientCacheOpts + mu *sync.RWMutex + clients map[string]*multiClient + opts ClientCacheOpts + allowExternal bool } ClientCacheOpts struct { @@ -23,6 +24,7 @@ type ( StreamTimeout time.Duration Key *ecdsa.PrivateKey ResponseCallback func(client.ResponseMetaInfo) error + AllowExternal bool } ) @@ -30,15 +32,19 @@ type ( // `opts` are used for new client creation. func NewSDKClientCache(opts ClientCacheOpts) *ClientCache { return &ClientCache{ - mu: new(sync.RWMutex), - clients: make(map[string]*multiClient), - opts: opts, + mu: new(sync.RWMutex), + clients: make(map[string]*multiClient), + opts: opts, + allowExternal: opts.AllowExternal, } } // Get function returns existing client or creates a new one. func (c *ClientCache) Get(info clientcore.NodeInfo) (clientcore.Client, error) { netAddr := info.AddressGroup() + if c.allowExternal { + netAddr = append(netAddr, info.ExternalAddressGroup()...) + } cacheKey := string(info.PublicKey()) c.mu.RLock() diff --git a/pkg/network/group.go b/pkg/network/group.go index e57c59748..657d752e0 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -75,6 +75,27 @@ type MultiAddressIterator interface { NumberOfAddresses() int } +// FromStringSlice forms AddressGroup from a string slice. +// +// Returns an error in the absence of addresses or if any of the addresses are incorrect. +func (x *AddressGroup) FromStringSlice(addr []string) error { + if len(addr) == 0 { + return errors.New("missing network addresses") + } + + res := make(AddressGroup, len(addr)) + for i := range addr { + var a Address + if err := a.FromString(addr[i]); err != nil { + return err // invalid format, ignore the whole field + } + res[i] = a + } + + *x = res + return nil +} + // FromIterator forms AddressGroup from MultiAddressIterator structure. // The result is sorted with sort.Sort. // diff --git a/pkg/network/group_test.go b/pkg/network/group_test.go new file mode 100644 index 000000000..f53526163 --- /dev/null +++ b/pkg/network/group_test.go @@ -0,0 +1,69 @@ +package network + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAddressGroup_FromStringSlice(t *testing.T) { + addrs := []string{ + "/dns4/node1.neofs/tcp/8080", + "/dns4/node2.neofs/tcp/1234/tls", + } + expected := make(AddressGroup, len(addrs)) + for i := range addrs { + expected[i] = Address{buildMultiaddr(addrs[i], t)} + } + + var ag AddressGroup + t.Run("empty", func(t *testing.T) { + require.Error(t, ag.FromStringSlice(nil)) + }) + + require.NoError(t, ag.FromStringSlice(addrs)) + require.Equal(t, expected, ag) + + t.Run("error is returned, group is unchanged", func(t *testing.T) { + require.Error(t, ag.FromStringSlice([]string{"invalid"})) + require.Equal(t, expected, ag) + }) +} + +func TestAddressGroup_FromIterator(t *testing.T) { + addrs := testIterator{ + "/dns4/node1.neofs/tcp/8080", + "/dns4/node2.neofs/tcp/1234/tls", + } + expected := make(AddressGroup, len(addrs)) + for i := range addrs { + expected[i] = Address{buildMultiaddr(addrs[i], t)} + } + sort.Sort(expected) + + var ag AddressGroup + t.Run("empty", func(t *testing.T) { + require.Error(t, ag.FromIterator(testIterator{})) + }) + + require.NoError(t, ag.FromIterator(addrs)) + require.Equal(t, expected, ag) + + t.Run("error is returned, group is unchanged", func(t *testing.T) { + require.Error(t, ag.FromIterator(testIterator{"invalid"})) + require.Equal(t, expected, ag) + }) +} + +type testIterator []string + +func (t testIterator) IterateAddresses(f func(string) bool) { + for i := range t { + f(t[i]) + } +} + +func (t testIterator) NumberOfAddresses() int { + return len(t) +} diff --git a/pkg/services/container/announcement/load/route/deps.go b/pkg/services/container/announcement/load/route/deps.go index 8a27d7956..bef958c41 100644 --- a/pkg/services/container/announcement/load/route/deps.go +++ b/pkg/services/container/announcement/load/route/deps.go @@ -19,6 +19,9 @@ type ServerInfo interface { // Returns number of server's network addresses. NumberOfAddresses() int + + // ExternalAddresses returns external node's addresses. + ExternalAddresses() []string } // Builder groups methods to route values in the network. diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 66a56d1e1..5e7290926 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -126,6 +126,8 @@ func flatNodes(ns [][]netmap.NodeInfo) [][]netmap.NodeInfo { type Node struct { addresses network.AddressGroup + externalAddresses network.AddressGroup + key []byte } @@ -134,6 +136,11 @@ func (x Node) Addresses() network.AddressGroup { return x.addresses } +// ExternalAddresses returns group of network addresses. +func (x Node) ExternalAddresses() network.AddressGroup { + return x.externalAddresses +} + // PublicKey returns public key in a binary format. Should not be mutated. func (x Node) PublicKey() []byte { return x.key @@ -167,6 +174,12 @@ func (t *Traverser) Next() []Node { return nil } + ext := t.vectors[0][i].ExternalAddresses() + if len(ext) > 0 { + // Ignore the error if this field is incorrectly formed. + _ = nodes[i].externalAddresses.FromStringSlice(ext) + } + nodes[i].key = t.vectors[0][i].PublicKey() } diff --git a/pkg/services/reputation/common/deps.go b/pkg/services/reputation/common/deps.go index 0a10b1b51..69cd28e33 100644 --- a/pkg/services/reputation/common/deps.go +++ b/pkg/services/reputation/common/deps.go @@ -77,4 +77,7 @@ type ServerInfo interface { // Returns number of server's network addresses. NumberOfAddresses() int + + // ExternalAddresses returns external addresses of a node. + ExternalAddresses() []string } diff --git a/pkg/services/reputation/common/managers.go b/pkg/services/reputation/common/managers.go index 602ec86ed..646ba6c1d 100644 --- a/pkg/services/reputation/common/managers.go +++ b/pkg/services/reputation/common/managers.go @@ -69,6 +69,10 @@ func (x nodeServer) NumberOfAddresses() int { return (apiNetmap.NodeInfo)(x).NumberOfNetworkEndpoints() } +func (x nodeServer) ExternalAddresses() []string { + return (apiNetmap.NodeInfo)(x).ExternalAddresses() +} + // BuildManagers sorts nodes in NetMap with HRW algorithms and // takes the next node after the current one as the only manager. func (mb *managerBuilder) BuildManagers(epoch uint64, p apireputation.PeerID) ([]ServerInfo, error) {