[#1422] node: Add dialer source to config

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

cmd/frostfs-node/config.go

@@ -26,12 +26,14 @@ import (
 	fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree"
 	loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger"
 	morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/multinet"
 	nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
 	objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
 	replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
 	tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics"
+	internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
 	frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid"
@@ -436,6 +438,8 @@ type shared struct {
 	metricsCollector *metrics.NodeMetrics
 
 	metricsSvc *objectService.MetricCollector
+
+	dialerSource *internalNet.DialerSource
 }
 
 // dynamicConfiguration stores parameters of the
@@ -760,6 +764,16 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt
 	persistate, err := state.NewPersistentStorage(nodeconfig.PersistentState(appCfg).Path())
 	fatalOnErr(err)
 
+	ds, err := internalNet.NewDialerSource(internalNet.Config{
+		SourceBasedRoutingEnabled: multinet.Enabled(appCfg),
+		SourceBasedSubnets:        multinet.Subnets(appCfg),
+		SourceBasedBalancer:       multinet.Balancer(appCfg),
+		SourceBasedRestrict:       multinet.Restrict(appCfg),
+		SourceBasedFallbackDelay:  multinet.FallbackDelay(appCfg),
+		DialTimeout:               apiclientconfig.DialTimeout(appCfg),
+	})
+	fatalOnErr(err)
+
 	cacheOpts := cache.ClientCacheOpts{
 		DialTimeout:      apiclientconfig.DialTimeout(appCfg),
 		StreamTimeout:    apiclientconfig.StreamTimeout(appCfg),
@@ -778,6 +792,7 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt
 		putClientCache:   cache.NewSDKClientCache(cacheOpts),
 		persistate:       persistate,
 		metricsCollector: metrics.NewNodeMetrics(),
+		dialerSource:     ds,
 	}
 }
 
@@ -1336,6 +1351,18 @@ func (c *cfg) reloadConfig(ctx context.Context) {
 		}
 	}
 
+	if err := c.dialerSource.Update(internalNet.Config{
+		SourceBasedRoutingEnabled: multinet.Enabled(c.appCfg),
+		SourceBasedSubnets:        multinet.Subnets(c.appCfg),
+		SourceBasedBalancer:       multinet.Balancer(c.appCfg),
+		SourceBasedRestrict:       multinet.Restrict(c.appCfg),
+		SourceBasedFallbackDelay:  multinet.FallbackDelay(c.appCfg),
+		DialTimeout:               apiclientconfig.DialTimeout(c.appCfg),
+	}); err != nil {
+		c.log.Error(logs.FailedToUpdateMultinetConfiguration, zap.Error(err))
+		return
+	}
+
 	c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
 }
 

go.mod

@@ -11,6 +11,7 @@ require (
 	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573
 	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa
 	git.frostfs.info/TrueCloudLab/hrw v1.2.1
+	git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241008110111-f65c788d7392
 	git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88
 	git.frostfs.info/TrueCloudLab/tzhash v1.8.0
 	git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
@@ -115,6 +116,8 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
 	github.com/twmb/murmur3 v1.1.8 // indirect
 	github.com/urfave/cli v1.22.14 // indirect
+	github.com/vishvananda/netlink v1.1.0 // indirect
+	github.com/vishvananda/netns v0.0.4 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect

internal/logs/logs.go

@@ -540,4 +540,5 @@ const (
 	WritecacheShrinkSkippedNotEmpty                                      = "writecache shrink skipped: not empty"
 	BlobovniczatreeFailedToRemoveRebuildTempFile                         = "failed to remove rebuild temp file"
 	WritecacheCantGetObject                                              = "can't get an object from fstree"
+	FailedToUpdateMultinetConfiguration                                  = "failed to update multinet configuration"
 )

internal/net/config.go

@@ -0,0 +1,39 @@
+package net
+
+import (
+	"net"
+	"slices"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/multinet"
+)
+
+type Config struct {
+	SourceBasedRoutingEnabled bool
+	SourceBasedSubnets        []string
+	SourceBasedBalancer       string
+	SourceBasedRestrict       bool
+	SourceBasedFallbackDelay  time.Duration
+	DialTimeout               time.Duration
+}
+
+func (c Config) toMultinetConfig() multinet.Config {
+	return multinet.Config{
+		Subnets:       c.SourceBasedSubnets,
+		Balancer:      multinet.BalancerType(c.SourceBasedBalancer),
+		Restrict:      c.SourceBasedRestrict,
+		FallbackDelay: c.SourceBasedFallbackDelay,
+		Dialer: net.Dialer{
+			Timeout: c.DialTimeout,
+		},
+	}
+}
+
+func (c Config) equals(other Config) bool {
+	return c.SourceBasedRoutingEnabled == other.SourceBasedRoutingEnabled &&
+		slices.Equal(c.SourceBasedSubnets, other.SourceBasedSubnets) &&
+		c.SourceBasedBalancer == other.SourceBasedBalancer &&
+		c.SourceBasedRestrict == other.SourceBasedRestrict &&
+		c.SourceBasedFallbackDelay == other.SourceBasedFallbackDelay &&
+		c.DialTimeout == other.DialTimeout
+}

internal/net/dialer.go

@@ -0,0 +1,10 @@
+package net
+
+import (
+	"context"
+	"net"
+)
+
+type Dialer interface {
+	DialContext(ctx context.Context, network, address string) (net.Conn, error)
+}

internal/net/dialer_source.go

@@ -0,0 +1,81 @@
+package net
+
+import (
+	"net"
+	"sync"
+
+	"git.frostfs.info/TrueCloudLab/multinet"
+)
+
+type DialerSource struct {
+	guard sync.RWMutex
+
+	c Config
+
+	md multinet.Multidialer
+	nw *multinet.NetlinkWatcher
+
+	d *net.Dialer
+}
+
+func NewDialerSource(c Config) (*DialerSource, error) {
+	result := &DialerSource{}
+	if err := result.build(c); err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (s *DialerSource) build(c Config) error {
+	if c.SourceBasedRoutingEnabled {
+		md, err := multinet.NewDialer(c.toMultinetConfig())
+		if err != nil {
+			return err
+		}
+		nw := multinet.NewNetlinkWatcher(md)
+		if err := nw.Start(); err != nil {
+			return err
+		}
+
+		if s.nw != nil {
+			s.nw.Stop()
+		}
+
+		s.d = nil
+		s.md = md
+		s.nw = nw
+		s.c = c
+		return nil
+	}
+
+	if s.nw != nil {
+		s.nw.Stop()
+	}
+	s.md = nil
+	s.nw = nil
+	s.d = &net.Dialer{
+		Timeout: c.DialTimeout,
+	}
+	s.c = c
+	return nil
+}
+
+func (s *DialerSource) Dialer() Dialer {
+	s.guard.RLock()
+	defer s.guard.RUnlock()
+
+	if s.c.SourceBasedRoutingEnabled {
+		return s.md
+	}
+	return s.d
+}
+
+func (s *DialerSource) Update(c Config) error {
+	s.guard.Lock()
+	defer s.guard.Unlock()
+
+	if s.c.equals(c) {
+		return nil
+	}
+	return s.build(c)
+}