TrueCloudLab/frostfs-node
20
1
Fork 28
Code Issues 114 Pull requests 4 Projects Releases 2 Activity Actions

Support morph mTLS #1170

Merged
fyrchik merged 4 commits from fyrchik/frostfs-node:node-mtls into master 2024-09-04 19:51:09 +00:00
Conversation 1 Commits 4 Files changed 12 +103 -17
12 changed files with 103 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
cmd/frostfs-adm/internal/modules/morph/helper/n3client.go
View file

@ -2,6 +2,7 @@ package helper
import (
"context"
"crypto/tls"
"errors"
"fmt"
"time"
@ -60,9 +61,23 @@ func GetN3Client(v *viper.Viper) (Client, error) {
if endpoint == "" {
return nil, errors.New("missing endpoint")
}
var cfg *tls.Config
if rootCAs := v.GetStringSlice("tls.trusted_ca_list"); len(rootCAs) != 0 {
certFile := v.GetString("tls.certificate")
keyFile := v.GetString("tls.key")
tlsConfig, err := rpcclient.TLSClientConfig(rootCAs, certFile, keyFile)
if err != nil {
return nil, err
}
cfg = tlsConfig
}
c, err := rpcclient.New(ctx, endpoint, rpcclient.Options{
MaxConnsPerHost: maxConnsPerHost,
RequestTimeout: requestTimeout,
TLSClientConfig: cfg,
})
if err != nil {
return nil, err

15
cmd/frostfs-node/config/morph/config.go
View file

@ -54,9 +54,20 @@ func RPCEndpoint(c *config.Config) []client.Endpoint {
priority = PriorityDefault
}
var mtlsConfig *client.MTLSConfig
rootCAs := config.StringSliceSafe(s, "trusted_ca_list")
if len(rootCAs) != 0 {
mtlsConfig = &client.MTLSConfig{
TrustedCAList: rootCAs,
KeyFile: config.StringSafe(s, "key"),
CertFile: config.StringSafe(s, "certificate"),
}
}
es = append(es, client.Endpoint{
Address: addr,
Priority: priority,
Address: addr,
Priority: priority,
MTLSConfig: mtlsConfig,
})
}

7
cmd/frostfs-node/config/morph/config_test.go
View file

@ -28,6 +28,13 @@ func TestMorphSection(t *testing.T) {
{
Address: "wss://rpc1.morph.frostfs.info:40341/ws",
Priority: 1,
MTLSConfig: &client.MTLSConfig{
TrustedCAList: []string{
"/path/to/ca.pem",
},
KeyFile: "/path/to/key",
CertFile: "/path/to/cert",
},
},
{
Address: "wss://rpc2.morph.frostfs.info:40341/ws",

3
config/example/node.env
View file

@ -61,6 +61,9 @@ FROSTFS_MORPH_CACHE_TTL=15s
FROSTFS_MORPH_SWITCH_INTERVAL=3m
FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS="wss://rpc1.morph.frostfs.info:40341/ws"
FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY=0
FROSTFS_MORPH_RPC_ENDPOINT_0_TRUSTED_CA_LIST="/path/to/ca.pem"
FROSTFS_MORPH_RPC_ENDPOINT_0_CERTIFICATE="/path/to/cert"
FROSTFS_MORPH_RPC_ENDPOINT_0_KEY="/path/to/key"
FROSTFS_MORPH_RPC_ENDPOINT_1_ADDRESS="wss://rpc2.morph.frostfs.info:40341/ws"
FROSTFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2
FROSTFS_MORPH_APE_CHAIN_CACHE_SIZE=100000

7
config/example/node.json
View file

@ -95,7 +95,12 @@
"rpc_endpoint": [
{
"address": "wss://rpc1.morph.frostfs.info:40341/ws",
"priority": 0
"priority": 0,
"trusted_ca_list": [
"/path/to/ca.pem"
],
"certificate": "/path/to/cert",
"key": "/path/to/key"
},
{
"address": "wss://rpc2.morph.frostfs.info:40341/ws",

4
config/example/node.yaml
View file

@ -84,6 +84,10 @@ morph:
rpc_endpoint: # side chain NEO RPC endpoints; are shuffled and used one by one until the first success
- address: wss://rpc1.morph.frostfs.info:40341/ws
priority: 0
trusted_ca_list:
- "/path/to/ca.pem"
certificate: "/path/to/cert"
key: "/path/to/key"
- address: wss://rpc2.morph.frostfs.info:40341/ws
priority: 2
ape_chain_cache_size: 100000

2
go.mod
View file

@ -127,3 +127,5 @@ require (
lukechampine.com/blake3 v1.2.1 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18

4
go.sum
View file

@ -12,6 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc=
git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d h1:hHk8FWbWgEnwm2I045CaBIrZBjy/o81CehIVOySA/pQ=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=
@ -186,8 +188,6 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk=
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc=
github.com/nspcc-dev/neo-go v0.106.0 h1:YiOdW/GcLmbVSvxMRfD5aytO6n3TDHrEA97VHMawy6g=
github.com/nspcc-dev/neo-go v0.106.0/go.mod h1:9k7vBqqQeePuj4c2CqMN9uPFsfxvwSk9vnQFrkE5eg8=
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k=
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY=
github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM=

15
pkg/innerring/innerring.go
View file

@ -463,9 +463,20 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c
priority = defaultPriority
}
var mtlsConfig *client.MTLSConfig
rootCAs := p.cfg.GetStringSlice(fmt.Sprintf("%s.%d.trusted_ca_list", section, i))
if len(rootCAs) != 0 {
mtlsConfig = &client.MTLSConfig{
TrustedCAList: rootCAs,
KeyFile: p.cfg.GetString(fmt.Sprintf("%s.%d.key", section, i)),
CertFile: p.cfg.GetString(fmt.Sprintf("%s.%d.certificate", section, i)),
}
}
endpoints = append(endpoints, client.Endpoint{
Address: addr,
Priority: priority,
Address: addr,
Priority: priority,
MTLSConfig: mtlsConfig,
})
}

13
pkg/morph/client/constructor.go
View file

@ -141,7 +141,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er
} else {
var endpoint Endpoint
for cli.endpoints.curr, endpoint = range cli.endpoints.list {
cli.client, act, err = cli.newCli(ctx, endpoint.Address)
cli.client, act, err = cli.newCli(ctx, endpoint)
if err != nil {
cli.logger.Warn(logs.FrostFSIRCouldntCreateRPCClientForEndpoint,
zap.Error(err), zap.String("endpoint", endpoint.Address))
@ -162,10 +162,15 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er
return cli, nil
}
func (c *Client) newCli(ctx context.Context, endpoint string) (*rpcclient.WSClient, *actor.Actor, error) {
cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.WSOptions{
func (c *Client) newCli(ctx context.Context, endpoint Endpoint) (*rpcclient.WSClient, *actor.Actor, error) {
cfg, err := endpoint.MTLSConfig.parse()
if err != nil {
return nil, nil, fmt.Errorf("read mtls certificates: %w", err)
}
cli, err := rpcclient.NewWS(ctx, endpoint.Address, rpcclient.WSOptions{
Options: rpcclient.Options{
DialTimeout: c.cfg.dialTimeout,
DialTimeout: c.cfg.dialTimeout,
TLSClientConfig: cfg,
},
})
if err != nil {

22
pkg/morph/client/mtls.go Normal file
View file

@ -0,0 +1,22 @@
package client
import (
"crypto/tls"
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
)
// MTLSConfig represents endpoint mTLS configuration.
type MTLSConfig struct {
TrustedCAList []string
KeyFile string
CertFile string
}
func (m *MTLSConfig) parse() (*tls.Config, error) {
if m == nil {
return nil, nil
}
return rpcclient.TLSClientConfig(m.TrustedCAList, m.CertFile, m.KeyFile)
}

13
pkg/morph/client/multi.go
View file

@ -11,8 +11,9 @@ import (
// Endpoint represents morph endpoint together with its priority.
type Endpoint struct {
Address string
Priority int
Address string
Priority int
MTLSConfig *MTLSConfig
}
type endpoints struct {
@ -38,11 +39,11 @@ func (c *Client) SwitchRPC(ctx context.Context) bool {
// Iterate endpoints in the order of decreasing priority.
for c.endpoints.curr = range c.endpoints.list {
newEndpoint := c.endpoints.list[c.endpoints.curr].Address
newEndpoint := c.endpoints.list[c.endpoints.curr]
cli, act, err := c.newCli(ctx, newEndpoint)
if err != nil {
c.logger.Warn(logs.ClientCouldNotEstablishConnectionToTheSwitchedRPCNode,
zap.String("endpoint", newEndpoint),
zap.String("endpoint", newEndpoint.Address),
zap.Error(err),
)
@ -52,7 +53,7 @@ func (c *Client) SwitchRPC(ctx context.Context) bool {
c.cache.invalidate()
c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished,
zap.String("endpoint", newEndpoint))
zap.String("endpoint", newEndpoint.Address))
c.client = cli
c.setActor(act)
@ -119,7 +120,7 @@ mainLoop:
tryE := e.Address
cli, act, err := c.newCli(ctx, tryE)
cli, act, err := c.newCli(ctx, e)
if err != nil {
c.logger.Warn(logs.ClientCouldNotCreateClientToTheHigherPriorityNode,
zap.String("endpoint", tryE),