realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1689] tree: Dial tree service after create connection on sync

Browse source 
Background trees sync creates grpc connection with
`grpc.WithDefaultCallOptions(grpc.WaitForReady(true))` option. When grpc
connection created with this option, client will wait until a connection
becomes available or the RPC's deadline is reached. As background sync
has no timeout in context, so in case of client is in TRANSIENT_FAILURE
RPC call will hang forever.

Change-Id: I17c8c1d2779bb81c541f47dd0e558e0b8ed2e7c1
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2025-04-24 16:54:47 +03:00
parent 5b6cba04cb
commit 652237500f
Signed by: dstepanov-yadro
GPG key ID: 237AF1A763293BC0
2 changed files with 33 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

36
pkg/services/tree/cache.go
View file

@ -66,7 +66,12 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl
}
}
cc, err := c.dialTreeService(ctx, netmapAddr)
var netAddr network.Address
if err := netAddr.FromString(netmapAddr); err != nil {
return nil, err
}
cc, err := dialTreeService(ctx, netAddr, c.key, c.ds)
lastTry := time.Now()
c.Lock()
@ -83,32 +88,3 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl
return NewTreeServiceClient(cc), nil
}
func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, error) {
var netAddr network.Address
if err := netAddr.FromString(netmapAddr); err != nil {
return nil, err
}
cc, err := createConnection(netAddr, grpc.WithContextDialer(c.ds.GrpcContextDialer()))
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout)
defer cancel()
req := &HealthcheckRequest{
Body: &HealthcheckRequest_Body{},
}
if err := SignMessage(req, c.key); err != nil {
return nil, err
}
// perform some request to check connection
if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil {
_ = cc.Close()
return nil, err
}
return cc, nil
}

28
pkg/services/tree/sync.go
View file

@ -2,6 +2,7 @@ package tree
import (
"context"
"crypto/ecdsa"
"crypto/sha256"
"crypto/tls"
"errors"
@ -14,6 +15,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/net"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
@ -304,7 +306,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64,
continue
}
cc, err := createConnection(a, grpc.WithContextDialer(s.ds.GrpcContextDialer()))
cc, err := dialTreeService(ctx, a, s.key, s.ds)
if err != nil {
s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr))
continue
@ -342,6 +344,30 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64,
return from
}
func dialTreeService(ctx context.Context, netAddr network.Address, key *ecdsa.PrivateKey, ds *net.DialerSource) (*grpc.ClientConn, error) {
cc, err := createConnection(netAddr, grpc.WithContextDialer(ds.GrpcContextDialer()))
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout)
defer cancel()
req := &HealthcheckRequest{
Body: &HealthcheckRequest_Body{},
}
if err := SignMessage(req, key); err != nil {
return nil, err
}
// perform some request to check connection
if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil {
_ = cc.Close()
return nil, err
}
return cc, nil
}
func createConnection(a network.Address, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
host, isTLS, err := client.ParseURI(a.URIAddr())
if err != nil {