frostfs-node/pkg/network/peers/peers.go

235 lines
5.6 KiB
Go
Raw Normal View History

2020-07-24 13:54:03 +00:00
package peers
import (
"context"
"net"
"sync"
"time"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr-net"
"github.com/nspcc-dev/neofs-node/pkg/network"
"github.com/pkg/errors"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/status"
)
type (
// Interface is an interface of network connections controller.
Interface interface {
Job(context.Context)
GRPCConnector
}
// GRPCConnector is an interface of gRPC virtual connector.
GRPCConnector interface {
GRPCConnection(ctx context.Context, maddr multiaddr.Multiaddr) (*grpc.ClientConn, error)
}
// Params groups the parameters of Interface.
Params struct {
Logger *zap.Logger
ConnectionTTL time.Duration
ConnectionIDLE time.Duration
MetricsTimeout time.Duration
KeepAliveTTL time.Duration
KeepAlivePingTTL time.Duration
}
connItem struct {
sync.RWMutex
conn *grpc.ClientConn
used time.Time
}
iface struct {
log *zap.Logger
tick time.Duration
idle time.Duration
keepAlive time.Duration
pingTTL time.Duration
metricsTimeout time.Duration
grpc struct {
// globalMutex used by garbage collector and other high
globalMutex *sync.RWMutex
// bookMutex resolves concurrent access to the new connection
bookMutex *sync.RWMutex
// connBook contains connection info
// it's mutex resolves concurrent access to existed connection
connBook map[string]*connItem
}
}
)
const (
defaultCloseTimer = 30 * time.Second
defaultConIdleTTL = 30 * time.Second
defaultKeepAliveTTL = 5 * time.Second
defaultMetricsTimeout = 5 * time.Second
defaultKeepAlivePingTTL = 50 * time.Millisecond
)
var errNilMultiaddr = errors.New("empty multi-address")
func (s *iface) removeGRPCConnection(addr string) error {
if gCon, ok := s.grpc.connBook[addr]; ok && gCon.conn != nil {
if err := gCon.conn.Close(); err != nil {
state, ok := status.FromError(err)
if !ok {
return err
}
s.log.Debug("error state",
zap.String("address", addr),
zap.Any("code", state.Code()),
zap.String("state", state.Message()),
zap.Any("details", state.Details()))
}
}
delete(s.grpc.connBook, addr)
return nil
}
func isGRPCClosed(con *grpc.ClientConn) bool {
switch con.GetState() {
case connectivity.Idle, connectivity.Connecting, connectivity.Ready:
return false
default:
// connectivity.TransientFailure, connectivity.Shutdown
return true
}
}
func convertAddress(maddr multiaddr.Multiaddr) (string, error) {
if maddr == nil {
return "", errNilMultiaddr
}
addr, err := manet.ToNetAddr(maddr)
if err != nil {
return "", errors.Wrapf(err, "could not convert address `%s`", maddr)
}
return addr.String(), nil
}
// GRPCConnection creates gRPC connection over peers connection.
func (s *iface) GRPCConnection(ctx context.Context, maddr multiaddr.Multiaddr) (*grpc.ClientConn, error) {
addr, err := convertAddress(maddr)
if err != nil {
return nil, errors.Wrapf(err, "could not convert `%v`", maddr)
}
// Get global mutex on read.
// All high level function e.g. peers garbage collector
// or shutdown must use globalMutex.Lock instead
s.grpc.globalMutex.RLock()
// Get connection item from connection book or create a new one.
// Concurrent map access resolved by bookMutex.
s.grpc.bookMutex.Lock()
item, ok := s.grpc.connBook[addr]
if !ok {
item = new(connItem)
s.grpc.connBook[addr] = item
}
s.grpc.bookMutex.Unlock()
// Now lock connection item.
// This denies concurrent access to the same address,
// but allows concurrent access to a different addresses.
item.Lock()
if item.conn != nil && !isGRPCClosed(item.conn) {
item.used = time.Now()
item.Unlock()
s.grpc.globalMutex.RUnlock()
return item.conn, nil
}
// Если вышеописанные строки переместить внутрь WithDialer,
// мы получим сломанный коннекшн, но ошибка не будет возвращена,
// поэтому мы сначала проверяем коннекшн и лишь потом возвращаем
// *gRPC.ClientConn
//
// Это будет работать с `grpc.WithBlock()`, см. ниже
conn, err := grpc.DialContext(ctx, maddr.String(),
grpc.WithKeepaliveParams(keepalive.ClientParameters{
Time: s.pingTTL,
Timeout: s.keepAlive,
PermitWithoutStream: true,
}),
// TODO: we must provide grpc.WithInsecure() or set credentials
grpc.WithInsecure(),
grpc.WithBlock(),
grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
return network.Dial(ctx, maddr)
}),
)
if err == nil {
item.conn = conn
item.used = time.Now()
}
item.Unlock()
s.grpc.globalMutex.RUnlock()
return conn, err
}
// New create iface instance and check arguments.
func New(p Params) (Interface, error) {
if p.ConnectionTTL <= 0 {
p.ConnectionTTL = defaultCloseTimer
}
if p.ConnectionIDLE <= 0 {
p.ConnectionIDLE = defaultConIdleTTL
}
if p.KeepAliveTTL <= 0 {
p.KeepAliveTTL = defaultKeepAliveTTL
}
if p.KeepAlivePingTTL <= 0 {
p.KeepAlivePingTTL = defaultKeepAlivePingTTL
}
if p.MetricsTimeout <= 0 {
p.MetricsTimeout = defaultMetricsTimeout
}
return &iface{
tick: p.ConnectionTTL,
idle: p.ConnectionIDLE,
keepAlive: p.KeepAliveTTL,
pingTTL: p.KeepAlivePingTTL,
metricsTimeout: p.MetricsTimeout,
log: p.Logger,
grpc: struct {
globalMutex *sync.RWMutex
bookMutex *sync.RWMutex
connBook map[string]*connItem
}{
globalMutex: new(sync.RWMutex),
bookMutex: new(sync.RWMutex),
connBook: make(map[string]*connItem),
},
}, nil
}