frostfs-node/pkg/network/peers/peers.go

package peers

import (
	"context"
	"net"
	"sync"
	"time"

	"github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
	"github.com/nspcc-dev/neofs-node/pkg/network"
	"github.com/pkg/errors"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/status"
)

type (
	// Interface is an interface of network connections controller.
	Interface interface {
		Job(context.Context)
		GRPCConnector
	}

	// GRPCConnector is an interface of gRPC virtual connector.
	GRPCConnector interface {
		GRPCConnection(ctx context.Context, maddr multiaddr.Multiaddr) (*grpc.ClientConn, error)
	}

	// Params groups the parameters of Interface.
	Params struct {
		Logger           *zap.Logger
		ConnectionTTL    time.Duration
		ConnectionIDLE   time.Duration
		MetricsTimeout   time.Duration
		KeepAliveTTL     time.Duration
		KeepAlivePingTTL time.Duration
	}

	connItem struct {
		sync.RWMutex
		conn *grpc.ClientConn
		used time.Time
	}

	iface struct {
		log  *zap.Logger
		tick time.Duration
		idle time.Duration

		keepAlive time.Duration
		pingTTL   time.Duration

		metricsTimeout time.Duration

		grpc struct {
			// globalMutex used by garbage collector and other high
			globalMutex *sync.RWMutex
			// bookMutex resolves concurrent access to the new connection
			bookMutex *sync.RWMutex
			// connBook contains connection info
			// it's mutex resolves concurrent access to existed connection
			connBook map[string]*connItem
		}
	}
)

const (
	defaultCloseTimer       = 30 * time.Second
	defaultConIdleTTL       = 30 * time.Second
	defaultKeepAliveTTL     = 5 * time.Second
	defaultMetricsTimeout   = 5 * time.Second
	defaultKeepAlivePingTTL = 50 * time.Millisecond
)

var errNilMultiaddr = errors.New("empty multi-address")

func (s *iface) removeGRPCConnection(addr string) error {
	if gCon, ok := s.grpc.connBook[addr]; ok && gCon.conn != nil {
		if err := gCon.conn.Close(); err != nil {
			state, ok := status.FromError(err)
			if !ok {
				return err
			}

			s.log.Debug("error state",
				zap.String("address", addr),
				zap.Any("code", state.Code()),
				zap.String("state", state.Message()),
				zap.Any("details", state.Details()))
		}
	}

	delete(s.grpc.connBook, addr)

	return nil
}

func isGRPCClosed(con *grpc.ClientConn) bool {
	switch con.GetState() {
	case connectivity.Idle, connectivity.Connecting, connectivity.Ready:
		return false
	default:
		// connectivity.TransientFailure, connectivity.Shutdown
		return true
	}
}

func convertAddress(maddr multiaddr.Multiaddr) (string, error) {
	if maddr == nil {
		return "", errNilMultiaddr
	}

	addr, err := manet.ToNetAddr(maddr)
	if err != nil {
		return "", errors.Wrapf(err, "could not convert address `%s`", maddr)
	}

	return addr.String(), nil
}

// GRPCConnection creates gRPC connection over peers connection.
func (s *iface) GRPCConnection(ctx context.Context, maddr multiaddr.Multiaddr) (*grpc.ClientConn, error) {
	addr, err := convertAddress(maddr)
	if err != nil {
		return nil, errors.Wrapf(err, "could not convert `%v`", maddr)
	}

	// Get global mutex on read.
	// All high level function e.g. peers garbage collector
	// or shutdown must use globalMutex.Lock instead
	s.grpc.globalMutex.RLock()

	// Get connection item from connection book or create a new one.
	// Concurrent map access resolved by bookMutex.
	s.grpc.bookMutex.Lock()

	item, ok := s.grpc.connBook[addr]
	if !ok {
		item = new(connItem)
		s.grpc.connBook[addr] = item
	}

	s.grpc.bookMutex.Unlock()

	// Now lock connection item.
	// This denies concurrent access to the same address,
	// but allows concurrent access to a different addresses.
	item.Lock()

	if item.conn != nil && !isGRPCClosed(item.conn) {
		item.used = time.Now()

		item.Unlock()
		s.grpc.globalMutex.RUnlock()

		return item.conn, nil
	}

	// Если вышеописанные строки переместить внутрь WithDialer,
	// мы получим сломанный коннекшн, но ошибка не будет возвращена,
	// поэтому мы сначала проверяем коннекшн и лишь потом возвращаем
	// *gRPC.ClientConn
	//
	// Это будет работать с `grpc.WithBlock()`, см. ниже
	conn, err := grpc.DialContext(ctx, maddr.String(),
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                s.pingTTL,
			Timeout:             s.keepAlive,
			PermitWithoutStream: true,
		}),
		// TODO: we must provide grpc.WithInsecure() or set credentials
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
			return network.Dial(ctx, maddr)
		}),
	)
	if err == nil {
		item.conn = conn
		item.used = time.Now()
	}

	item.Unlock()
	s.grpc.globalMutex.RUnlock()

	return conn, err
}

// New create iface instance and check arguments.
func New(p Params) (Interface, error) {
	if p.ConnectionTTL <= 0 {
		p.ConnectionTTL = defaultCloseTimer
	}

	if p.ConnectionIDLE <= 0 {
		p.ConnectionIDLE = defaultConIdleTTL
	}

	if p.KeepAliveTTL <= 0 {
		p.KeepAliveTTL = defaultKeepAliveTTL
	}

	if p.KeepAlivePingTTL <= 0 {
		p.KeepAlivePingTTL = defaultKeepAlivePingTTL
	}

	if p.MetricsTimeout <= 0 {
		p.MetricsTimeout = defaultMetricsTimeout
	}

	return &iface{
		tick: p.ConnectionTTL,
		idle: p.ConnectionIDLE,

		keepAlive: p.KeepAliveTTL,
		pingTTL:   p.KeepAlivePingTTL,

		metricsTimeout: p.MetricsTimeout,

		log: p.Logger,
		grpc: struct {
			globalMutex *sync.RWMutex
			bookMutex   *sync.RWMutex
			connBook    map[string]*connItem
		}{
			globalMutex: new(sync.RWMutex),
			bookMutex:   new(sync.RWMutex),
			connBook:    make(map[string]*connItem),
		},
	}, nil
}