Add connection pool implementation (part 2)

Signed-off-by: Pavel Korotkov <pavel@nspcc.ru>
This commit is contained in:
Pavel Korotkov 2021-04-05 20:10:03 +03:00 committed by Pavel Korotkov
parent 62a03251ce
commit a44551d42b
6 changed files with 70 additions and 57 deletions

View file

@ -9,7 +9,7 @@ HTTP_GW_KEEPALIVE_TIMEOUT=300s
HTTP_GW_KEEPALIVE_TIME=120s
HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True
HTTP_GW_CONN_TTL=1h
HTTP_GW_PEERS_0_WEIGHT=0.6
HTTP_GW_PEERS_0_WEIGHT=0.4
HTTP_GW_PEERS_0_ADDRESS=s01.neofs.devenv:8080
HTTP_GW_PEERS_1_WEIGHT=0.4
HTTP_GW_PEERS_1_WEIGHT=0.6
HTTP_GW_PEERS_1_ADDRESS=s02.neofs.devenv:8080

View file

@ -3,7 +3,7 @@ package connections
import "math/rand"
// https://www.keithschwarz.com/darts-dice-coins/
type Generator struct {
type Sampler struct {
randomGenerator *rand.Rand
probabilities []float64
alias []int
@ -22,8 +22,8 @@ func (wl *workList) pop() int {
return n
}
func NewGenerator(probabilities []float64, source rand.Source) *Generator {
generator := &Generator{}
func NewSampler(probabilities []float64, source rand.Source) *Sampler {
generator := &Sampler{}
var (
small workList
large workList
@ -66,7 +66,7 @@ func NewGenerator(probabilities []float64, source rand.Source) *Generator {
return generator
}
func (g *Generator) Next() int {
func (g *Sampler) Next() int {
n := len(g.alias)
i := g.randomGenerator.Intn(n)
if g.randomGenerator.Float64() < g.probabilities[i] {

View file

@ -6,6 +6,7 @@ import (
"errors"
"math"
"math/rand"
"sync"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
@ -17,6 +18,8 @@ type PoolBuilderOptions struct {
NodeConnectionTimeout time.Duration
NodeRequestTimeout time.Duration
ClientRebalanceInterval time.Duration
weights []float64
connections []*grpc.ClientConn
}
type PoolBuilder struct {
@ -50,7 +53,9 @@ func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) (
}
cons[i] = con
}
return new(pb.weights, options.Key, cons)
options.weights = pb.weights
options.connections = cons
return new(ctx, options)
}
type Pool interface {
@ -58,29 +63,70 @@ type Pool interface {
}
type pool struct {
generator *Generator
clients []client.Client
lock sync.RWMutex
sampler *Sampler
clients []client.Client
healthy []bool
}
func new(weights []float64, key *ecdsa.PrivateKey, connections []*grpc.ClientConn) (Pool, error) {
clients := make([]client.Client, len(weights))
for i, con := range connections {
c, err := client.New(client.WithDefaultPrivateKey(key), client.WithGRPCConnection(con))
func new(ctx context.Context, options *PoolBuilderOptions) (Pool, error) {
n := len(options.weights)
clients := make([]client.Client, n)
healthy := make([]bool, n)
for i, con := range options.connections {
c, err := client.New(client.WithDefaultPrivateKey(options.Key), client.WithGRPCConnection(con))
if err != nil {
return nil, err
}
clients[i] = c
healthy[i] = true
}
source := rand.NewSource(time.Now().UnixNano())
return &pool{
generator: NewGenerator(weights, source),
clients: clients,
}, nil
pool := &pool{
sampler: NewSampler(options.weights, source),
clients: clients,
healthy: healthy,
}
go func() {
ticker := time.NewTimer(options.ClientRebalanceInterval)
for range ticker.C {
ok := true
for i, client := range pool.clients {
func() {
tctx, c := context.WithTimeout(ctx, options.NodeRequestTimeout)
defer c()
if _, err := client.EndpointInfo(tctx); err != nil {
ok = false
}
pool.lock.Lock()
pool.healthy[i] = ok
pool.lock.Unlock()
}()
}
ticker.Reset(options.ClientRebalanceInterval)
}
}()
return pool, nil
}
func (p *pool) Client() client.Client {
p.lock.RLock()
defer p.lock.RUnlock()
if len(p.clients) == 1 {
return p.clients[0]
if p.healthy[0] {
return p.clients[0]
}
return nil
}
return p.clients[p.generator.Next()]
var i *int = nil
for k := 0; k < 10; k++ {
i_ := p.sampler.Next()
if p.healthy[i_] {
i = &i_
}
}
if i != nil {
return p.clients[*i]
}
return nil
}

View file

@ -8,16 +8,13 @@ import (
)
var (
globalContext context.Context
globalContextOnce sync.Once
globalContextBarrier = make(chan struct{})
globalContext context.Context
globalContextOnce sync.Once
)
func Context() context.Context {
globalContextOnce.Do(func() {
globalContext, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
close(globalContextBarrier)
})
<-globalContextBarrier
return globalContext
}

View file

@ -1,33 +0,0 @@
package main
import (
"github.com/fasthttp/router"
"github.com/valyala/fasthttp"
)
type stater func() error
const (
healthyState = "NeoFS HTTP Gateway is "
defaultContentType = "text/plain; charset=utf-8"
)
func attachHealthy(r *router.Router, e stater) {
r.GET("/-/ready/", func(ctx *fasthttp.RequestCtx) {
ctx.SetStatusCode(fasthttp.StatusOK)
ctx.SetBodyString(healthyState + "ready")
})
r.GET("/-/healthy/", func(c *fasthttp.RequestCtx) {
code := fasthttp.StatusOK
msg := "healthy"
if err := e(); err != nil {
msg = "unhealthy: " + err.Error()
code = fasthttp.StatusBadRequest
}
c.Response.Reset()
c.SetStatusCode(code)
c.SetContentType(defaultContentType)
c.SetBodyString(healthyState + msg)
})
}

View file

@ -80,6 +80,9 @@ type neofsClientPlant struct {
func (cp *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) {
c := cp.pool.Client()
if c == nil {
return nil, nil, errors.New("failed to peek a healthy node to connect to")
}
st, err := c.CreateSession(ctx, math.MaxUint64)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create reusable neofs session token")