TrueCloudLab/frostfs-sdk-go
30
0
Fork 21
Code Issues 29 Pull requests 2 Releases Activity Actions

[#130] pool: Replace expiration value with expiration duration

Browse source 
Based on the applications that use pool, most of them simply
set session token duration to MaxUint64 value. It is completely
understandable, because epochs are incrementing and expiration
value will be surpassed sooner or later, unless it is MaxUint64.

As an alternative I suggest specifying duration instead of
absolute epoch values. Now apps can set duration of 100-200
epochs and pool automatically calculated expiration epoch
base on the network info.

Signed-off-by: Alex Vanin <alexey@nspcc.ru>
This commit is contained in:
Alex Vanin 2022-02-02 16:25:36 +03:00 committed by Angira Kekteeva
parent 2a72f180dc
commit a86c08b3ee
2 changed files with 59 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
pool/pool.go
View file

@ -59,14 +59,14 @@ type Client interface {
// BuilderOptions contains options used to build connection pool. // BuilderOptions contains options used to build connection pool.
type BuilderOptions struct { type BuilderOptions struct {
Key *ecdsa.PrivateKey Key *ecdsa.PrivateKey
Logger *zap.Logger Logger *zap.Logger
NodeConnectionTimeout time.Duration NodeConnectionTimeout time.Duration
NodeRequestTimeout time.Duration NodeRequestTimeout time.Duration
ClientRebalanceInterval time.Duration ClientRebalanceInterval time.Duration
SessionExpirationEpoch uint64 SessionExpirationDuration uint64
nodesParams []*NodesParam nodesParams []*NodesParam
clientBuilder func(opts ...client.Option) (Client, error) clientBuilder func(opts ...client.Option) (Client, error)
} }
type NodesParam struct { type NodesParam struct {
@ -239,12 +239,13 @@ func cfgFromOpts(opts ...CallOption) *callConfig {
var _ Pool = (*pool)(nil) var _ Pool = (*pool)(nil)
type pool struct { type pool struct {
innerPools []*innerPool innerPools []*innerPool
key *ecdsa.PrivateKey key *ecdsa.PrivateKey
owner *owner.ID owner *owner.ID
cancel context.CancelFunc cancel context.CancelFunc
closedCh chan struct{} closedCh chan struct{}
cache *SessionCache cache *SessionCache
stokenDuration uint64
} }
type innerPool struct { type innerPool struct {
@ -264,10 +265,6 @@ func newPool(ctx context.Context, options *BuilderOptions) (Pool, error) {
inner := make([]*innerPool, len(options.nodesParams)) inner := make([]*innerPool, len(options.nodesParams))
var atLeastOneHealthy bool var atLeastOneHealthy bool
var cliPrm client.CreateSessionPrm
cliPrm.SetExp(options.SessionExpirationEpoch)
for i, params := range options.nodesParams { for i, params := range options.nodesParams {
clientPacks := make([]*clientPack, len(params.weights)) clientPacks := make([]*clientPack, len(params.weights))
for j, address := range params.addresses { for j, address := range params.addresses {
@ -279,7 +276,7 @@ func newPool(ctx context.Context, options *BuilderOptions) (Pool, error) {
return nil, err return nil, err
} }
var healthy bool var healthy bool
cliRes, err := c.CreateSession(ctx, cliPrm) cliRes, err := createSessionTokenForDuration(ctx, c, options.SessionExpirationDuration)
if err != nil && options.Logger != nil { if err != nil && options.Logger != nil {
options.Logger.Warn("failed to create neofs session token for client", options.Logger.Warn("failed to create neofs session token for client",
zap.String("address", address), zap.String("address", address),
@ -306,12 +303,13 @@ func newPool(ctx context.Context, options *BuilderOptions) (Pool, error) {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
pool := &pool{ pool := &pool{
innerPools: inner, innerPools: inner,
key: options.Key, key: options.Key,
owner: ownerID, owner: ownerID,
cancel: cancel, cancel: cancel,
closedCh: make(chan struct{}), closedCh: make(chan struct{}),
cache: cache, cache: cache,
stokenDuration: options.SessionExpirationDuration,
} }
go startRebalance(ctx, pool, options) go startRebalance(ctx, pool, options)
return pool, nil return pool, nil
@ -359,12 +357,7 @@ func updateInnerNodesHealth(ctx context.Context, pool *pool, i int, options *Bui
healthyChanged := false healthyChanged := false
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
var ( var prmEndpoint client.EndpointInfoPrm
prmEndpoint client.EndpointInfoPrm
prmSession client.CreateSessionPrm
)
prmSession.SetExp(options.SessionExpirationEpoch)
for j, cPack := range p.clientPacks { for j, cPack := range p.clientPacks {
wg.Add(1) wg.Add(1)
@ -385,7 +378,8 @@ func updateInnerNodesHealth(ctx context.Context, pool *pool, i int, options *Bui
if ok { if ok {
bufferWeights[j] = options.nodesParams[i].weights[j] bufferWeights[j] = options.nodesParams[i].weights[j]
if !cp.healthy { if !cp.healthy {
if cliRes, err := cli.CreateSession(ctx, prmSession); err != nil { cliRes, err := createSessionTokenForDuration(ctx, cli, options.SessionExpirationDuration)
if err != nil {
ok = false ok = false
bufferWeights[j] = 0 bufferWeights[j] = 0
} else { } else {
@ -502,11 +496,7 @@ func (p *pool) conn(ctx context.Context, cfg *callConfig) (*clientPack, []client
cacheKey := formCacheKey(cp.address, key) cacheKey := formCacheKey(cp.address, key)
sessionToken = p.cache.Get(cacheKey) sessionToken = p.cache.Get(cacheKey)
if sessionToken == nil { if sessionToken == nil {
var cliPrm client.CreateSessionPrm cliRes, err := createSessionTokenForDuration(ctx, cp.client, p.stokenDuration)
cliPrm.SetExp(math.MaxUint32)
cliRes, err := cp.client.CreateSession(ctx, cliPrm)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -541,6 +531,24 @@ func (p *pool) checkSessionTokenErr(err error, address string) bool {
return false return false
} }
func createSessionTokenForDuration(ctx context.Context, c Client, dur uint64) (*client.CreateSessionRes, error) {
ni, err := c.NetworkInfo(ctx, client.NetworkInfoPrm{})
if err != nil {
return nil, err
}
epoch := ni.Info().CurrentEpoch()
var prm client.CreateSessionPrm
if math.MaxUint64-epoch < dur {
prm.SetExp(math.MaxUint64)
} else {
prm.SetExp(epoch + dur)
}
return c.CreateSession(ctx, prm)
}
func (p *pool) PutObject(ctx context.Context, params *client.PutObjectParams, opts ...CallOption) (*oid.ID, error) { func (p *pool) PutObject(ctx context.Context, params *client.PutObjectParams, opts ...CallOption) (*oid.ID, error) {
cfg := cfgFromOpts(append(opts, useDefaultSession())...) cfg := cfgFromOpts(append(opts, useDefaultSession())...)
cp, options, err := p.conn(ctx, cfg) cp, options, err := p.conn(ctx, cfg)

14
pool/pool_test.go
View file

@ -48,6 +48,7 @@ func TestBuildPoolCreateSessionFailed(t *testing.T) {
mockClient := NewMockClient(ctrl) mockClient := NewMockClient(ctrl)
mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error session")).AnyTimes() mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error session")).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
return mockClient, nil return mockClient, nil
} }
@ -94,10 +95,12 @@ func TestBuildPoolOneNodeFailed(t *testing.T) {
}).AnyTimes() }).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
mockClient2 := NewMockClient(ctrl2) mockClient2 := NewMockClient(ctrl2)
mockClient2.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient2.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient2.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient2.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
if clientCount == 0 { if clientCount == 0 {
return mockClient, nil return mockClient, nil
@ -153,6 +156,7 @@ func TestOneNode(t *testing.T) {
mockClient := NewMockClient(ctrl) mockClient := NewMockClient(ctrl)
mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(tok, nil) mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(tok, nil)
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
return mockClient, nil return mockClient, nil
} }
@ -190,6 +194,7 @@ func TestTwoNodes(t *testing.T) {
return tok, err return tok, err
}) })
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
return mockClient, nil return mockClient, nil
} }
@ -228,6 +233,7 @@ func TestOneOfTwoFailed(t *testing.T) {
return tok, nil return tok, nil
}).AnyTimes() }).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
mockClient2 := NewMockClient(ctrl2) mockClient2 := NewMockClient(ctrl2)
mockClient2.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(_, _ interface{}, _ ...interface{}) (*session.Token, error) { mockClient2.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(_, _ interface{}, _ ...interface{}) (*session.Token, error) {
@ -238,6 +244,9 @@ func TestOneOfTwoFailed(t *testing.T) {
mockClient2.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).DoAndReturn(func(_ interface{}, _ ...interface{}) (*client.EndpointInfoRes, error) { mockClient2.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).DoAndReturn(func(_ interface{}, _ ...interface{}) (*client.EndpointInfoRes, error) {
return nil, fmt.Errorf("error") return nil, fmt.Errorf("error")
}).AnyTimes() }).AnyTimes()
mockClient2.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).DoAndReturn(func(_ interface{}, _ ...interface{}) (*client.NetworkInfoRes, error) {
return nil, fmt.Errorf("error")
}).AnyTimes()
if clientCount == 0 { if clientCount == 0 {
return mockClient, nil return mockClient, nil
@ -277,6 +286,7 @@ func TestTwoFailed(t *testing.T) {
mockClient := NewMockClient(ctrl) mockClient := NewMockClient(ctrl)
mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error")).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error")).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error")).AnyTimes()
return mockClient, nil return mockClient, nil
} }
@ -379,6 +389,7 @@ func TestPriority(t *testing.T) {
return tok, nil return tok, nil
}).AnyTimes() }).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error")).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error")).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error")).AnyTimes()
mockClient2 := NewMockClient(ctrl2) mockClient2 := NewMockClient(ctrl2)
mockClient2.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(_, _ interface{}, _ ...interface{}) (*session.Token, error) { mockClient2.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(_, _ interface{}, _ ...interface{}) (*session.Token, error) {
@ -387,6 +398,7 @@ func TestPriority(t *testing.T) {
return tok, nil return tok, nil
}).AnyTimes() }).AnyTimes()
mockClient2.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient2.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient2.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
if clientCount == 0 { if clientCount == 0 {
return mockClient, nil return mockClient, nil
@ -489,6 +501,7 @@ func TestSessionTokenOwner(t *testing.T) {
mockClient := NewMockClient(ctrl) mockClient := NewMockClient(ctrl)
mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(&client.CreateSessionRes{}, nil).AnyTimes() mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(&client.CreateSessionRes{}, nil).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(&client.EndpointInfoRes{}, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
return mockClient, nil return mockClient, nil
} }
@ -527,6 +540,7 @@ func TestWaitPresence(t *testing.T) {
mockClient := NewMockClient(ctrl) mockClient := NewMockClient(ctrl)
mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().NetworkInfo(gomock.Any(), gomock.Any()).Return(&client.NetworkInfoRes{}, nil).AnyTimes()
mockClient.EXPECT().GetContainer(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() mockClient.EXPECT().GetContainer(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
cache, err := NewCache() cache, err := NewCache()