TrueCloudLab/frostfs-sdk-go
30
0
Fork 21
Code Issues 30 Pull requests 3 Releases Activity Actions

[#38] Add session token cache

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-10-25 15:57:55 +03:00 committed by Alex Vanin
parent 10b78e74af
commit c651a714fd
6 changed files with 323 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -4,6 +4,7 @@ go 1.16
require (
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1
github.com/bluele/gcache v0.0.2
github.com/golang/mock v1.6.0
github.com/google/uuid v1.2.0
github.com/mr-tron/base58 v1.2.0

BIN
go.sum
View file

Binary file not shown.

47
pool/cache.go Normal file
View file

@ -0,0 +1,47 @@
package pool
import (
"strings"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-api-go/pkg/session"
)
type SessionCache struct {
cache gcache.Cache
}
func NewCache() *SessionCache {
return &SessionCache{
cache: gcache.New(100).Build(),
}
}
func (c *SessionCache) Get(key string) *session.Token {
tokenRaw, err := c.cache.Get(key)
if err != nil {
return nil
}
token, ok := tokenRaw.(*session.Token)
if !ok {
return nil
}
return token
}
func (c *SessionCache) Put(key string, token *session.Token) error {
return c.cache.Set(key, token)
}
func (c *SessionCache) DeleteByPrefix(prefix string) {
for _, key := range c.cache.Keys(false) {
keyStr, ok := key.(string)
if !ok {
continue
}
if strings.HasPrefix(keyStr, prefix) {
c.cache.Remove(key)
}
}
}

277
pool/pool.go
View file

@ -5,10 +5,13 @@ import (
"crypto/ecdsa"
"errors"
"fmt"
"math"
"math/rand"
"strings"
"sync"
"time"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
@ -89,12 +92,34 @@ type Pool interface {
OwnerID() *owner.ID
WaitForContainerPresence(context.Context, *cid.ID, *ContainerPollingParams) error
Close()
PutObjectParam(ctx context.Context, params *client.PutObjectParams, callParam CallParam) (*object.ID, error)
DeleteObjectParam(ctx context.Context, params *client.DeleteObjectParams, callParam CallParam) error
GetObjectParam(ctx context.Context, params *client.GetObjectParams, callParam CallParam) (*object.Object, error)
GetObjectHeaderParam(ctx context.Context, params *client.ObjectHeaderParams, callParam CallParam) (*object.Object, error)
ObjectPayloadRangeDataParam(ctx context.Context, params *client.RangeDataParams, callParam CallParam) ([]byte, error)
ObjectPayloadRangeSHA256Param(ctx context.Context, params *client.RangeChecksumParams, callParam CallParam) ([][32]byte, error)
ObjectPayloadRangeTZParam(ctx context.Context, params *client.RangeChecksumParams, callParam CallParam) ([][64]byte, error)
SearchObjectParam(ctx context.Context, params *client.SearchObjectParams, callParam CallParam) ([]*object.ID, error)
PutContainerParam(ctx context.Context, cnr *container.Container, callParam CallParam) (*cid.ID, error)
GetContainerParam(ctx context.Context, cid *cid.ID, callParam CallParam) (*container.Container, error)
ListContainersParam(ctx context.Context, ownerID *owner.ID, callParam CallParam) ([]*cid.ID, error)
DeleteContainerParam(ctx context.Context, cid *cid.ID, callParam CallParam) error
GetEACLParam(ctx context.Context, cid *cid.ID, callParam CallParam) (*client.EACLWithSignature, error)
SetEACLParam(ctx context.Context, table *eacl.Table, callParam CallParam) error
AnnounceContainerUsedSpaceParam(ctx context.Context, announce []container.UsedSpaceAnnouncement, callParam CallParam) error
}
type clientPack struct {
client client.Client
sessionToken *session.Token
healthy bool
client client.Client
healthy bool
address string
}
type CallParam struct {
Key *ecdsa.PrivateKey
Options []client.CallOption
}
var _ Pool = (*pool)(nil)
@ -102,13 +127,17 @@ var _ Pool = (*pool)(nil)
type pool struct {
lock sync.RWMutex
sampler *Sampler
key *ecdsa.PrivateKey
owner *owner.ID
clientPacks []*clientPack
cancel context.CancelFunc
closedCh chan struct{}
cache *SessionCache
}
func newPool(ctx context.Context, options *BuilderOptions) (Pool, error) {
cache := NewCache()
clientPacks := make([]*clientPack, len(options.weights))
var atLeastOneHealthy bool
for i, address := range options.addresses {
@ -126,8 +155,9 @@ func newPool(ctx context.Context, options *BuilderOptions) (Pool, error) {
zap.Error(err))
} else if err == nil {
healthy, atLeastOneHealthy = true, true
_ = cache.Put(formCacheKey(address, options.Key), st)
}
clientPacks[i] = &clientPack{client: c, sessionToken: st, healthy: healthy}
clientPacks[i] = &clientPack{client: c, healthy: healthy, address: address}
}
if !atLeastOneHealthy {
@ -143,7 +173,15 @@ func newPool(ctx context.Context, options *BuilderOptions) (Pool, error) {
ownerID := owner.NewIDFromNeo3Wallet(wallet)
ctx, cancel := context.WithCancel(ctx)
pool := &pool{sampler: sampler, owner: ownerID, clientPacks: clientPacks, cancel: cancel, closedCh: make(chan struct{})}
pool := &pool{
sampler: sampler,
key: options.Key,
owner: ownerID,
clientPacks: clientPacks,
cancel: cancel,
closedCh: make(chan struct{}),
cache: cache,
}
go startRebalance(ctx, pool, options)
return pool, nil
}
@ -175,33 +213,34 @@ func updateNodesHealth(ctx context.Context, p *pool, options *BuilderOptions, bu
go func(i int, client client.Client) {
defer wg.Done()
var (
tkn *session.Token
err error
)
ok := true
tctx, c := context.WithTimeout(ctx, options.NodeRequestTimeout)
defer c()
if _, err = client.EndpointInfo(tctx); err != nil {
if _, err := client.EndpointInfo(tctx); err != nil {
ok = false
bufferWeights[i] = 0
}
p.lock.RLock()
cp := *p.clientPacks[i]
p.lock.RUnlock()
if ok {
bufferWeights[i] = options.weights[i]
p.lock.RLock()
if !p.clientPacks[i].healthy {
if tkn, err = client.CreateSession(ctx, options.SessionExpirationEpoch); err != nil {
if !cp.healthy {
if tkn, err := client.CreateSession(ctx, options.SessionExpirationEpoch); err != nil {
ok = false
bufferWeights[i] = 0
} else {
_ = p.cache.Put(formCacheKey(cp.address, p.key), tkn)
}
}
p.lock.RUnlock()
} else {
p.cache.DeleteByPrefix(cp.address)
}
p.lock.Lock()
if p.clientPacks[i].healthy != ok {
p.clientPacks[i].healthy = ok
p.clientPacks[i].sessionToken = tkn
healthyChanged = true
}
p.lock.Unlock()
@ -234,23 +273,33 @@ func adjustWeights(weights []float64) []float64 {
}
func (p *pool) Connection() (client.Client, *session.Token, error) {
cp, err := p.connection()
if err != nil {
return nil, nil, err
}
token := p.cache.Get(formCacheKey(cp.address, p.key))
return cp.client, token, nil
}
func (p *pool) connection() (*clientPack, error) {
p.lock.RLock()
defer p.lock.RUnlock()
if len(p.clientPacks) == 1 {
cp := p.clientPacks[0]
if cp.healthy {
return cp.client, cp.sessionToken, nil
return cp, nil
}
return nil, nil, errors.New("no healthy client")
return nil, errors.New("no healthy client")
}
attempts := 3 * len(p.clientPacks)
for k := 0; k < attempts; k++ {
i := p.sampler.Next()
if cp := p.clientPacks[i]; cp.healthy {
return cp.client, cp.sessionToken, nil
return cp, nil
}
}
return nil, nil, errors.New("no healthy client")
return nil, errors.New("no healthy client")
}
func (p *pool) OwnerID() *owner.ID {
@ -265,6 +314,36 @@ func (p *pool) conn(option []client.CallOption) (client.Client, []client.CallOpt
return conn, append([]client.CallOption{client.WithSession(token)}, option...), nil
}
func formCacheKey(address string, key *ecdsa.PrivateKey) string {
k := keys.PrivateKey{PrivateKey: *key}
return address + k.String()
}
func (p *pool) connParam(ctx context.Context, param CallParam) (*clientPack, []client.CallOption, error) {
cp, err := p.connection()
if err != nil {
return nil, nil, err
}
key := p.key
if param.Key != nil {
key = param.Key
}
param.Options = append(param.Options, client.WithKey(key))
cacheKey := formCacheKey(cp.address, key)
token := p.cache.Get(cacheKey)
if token == nil {
token, err = cp.client.CreateSession(ctx, math.MaxUint32, param.Options...)
if err != nil {
return nil, nil, err
}
_ = p.cache.Put(cacheKey, token)
}
return cp, append([]client.CallOption{client.WithSession(token)}, param.Options...), nil
}
func (p *pool) PutObject(ctx context.Context, params *client.PutObjectParams, option ...client.CallOption) (*object.ID, error) {
conn, options, err := p.conn(option)
if err != nil {
@ -385,6 +464,166 @@ func (p *pool) AnnounceContainerUsedSpace(ctx context.Context, announce []contai
return conn.AnnounceContainerUsedSpace(ctx, announce, options...)
}
func (p *pool) checkSessionTokenErr(err error, address string) {
if err == nil {
return
}
if strings.Contains(err.Error(), "session token does not exist") {
p.cache.DeleteByPrefix(address)
}
}
func (p *pool) PutObjectParam(ctx context.Context, params *client.PutObjectParams, callParam CallParam) (*object.ID, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.PutObject(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) DeleteObjectParam(ctx context.Context, params *client.DeleteObjectParams, callParam CallParam) error {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return err
}
err = cp.client.DeleteObject(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return err
}
func (p *pool) GetObjectParam(ctx context.Context, params *client.GetObjectParams, callParam CallParam) (*object.Object, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.GetObject(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) GetObjectHeaderParam(ctx context.Context, params *client.ObjectHeaderParams, callParam CallParam) (*object.Object, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.GetObjectHeader(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) ObjectPayloadRangeDataParam(ctx context.Context, params *client.RangeDataParams, callParam CallParam) ([]byte, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.ObjectPayloadRangeData(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) ObjectPayloadRangeSHA256Param(ctx context.Context, params *client.RangeChecksumParams, callParam CallParam) ([][32]byte, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.ObjectPayloadRangeSHA256(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) ObjectPayloadRangeTZParam(ctx context.Context, params *client.RangeChecksumParams, callParam CallParam) ([][64]byte, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.ObjectPayloadRangeTZ(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) SearchObjectParam(ctx context.Context, params *client.SearchObjectParams, callParam CallParam) ([]*object.ID, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.SearchObject(ctx, params, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) PutContainerParam(ctx context.Context, cnr *container.Container, callParam CallParam) (*cid.ID, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.PutContainer(ctx, cnr, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) GetContainerParam(ctx context.Context, cid *cid.ID, callParam CallParam) (*container.Container, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.GetContainer(ctx, cid, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) ListContainersParam(ctx context.Context, ownerID *owner.ID, callParam CallParam) ([]*cid.ID, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.ListContainers(ctx, ownerID, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) DeleteContainerParam(ctx context.Context, cid *cid.ID, callParam CallParam) error {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return err
}
err = cp.client.DeleteContainer(ctx, cid, options...)
p.checkSessionTokenErr(err, cp.address)
return err
}
func (p *pool) GetEACLParam(ctx context.Context, cid *cid.ID, callParam CallParam) (*client.EACLWithSignature, error) {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return nil, err
}
res, err := cp.client.GetEACL(ctx, cid, options...)
p.checkSessionTokenErr(err, cp.address)
return res, err
}
func (p *pool) SetEACLParam(ctx context.Context, table *eacl.Table, callParam CallParam) error {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return err
}
err = cp.client.SetEACL(ctx, table, options...)
p.checkSessionTokenErr(err, cp.address)
return err
}
func (p *pool) AnnounceContainerUsedSpaceParam(ctx context.Context, announce []container.UsedSpaceAnnouncement, callParam CallParam) error {
cp, options, err := p.connParam(ctx, callParam)
if err != nil {
return err
}
err = cp.client.AnnounceContainerUsedSpace(ctx, announce, options...)
p.checkSessionTokenErr(err, cp.address)
return err
}
func (p *pool) WaitForContainerPresence(ctx context.Context, cid *cid.ID, pollParams *ContainerPollingParams) error {
conn, _, err := p.Connection()
if err != nil {

9
pool/pool_test.go
View file

@ -307,6 +307,10 @@ func TestTwoFailed(t *testing.T) {
require.Contains(t, err.Error(), "no healthy")
}
func TestSessionCache(t *testing.T) {
}
func newToken(t *testing.T) *session.Token {
tok := session.NewToken()
uid, err := uuid.New().MarshalBinary()
@ -324,12 +328,17 @@ func TestWaitPresence(t *testing.T) {
mockClient.EXPECT().EndpointInfo(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
mockClient.EXPECT().GetContainer(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
key, err := keys.NewPrivateKey()
require.NoError(t, err)
p := &pool{
sampler: NewSampler([]float64{1}, rand.NewSource(0)),
clientPacks: []*clientPack{{
client: mockClient,
healthy: true,
}},
key: &key.PrivateKey,
cache: NewCache(),
}
t.Run("context canceled", func(t *testing.T) {

12
pool/sampler_test.go
View file

@ -6,6 +6,7 @@ import (
"math/rand"
"testing"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/stretchr/testify/require"
@ -73,11 +74,16 @@ func TestHealthyReweight(t *testing.T) {
buffer = make([]float64, len(weights))
)
key, err := keys.NewPrivateKey()
require.NoError(t, err)
p := &pool{
sampler: NewSampler(weights, rand.NewSource(0)),
clientPacks: []*clientPack{
{client: newNetmapMock(names[0], true), healthy: true},
{client: newNetmapMock(names[1], false), healthy: true}},
{client: newNetmapMock(names[0], true), healthy: true, address: "address0"},
{client: newNetmapMock(names[1], false), healthy: true, address: "address1"}},
cache: NewCache(),
key: &key.PrivateKey,
}
// check getting first node connection before rebalance happened
@ -105,8 +111,6 @@ func TestHealthyReweight(t *testing.T) {
require.NoError(t, err)
mock0 = connection0.(clientMock)
require.Equal(t, names[0], mock0.name)
require.NotNil(t, p.clientPacks[0].sessionToken)
}
func TestHealthyNoReweight(t *testing.T) {