TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 28 Pull requests 6 Projects Releases 18 Packages Activity Actions

[#114] tree: Fix retry tests

Browse source 
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2023-05-19 17:08:59 +03:00
parent 136a186c14
commit 11f30a037b
2 changed files with 64 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
internal/frostfs/services/tree_client_grpc.go
View file

@ -8,14 +8,13 @@ import (
"strings"
"sync/atomic"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
treeClient "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/client"
grpcService "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
"google.golang.org/grpc"
@ -69,10 +68,15 @@ func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta {
return res
}
type TreeClient interface {
TreeClient(ctx context.Context) (grpcService.TreeServiceClient, error)
Address() string
}
type ServiceClientGRPC struct {
key *keys.PrivateKey
log *zap.Logger
clients []*treeClient.TreeClient
clients []TreeClient
startIndex int32
}
@ -100,7 +104,7 @@ func NewTreeServiceClientGRPC(ctx context.Context, endpoints []string, key *keys
firstHealthy := -1
res.clients = make([]*treeClient.TreeClient, len(endpoints))
res.clients = make([]TreeClient, len(endpoints))
for i, addr := range endpoints {
res.clients[i] = treeClient.NewTreeClient(addr, grpcOpts...)
if _, err := res.clients[i].TreeClient(ctx); err != nil {

84
internal/frostfs/services/tree_client_grpc_test.go
View file

@ -3,17 +3,30 @@ package services
import (
"context"
"errors"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/client"
grpcService "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/tree"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
type treeClientMock struct {
address string
err bool
}
func (t *treeClientMock) TreeClient(context.Context) (grpcService.TreeServiceClient, error) {
if t.err {
return nil, errors.New("error")
}
return nil, nil
}
func (t *treeClientMock) Address() string {
return t.address
}
func TestHandleError(t *testing.T) {
defaultError := errors.New("default error")
for _, tc := range []struct {
@ -44,58 +57,73 @@ func TestRetry(t *testing.T) {
ctx := context.Background()
log := zaptest.NewLogger(t)
grpcDialOpt := grpc.WithTransportCredentials(insecure.NewCredentials())
cl := &ServiceClientGRPC{
log: zaptest.NewLogger(t),
clients: []*client.TreeClient{
client.NewTreeClient("node0", grpcDialOpt),
client.NewTreeClient("node1", grpcDialOpt),
client.NewTreeClient("node2", grpcDialOpt),
client.NewTreeClient("node3", grpcDialOpt),
clients: []TreeClient{
&treeClientMock{address: "node0"},
&treeClientMock{address: "node1"},
&treeClientMock{address: "node2"},
&treeClientMock{address: "node3"},
},
}
makeFn := func(shouldFail []string) func(grpcService.TreeServiceClient) error {
return func(client grpcService.TreeServiceClient) error {
//for _, item := range shouldFail {
//if item == client.address {
// return errors.New("not found")
//}
//}
return nil
}
makeFn := func(client grpcService.TreeServiceClient) error {
return nil
}
t.Run("first ok", func(t *testing.T) {
err := cl.requestWithRetry(ctx, log, makeFn([]string{}))
err := cl.requestWithRetry(ctx, log, makeFn)
require.NoError(t, err)
require.Equal(t, 0, cl.getStartIndex())
cl.setStartIndex(0)
resetClients(cl)
})
t.Run("first failed", func(t *testing.T) {
err := cl.requestWithRetry(ctx, log, makeFn([]string{"node0"}))
setErrors(cl.clients[:1])
err := cl.requestWithRetry(ctx, log, makeFn)
require.NoError(t, err)
require.Equal(t, 1, cl.getStartIndex())
cl.setStartIndex(0)
resetClients(cl)
})
t.Run("all failed", func(t *testing.T) {
err := cl.requestWithRetry(ctx, log, makeFn([]string{"node0", "node1", "node2", "node3"}))
setErrors(cl.clients)
err := cl.requestWithRetry(ctx, log, makeFn)
require.Error(t, err)
require.Equal(t, 0, cl.getStartIndex())
cl.setStartIndex(0)
resetClients(cl)
})
t.Run("round", func(t *testing.T) {
err := cl.requestWithRetry(ctx, log, makeFn([]string{"node0", "node1"}))
setErrors(cl.clients[:2])
err := cl.requestWithRetry(ctx, log, makeFn)
require.NoError(t, err)
require.Equal(t, 2, cl.getStartIndex())
resetClientsErrors(cl)
err = cl.requestWithRetry(ctx, log, makeFn([]string{"node2", "node3"}))
setErrors(cl.clients[2:])
err = cl.requestWithRetry(ctx, log, makeFn)
require.NoError(t, err)
require.Equal(t, 0, cl.getStartIndex())
cl.setStartIndex(0)
resetClients(cl)
})
}
func resetClients(cl *ServiceClientGRPC) {
resetClientsErrors(cl)
cl.setStartIndex(0)
}
func resetClientsErrors(cl *ServiceClientGRPC) {
for _, client := range cl.clients {
node := client.(*treeClientMock)
node.err = false
}
}
func setErrors(clients []TreeClient) {
for _, client := range clients {
node := client.(*treeClientMock)
node.err = true
}
}