frostfs-sdk-go/pool/pool_test.go

package pool

import (
	"context"
	"crypto/ecdsa"
	"errors"
	"math/rand"
	"testing"
	"time"

	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
	frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
	"go.uber.org/zap/zaptest"
	"go.uber.org/zap/zaptest/observer"
)

func TestBuildPoolClientFailed(t *testing.T) {
	mockClientBuilder := func(addr string) client {
		mockCli := newMockClient(addr, *newPrivateKey(t))
		mockCli.errOnDial()
		return mockCli
	}

	opts := InitParameters{
		key:        newPrivateKey(t),
		nodeParams: []NodeParam{{1, "peer0", 1}},
	}
	opts.setClientBuilder(mockClientBuilder)

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(context.Background())
	require.Error(t, err)
}

func TestBuildPoolCreateSessionFailed(t *testing.T) {
	clientMockBuilder := func(addr string) client {
		mockCli := newMockClient(addr, *newPrivateKey(t))
		mockCli.errOnCreateSession()
		return mockCli
	}

	opts := InitParameters{
		key:        newPrivateKey(t),
		nodeParams: []NodeParam{{1, "peer0", 1}},
	}
	opts.setClientBuilder(clientMockBuilder)

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(context.Background())
	require.Error(t, err)
}

func newPrivateKey(t *testing.T) *ecdsa.PrivateKey {
	p, err := keys.NewPrivateKey()
	require.NoError(t, err)
	return &p.PrivateKey
}

func TestBuildPoolOneNodeFailed(t *testing.T) {
	nodes := []NodeParam{
		{1, "peer0", 1},
		{2, "peer1", 1},
	}

	var clientKeys []*ecdsa.PrivateKey
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		clientKeys = append(clientKeys, key)

		if addr == nodes[0].address {
			mockCli := newMockClient(addr, *key)
			mockCli.errOnEndpointInfo()
			return mockCli
		}

		return newMockClient(addr, *key)
	}

	log, err := zap.NewProduction()
	require.NoError(t, err)
	opts := InitParameters{
		key:                     newPrivateKey(t),
		clientRebalanceInterval: 1000 * time.Millisecond,
		logger:                  log,
		nodeParams:              nodes,
	}
	opts.setClientBuilder(mockClientBuilder)

	clientPool, err := NewPool(opts)
	require.NoError(t, err)
	err = clientPool.Dial(context.Background())
	require.NoError(t, err)
	t.Cleanup(clientPool.Close)

	expectedAuthKey := frostfsecdsa.PublicKey(clientKeys[1].PublicKey)
	condition := func() bool {
		cp, err := clientPool.connection()
		if err != nil {
			return false
		}
		st, _ := clientPool.cache.Get(formCacheKey(cp.address(), clientPool.key, false))
		return st.AssertAuthKey(&expectedAuthKey)
	}
	require.Never(t, condition, 900*time.Millisecond, 100*time.Millisecond)
	require.Eventually(t, condition, 3*time.Second, 300*time.Millisecond)
}

func TestBuildPoolZeroNodes(t *testing.T) {
	opts := InitParameters{
		key: newPrivateKey(t),
	}
	_, err := NewPool(opts)
	require.Error(t, err)
}

func TestOneNode(t *testing.T) {
	key1 := newPrivateKey(t)
	mockClientBuilder := func(addr string) client {
		return newMockClient(addr, *key1)
	}

	opts := InitParameters{
		key:        newPrivateKey(t),
		nodeParams: []NodeParam{{1, "peer0", 1}},
	}
	opts.setClientBuilder(mockClientBuilder)

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(context.Background())
	require.NoError(t, err)
	t.Cleanup(pool.Close)

	cp, err := pool.connection()
	require.NoError(t, err)
	st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
	expectedAuthKey := frostfsecdsa.PublicKey(key1.PublicKey)
	require.True(t, st.AssertAuthKey(&expectedAuthKey))
}

func TestTwoNodes(t *testing.T) {
	var clientKeys []*ecdsa.PrivateKey
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		clientKeys = append(clientKeys, key)
		return newMockClient(addr, *key)
	}

	opts := InitParameters{
		key: newPrivateKey(t),
		nodeParams: []NodeParam{
			{1, "peer0", 1},
			{1, "peer1", 1},
		},
	}
	opts.setClientBuilder(mockClientBuilder)

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(context.Background())
	require.NoError(t, err)
	t.Cleanup(pool.Close)

	cp, err := pool.connection()
	require.NoError(t, err)
	st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
	require.True(t, assertAuthKeyForAny(st, clientKeys))
}

func assertAuthKeyForAny(st session.Object, clientKeys []*ecdsa.PrivateKey) bool {
	for _, key := range clientKeys {
		expectedAuthKey := frostfsecdsa.PublicKey(key.PublicKey)
		if st.AssertAuthKey(&expectedAuthKey) {
			return true
		}
	}
	return false
}

func TestOneOfTwoFailed(t *testing.T) {
	nodes := []NodeParam{
		{1, "peer0", 1},
		{9, "peer1", 1},
	}

	var clientKeys []*ecdsa.PrivateKey
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		clientKeys = append(clientKeys, key)

		if addr == nodes[0].address {
			return newMockClient(addr, *key)
		}

		mockCli := newMockClient(addr, *key)
		mockCli.errOnEndpointInfo()
		mockCli.errOnNetworkInfo()
		return mockCli
	}

	opts := InitParameters{
		key:                     newPrivateKey(t),
		nodeParams:              nodes,
		clientRebalanceInterval: 200 * time.Millisecond,
	}
	opts.setClientBuilder(mockClientBuilder)

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(context.Background())
	require.NoError(t, err)

	require.NoError(t, err)
	t.Cleanup(pool.Close)

	time.Sleep(2 * time.Second)

	for range 5 {
		cp, err := pool.connection()
		require.NoError(t, err)
		st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
		require.True(t, assertAuthKeyForAny(st, clientKeys))
	}
}

func TestUpdateNodesHealth(t *testing.T) {
	ctx := context.Background()
	key := newPrivateKey(t)

	for _, tc := range []struct {
		name        string
		wasHealthy  bool
		willHealthy bool
		prepareCli  func(*mockClient)
	}{
		{
			name:        "healthy, maintenance, unhealthy",
			wasHealthy:  true,
			willHealthy: false,
			prepareCli:  func(c *mockClient) { c.resOnEndpointInfo.SetStatus(netmap.Maintenance) },
		},
		{
			name:        "unhealthy, maintenance, unhealthy",
			wasHealthy:  false,
			willHealthy: false,
			prepareCli:  func(c *mockClient) { c.resOnEndpointInfo.SetStatus(netmap.Maintenance) },
		},
		{
			name:        "healthy, no error, healthy",
			wasHealthy:  true,
			willHealthy: true,
			prepareCli:  func(c *mockClient) { c.resOnEndpointInfo.SetStatus(netmap.Online) },
		},
		{
			name:        "unhealthy, no error, healthy",
			wasHealthy:  false,
			willHealthy: true,
			prepareCli:  func(c *mockClient) { c.resOnEndpointInfo.SetStatus(netmap.Online) },
		},
		{
			name:        "healthy, error, failed restart, unhealthy",
			wasHealthy:  true,
			willHealthy: false,
			prepareCli: func(c *mockClient) {
				c.errOnEndpointInfo()
				c.errorOnDial = true
			},
		},
		{
			name:        "unhealthy, error, failed restart, unhealthy",
			wasHealthy:  false,
			willHealthy: false,
			prepareCli: func(c *mockClient) {
				c.errOnEndpointInfo()
				c.errorOnDial = true
			},
		},
		{
			name:        "healthy, error, restart, error, unhealthy",
			wasHealthy:  true,
			willHealthy: false,
			prepareCli:  func(c *mockClient) { c.errOnEndpointInfo() },
		},
		{
			name:        "unhealthy, error, restart, error, unhealthy",
			wasHealthy:  false,
			willHealthy: false,
			prepareCli:  func(c *mockClient) { c.errOnEndpointInfo() },
		},
		{
			name:        "healthy, error, restart, maintenance, unhealthy",
			wasHealthy:  true,
			willHealthy: false,
			prepareCli: func(c *mockClient) {
				healthError := true
				c.healthcheckFn = func() {
					if healthError {
						c.errorOnEndpointInfo = errors.New("error")
					} else {
						c.errorOnEndpointInfo = nil
						c.resOnEndpointInfo.SetStatus(netmap.Maintenance)
					}
					healthError = !healthError
				}
			},
		},
		{
			name:        "unhealthy, error, restart, maintenance, unhealthy",
			wasHealthy:  false,
			willHealthy: false,
			prepareCli: func(c *mockClient) {
				healthError := true
				c.healthcheckFn = func() {
					if healthError {
						c.errorOnEndpointInfo = errors.New("error")
					} else {
						c.errorOnEndpointInfo = nil
						c.resOnEndpointInfo.SetStatus(netmap.Maintenance)
					}
					healthError = !healthError
				}
			},
		},
		{
			name:        "healthy, error, restart, healthy",
			wasHealthy:  true,
			willHealthy: true,
			prepareCli: func(c *mockClient) {
				healthError := true
				c.healthcheckFn = func() {
					if healthError {
						c.errorOnEndpointInfo = errors.New("error")
					} else {
						c.errorOnEndpointInfo = nil
					}
					healthError = !healthError
				}
			},
		},
		{
			name:        "unhealthy, error, restart, healthy",
			wasHealthy:  false,
			willHealthy: true,
			prepareCli: func(c *mockClient) {
				healthError := true
				c.healthcheckFn = func() {
					if healthError {
						c.errorOnEndpointInfo = errors.New("error")
					} else {
						c.errorOnEndpointInfo = nil
					}
					healthError = !healthError
				}
			},
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			cli := newMockClientHealthy("peer0", *key, tc.wasHealthy)
			tc.prepareCli(cli)
			p, log := newPool(t, cli)

			p.updateNodesHealth(ctx, [][]float64{{1}})

			changed := tc.wasHealthy != tc.willHealthy
			require.Equalf(t, tc.willHealthy, cli.isHealthy(), "healthy status should be: %v", tc.willHealthy)
			require.Equalf(t, changed, 1 == log.Len(), "healthy status should be changed: %v", changed)
		})
	}
}

func newPool(t *testing.T, cli *mockClient) (*Pool, *observer.ObservedLogs) {
	log, observedLog := getObservedLogger()

	cache, err := newCache(0)
	require.NoError(t, err)

	return &Pool{
		innerPools: []*innerPool{{
			sampler: newSampler([]float64{1}, rand.NewSource(0)),
			clients: []client{cli},
		}},
		cache:    cache,
		key:      newPrivateKey(t),
		closedCh: make(chan struct{}),
		rebalanceParams: rebalanceParameters{
			nodesParams:             []*nodesParam{{1, []string{"peer0"}, []float64{1}}},
			nodeRequestTimeout:      time.Second,
			clientRebalanceInterval: 200 * time.Millisecond,
		},
		logger: log,
	}, observedLog
}

func getObservedLogger() (*zap.Logger, *observer.ObservedLogs) {
	loggerCore, observedLog := observer.New(zap.DebugLevel)
	return zap.New(loggerCore), observedLog
}

func TestTwoFailed(t *testing.T) {
	var clientKeys []*ecdsa.PrivateKey
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		clientKeys = append(clientKeys, key)
		mockCli := newMockClient(addr, *key)
		mockCli.errOnEndpointInfo()
		return mockCli
	}

	opts := InitParameters{
		key: newPrivateKey(t),
		nodeParams: []NodeParam{
			{1, "peer0", 1},
			{1, "peer1", 1},
		},
		clientRebalanceInterval: 200 * time.Millisecond,
	}
	opts.setClientBuilder(mockClientBuilder)

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(context.Background())
	require.NoError(t, err)

	t.Cleanup(pool.Close)

	time.Sleep(2 * time.Second)

	_, err = pool.connection()
	require.Error(t, err)
	require.Contains(t, err.Error(), "no healthy")
}

func TestSessionCache(t *testing.T) {
	key := newPrivateKey(t)
	expectedAuthKey := frostfsecdsa.PublicKey(key.PublicKey)

	mockClientBuilder := func(addr string) client {
		mockCli := newMockClient(addr, *key)
		mockCli.statusOnGetObject(new(apistatus.SessionTokenNotFound))
		return mockCli
	}

	opts := InitParameters{
		key: newPrivateKey(t),
		nodeParams: []NodeParam{
			{1, "peer0", 1},
		},
		clientRebalanceInterval: 30 * time.Second,
	}
	opts.setClientBuilder(mockClientBuilder)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(ctx)
	require.NoError(t, err)
	t.Cleanup(pool.Close)

	// cache must contain session token
	cp, err := pool.connection()
	require.NoError(t, err)
	st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
	require.True(t, st.AssertAuthKey(&expectedAuthKey))

	var prm PrmObjectGet
	prm.SetAddress(oid.Address{})
	prm.UseSession(session.Object{})

	_, err = pool.GetObject(ctx, prm)
	require.Error(t, err)

	// cache must not contain session token
	cp, err = pool.connection()
	require.NoError(t, err)
	_, ok := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
	require.False(t, ok)

	var prm2 PrmObjectPut
	prm2.SetHeader(object.Object{})

	_, err = pool.PutObject(ctx, prm2)
	require.NoError(t, err)

	// cache must contain session token
	cp, err = pool.connection()
	require.NoError(t, err)
	st, _ = pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
	require.True(t, st.AssertAuthKey(&expectedAuthKey))
}

func TestPriority(t *testing.T) {
	nodes := []NodeParam{
		{1, "peer0", 1},
		{2, "peer1", 100},
	}

	var clientKeys []*ecdsa.PrivateKey
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		clientKeys = append(clientKeys, key)

		if addr == nodes[0].address {
			mockCli := newMockClient(addr, *key)
			mockCli.errOnEndpointInfo()
			return mockCli
		}

		return newMockClient(addr, *key)
	}

	opts := InitParameters{
		key:                     newPrivateKey(t),
		nodeParams:              nodes,
		clientRebalanceInterval: 1500 * time.Millisecond,
	}
	opts.setClientBuilder(mockClientBuilder)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(ctx)
	require.NoError(t, err)
	t.Cleanup(pool.Close)

	expectedAuthKey1 := frostfsecdsa.PublicKey(clientKeys[0].PublicKey)
	firstNode := func() bool {
		cp, err := pool.connection()
		require.NoError(t, err)
		st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
		return st.AssertAuthKey(&expectedAuthKey1)
	}

	expectedAuthKey2 := frostfsecdsa.PublicKey(clientKeys[1].PublicKey)
	secondNode := func() bool {
		cp, err := pool.connection()
		require.NoError(t, err)
		st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
		return st.AssertAuthKey(&expectedAuthKey2)
	}
	require.Never(t, secondNode, time.Second, 200*time.Millisecond)

	require.Eventually(t, secondNode, time.Second, 200*time.Millisecond)
	require.Never(t, firstNode, time.Second, 200*time.Millisecond)
}

func TestSessionCacheWithKey(t *testing.T) {
	key := newPrivateKey(t)
	expectedAuthKey := frostfsecdsa.PublicKey(key.PublicKey)

	mockClientBuilder := func(addr string) client {
		return newMockClient(addr, *key)
	}

	opts := InitParameters{
		key: newPrivateKey(t),
		nodeParams: []NodeParam{
			{1, "peer0", 1},
		},
		clientRebalanceInterval: 30 * time.Second,
	}
	opts.setClientBuilder(mockClientBuilder)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(ctx)
	require.NoError(t, err)

	// cache must contain session token
	cp, err := pool.connection()
	require.NoError(t, err)
	st, _ := pool.cache.Get(formCacheKey(cp.address(), pool.key, false))
	require.True(t, st.AssertAuthKey(&expectedAuthKey))

	var prm PrmObjectDelete
	prm.SetAddress(oid.Address{})
	anonKey := newPrivateKey(t)
	prm.UseKey(anonKey)

	err = pool.DeleteObject(ctx, prm)
	require.NoError(t, err)
	st, _ = pool.cache.Get(formCacheKey(cp.address(), anonKey, false))
	require.True(t, st.AssertAuthKey(&expectedAuthKey))
}

func TestSessionTokenOwner(t *testing.T) {
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		return newMockClient(addr, *key)
	}

	opts := InitParameters{
		key: newPrivateKey(t),
		nodeParams: []NodeParam{
			{1, "peer0", 1},
		},
	}
	opts.setClientBuilder(mockClientBuilder)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	p, err := NewPool(opts)
	require.NoError(t, err)
	err = p.Dial(ctx)
	require.NoError(t, err)
	t.Cleanup(p.Close)

	anonKey := newPrivateKey(t)
	var anonOwner user.ID
	user.IDFromKey(&anonOwner, anonKey.PublicKey)

	var prm prmCommon
	prm.UseKey(anonKey)
	var prmCtx prmContext
	prmCtx.useDefaultSession()

	var tkn session.Object
	var cc callContext
	cc.sessionTarget = func(tok session.Object) {
		tkn = tok
	}
	err = p.initCallContext(&cc, prm, prmCtx)
	require.NoError(t, err)

	err = p.openDefaultSession(ctx, &cc)
	require.NoError(t, err)
	require.True(t, tkn.VerifySignature())
	require.True(t, tkn.Issuer().Equals(anonOwner))
}

func TestWaitPresence(t *testing.T) {
	mockCli := newMockClient("", *newPrivateKey(t))

	t.Run("context canceled", func(t *testing.T) {
		ctx, cancel := context.WithCancel(context.Background())
		go func() {
			time.Sleep(500 * time.Millisecond)
			cancel()
		}()

		err := waitForContainerPresence(ctx, mockCli, PrmContainerGet{}, &WaitParams{
			Timeout:      120 * time.Second,
			PollInterval: 5 * time.Second,
		})
		require.Error(t, err)
		require.Contains(t, err.Error(), "context canceled")
	})

	t.Run("context deadline exceeded", func(t *testing.T) {
		ctx := context.Background()

		err := waitForContainerPresence(ctx, mockCli, PrmContainerGet{}, &WaitParams{
			Timeout:      500 * time.Millisecond,
			PollInterval: 5 * time.Second,
		})
		require.Error(t, err)
		require.Contains(t, err.Error(), "context deadline exceeded")
	})

	t.Run("ok", func(t *testing.T) {
		ctx := context.Background()

		err := waitForContainerPresence(ctx, mockCli, PrmContainerGet{}, &WaitParams{
			Timeout:      10 * time.Second,
			PollInterval: 500 * time.Millisecond,
		})
		require.NoError(t, err)
	})
}

func TestStatusMonitor(t *testing.T) {
	monitor := newClientStatusMonitor(zap.NewExample(), "", 10)
	monitor.errorThreshold = 3

	count := 10
	for range count {
		monitor.incErrorRate()
	}

	require.Equal(t, uint64(count), monitor.overallErrorRate())
	require.Equal(t, uint32(1), monitor.currentErrorRate())

	t.Run("healthy status", func(t *testing.T) {
		cases := []struct {
			action      func(*clientStatusMonitor)
			status      uint32
			isDialed    bool
			isHealthy   bool
			description string
		}{
			{
				action:      func(m *clientStatusMonitor) { m.setUnhealthy() },
				status:      statusUnhealthyOnRequest,
				isDialed:    true,
				isHealthy:   false,
				description: "set unhealthy on request",
			},
			{
				action:      func(m *clientStatusMonitor) { m.setHealthy() },
				status:      statusHealthy,
				isDialed:    true,
				isHealthy:   true,
				description: "set healthy",
			},
		}
		for _, tc := range cases {
			tc.action(&monitor)
			require.Equal(t, tc.status, monitor.healthy.Load())
			require.Equal(t, tc.isHealthy, monitor.isHealthy())
		}
	})
}

func TestHandleError(t *testing.T) {
	ctx := context.Background()
	log := zaptest.NewLogger(t)

	canceledCtx, cancel := context.WithCancel(context.Background())
	cancel()

	for _, tc := range []struct {
		name            string
		ctx             context.Context
		status          apistatus.Status
		err             error
		expectedError   bool
		countError      bool
		markedUnhealthy bool
	}{
		{
			name:          "no error, no status",
			ctx:           ctx,
			status:        nil,
			err:           nil,
			expectedError: false,
			countError:    false,
		},
		{
			name:          "no error, success status",
			ctx:           ctx,
			status:        new(apistatus.SuccessDefaultV2),
			err:           nil,
			expectedError: false,
			countError:    false,
		},
		{
			name:          "error, success status",
			ctx:           ctx,
			status:        new(apistatus.SuccessDefaultV2),
			err:           errors.New("error"),
			expectedError: true,
			countError:    true,
		},
		{
			name:          "error, no status",
			ctx:           ctx,
			status:        nil,
			err:           errors.New("error"),
			expectedError: true,
			countError:    true,
		},
		{
			name:          "no error, object not found status",
			ctx:           ctx,
			status:        new(apistatus.ObjectNotFound),
			err:           nil,
			expectedError: true,
			countError:    false,
		},
		{
			name:          "object not found error, object not found status",
			ctx:           ctx,
			status:        new(apistatus.ObjectNotFound),
			err:           &apistatus.ObjectNotFound{},
			expectedError: true,
			countError:    false,
		},
		{
			name:          "eacl not found error, no status",
			ctx:           ctx,
			status:        nil,
			err:           &apistatus.EACLNotFound{},
			expectedError: true,
			// we expect error be counted because status is nil
			// currently we assume that DisableFrostFSErrorResolution be always false for pool
			// and status be checked first in handleError
			countError: true,
		},
		{
			name:          "no error, internal status",
			ctx:           ctx,
			status:        new(apistatus.ServerInternal),
			err:           nil,
			expectedError: true,
			countError:    true,
		},
		{
			name:          "no error, wrong magic status",
			ctx:           ctx,
			status:        new(apistatus.WrongMagicNumber),
			err:           nil,
			expectedError: true,
			countError:    true,
		},
		{
			name:          "no error, signature verification status",
			ctx:           ctx,
			status:        new(apistatus.SignatureVerification),
			err:           nil,
			expectedError: true,
			countError:    true,
		},
		{
			name:            "no error, maintenance status",
			ctx:             ctx,
			status:          new(apistatus.NodeUnderMaintenance),
			err:             nil,
			expectedError:   true,
			countError:      true,
			markedUnhealthy: true,
		},
		{
			name:            "maintenance error, no status",
			ctx:             ctx,
			status:          nil,
			err:             &apistatus.NodeUnderMaintenance{},
			expectedError:   true,
			countError:      true,
			markedUnhealthy: true,
		},
		{
			name:          "no error, invalid argument status",
			ctx:           ctx,
			status:        new(apistatus.InvalidArgument),
			err:           nil,
			expectedError: true,
			countError:    false,
		},
		{
			name:          "context canceled error, no status",
			ctx:           canceledCtx,
			status:        nil,
			err:           errors.New("error"),
			expectedError: true,
			countError:    false,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			monitor := newClientStatusMonitor(log, "", 10)
			errCount := monitor.overallErrorRate()
			err := monitor.handleError(tc.ctx, tc.status, tc.err)
			if tc.expectedError {
				require.Error(t, err)
			} else {
				require.NoError(t, err)
			}
			if tc.countError {
				errCount++
			}
			require.Equal(t, errCount, monitor.overallErrorRate())
			if tc.markedUnhealthy {
				require.False(t, monitor.isHealthy())
			}
		})
	}
}

func TestSwitchAfterErrorThreshold(t *testing.T) {
	nodes := []NodeParam{
		{1, "peer0", 1},
		{2, "peer1", 100},
	}

	errorThreshold := 5

	var clientKeys []*ecdsa.PrivateKey
	mockClientBuilder := func(addr string) client {
		key := newPrivateKey(t)
		clientKeys = append(clientKeys, key)

		if addr == nodes[0].address {
			mockCli := newMockClient(addr, *key)
			mockCli.setThreshold(uint32(errorThreshold))
			mockCli.statusOnGetObject(new(apistatus.ServerInternal))
			return mockCli
		}

		return newMockClient(addr, *key)
	}

	opts := InitParameters{
		key:                     newPrivateKey(t),
		nodeParams:              nodes,
		clientRebalanceInterval: 30 * time.Second,
	}
	opts.setClientBuilder(mockClientBuilder)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	pool, err := NewPool(opts)
	require.NoError(t, err)
	err = pool.Dial(ctx)
	require.NoError(t, err)
	t.Cleanup(pool.Close)

	for range errorThreshold {
		conn, err := pool.connection()
		require.NoError(t, err)
		require.Equal(t, nodes[0].address, conn.address())
		_, err = conn.objectGet(ctx, PrmObjectGet{})
		require.Error(t, err)
	}

	conn, err := pool.connection()
	require.NoError(t, err)
	require.Equal(t, nodes[1].address, conn.address())
	_, err = conn.objectGet(ctx, PrmObjectGet{})
	require.NoError(t, err)
}