[#26] All: Remove V2 from naming

Rename project, namespaces and class names

Signed-off-by: Pavel Gross <p.gross@yadro.com>

src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs

@@ -0,0 +1,163 @@
+using System;
+using System.Threading;
+
+using Microsoft.Extensions.Logging;
+
+namespace FrostFS.SDK.Client;
+
+// clientStatusMonitor count error rate and other statistics for connection.
+public class ClientStatusMonitor : IClientStatus
+{
+    private static readonly MethodIndex[] MethodIndexes =
+    [
+        MethodIndex.methodBalanceGet,
+        MethodIndex.methodContainerPut,
+        MethodIndex.methodContainerGet,
+        MethodIndex.methodContainerList,
+        MethodIndex.methodContainerDelete,
+        MethodIndex.methodEndpointInfo,
+        MethodIndex.methodNetworkInfo,
+        MethodIndex.methodNetMapSnapshot,
+        MethodIndex.methodObjectPut,
+        MethodIndex.methodObjectDelete,
+        MethodIndex.methodObjectGet,
+        MethodIndex.methodObjectHead,
+        MethodIndex.methodObjectRange,
+        MethodIndex.methodObjectPatch,
+        MethodIndex.methodSessionCreate,
+        MethodIndex.methodAPEManagerAddChain,
+        MethodIndex.methodAPEManagerRemoveChain,
+        MethodIndex.methodAPEManagerListChains
+    ];
+
+    public static string GetMethodName(MethodIndex index)
+    {
+        return index switch
+        {
+            MethodIndex.methodBalanceGet => "BalanceGet",
+            MethodIndex.methodContainerPut => "ContainerPut",
+            MethodIndex.methodContainerGet => "ContainerGet",
+            MethodIndex.methodContainerList => "ContainerList",
+            MethodIndex.methodContainerDelete => "ContainerDelete",
+            MethodIndex.methodEndpointInfo => "EndpointInfo",
+            MethodIndex.methodNetworkInfo => "NetworkInfo",
+            MethodIndex.methodNetMapSnapshot => "NetMapSnapshot",
+            MethodIndex.methodObjectPut => "ObjectPut",
+            MethodIndex.methodObjectDelete => "ObjectDelete",
+            MethodIndex.methodObjectGet => "ObjectGet",
+            MethodIndex.methodObjectHead => "ObjectHead",
+            MethodIndex.methodObjectRange => "ObjectRange",
+            MethodIndex.methodObjectPatch => "ObjectPatch",
+            MethodIndex.methodSessionCreate => "SessionCreate",
+            MethodIndex.methodAPEManagerAddChain => "APEManagerAddChain",
+            MethodIndex.methodAPEManagerRemoveChain => "APEManagerRemoveChain",
+            MethodIndex.methodAPEManagerListChains => "APEManagerListChains",
+            _ => throw new ArgumentException("Unknown method", nameof(index)),
+        };
+    }
+
+    private readonly object _lock = new();
+
+    private readonly ILogger? logger;
+    private int healthy;
+
+    public ClientStatusMonitor(ILogger? logger, string address)
+    {
+        this.logger = logger;
+        healthy = (int)HealthyStatus.Healthy;
+
+        Address = address;
+        Methods = new MethodStatus[MethodIndexes.Length];
+
+        for (int i = 0; i < MethodIndexes.Length; i++)
+        {
+            Methods[i] = new MethodStatus(GetMethodName(MethodIndexes[i]));
+        }
+    }
+
+    public string Address { get; }
+
+    internal uint ErrorThreshold { get; set; }
+
+    public uint CurrentErrorCount { get; set; }
+
+    public ulong OverallErrorCount { get; set; }
+
+    public MethodStatus[] Methods { get; private set; }
+
+    public bool IsHealthy()
+    {
+        var res = Interlocked.CompareExchange(ref healthy, -1, -1) == (int)HealthyStatus.Healthy;
+        return res;
+    }
+
+    public bool IsDialed()
+    {
+        return Interlocked.CompareExchange(ref healthy, -1, -1) != (int)HealthyStatus.UnhealthyOnDial;
+    }
+
+    public void SetHealthy()
+    {
+        Interlocked.Exchange(ref healthy, (int)HealthyStatus.Healthy);
+    }
+    public void SetUnhealthy()
+    {
+        Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnRequest);
+    }
+
+    public void SetUnhealthyOnDial()
+    {
+        Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnDial);
+    }
+
+    public void IncErrorRate()
+    {
+        bool thresholdReached;
+        lock (_lock)
+        {
+            CurrentErrorCount++;
+            OverallErrorCount++;
+
+            thresholdReached = CurrentErrorCount >= ErrorThreshold;
+
+            if (thresholdReached)
+            {
+                SetUnhealthy();
+                CurrentErrorCount = 0;
+            }
+        }
+
+        if (thresholdReached && logger != null)
+        {
+            FrostFsMessages.ErrorЕhresholdReached(logger, Address, ErrorThreshold);
+        }
+    }
+
+    public uint GetCurrentErrorRate()
+    {
+        lock (_lock)
+        {
+            return CurrentErrorCount;
+        }
+    }
+
+    public ulong GetOverallErrorRate()
+    {
+        lock (_lock)
+        {
+            return OverallErrorCount;
+        }
+    }
+
+    public StatusSnapshot[] MethodsStatus()
+    {
+        var result = new StatusSnapshot[Methods.Length];
+
+        for (int i = 0; i < result.Length; i++)
+        {
+            result[i] = Methods[i].Snapshot!;
+        }
+
+        return result;
+    }
+}

src/FrostFS.SDK.Client/Pool/ClientWrapper.cs

@@ -0,0 +1,157 @@
+using System;
+using System.Threading.Tasks;
+
+using Grpc.Core;
+
+namespace FrostFS.SDK.Client;
+
+// clientWrapper is used by default, alternative implementations are intended for testing purposes only.
+public class ClientWrapper : ClientStatusMonitor
+{
+    private readonly object _lock = new();
+
+    private SessionCache sessionCache;
+
+
+    internal ClientWrapper(WrapperPrm wrapperPrm, Pool pool) : base(wrapperPrm.Logger, wrapperPrm.Address)
+    {
+        WrapperPrm = wrapperPrm;
+        ErrorThreshold = wrapperPrm.ErrorThreshold;
+
+        sessionCache = pool.SessionCache;
+        Client = new FrostFSClient(WrapperPrm, sessionCache);
+    }
+
+    internal FrostFSClient? Client { get; private set; }
+
+    internal WrapperPrm WrapperPrm { get; }
+
+    internal FrostFSClient? GetClient()
+    {
+        lock (_lock)
+        {
+            if (IsHealthy())
+            {
+                return Client;
+            }
+
+            return null;
+        }
+    }
+
+    // dial establishes a connection to the server from the FrostFS network.
+    // Returns an error describing failure reason. If failed, the client
+    // SHOULD NOT be used.
+    internal async Task Dial(CallContext ctx)
+    {
+        var client = GetClient() ?? throw new FrostFsInvalidObjectException("pool client unhealthy");
+
+        await client.Dial(ctx).ConfigureAwait(false);
+    }
+
+    internal void HandleError(Exception ex)
+    {
+        if (ex is FrostFsResponseException responseException && responseException.Status != null)
+        {
+            switch (responseException.Status.Code)
+            {
+                case FrostFsStatusCode.Internal:
+                case FrostFsStatusCode.WrongMagicNumber:
+                case FrostFsStatusCode.SignatureVerificationFailure:
+                case FrostFsStatusCode.NodeUnderMaintenance:
+                    IncErrorRate();
+                    return;
+            }
+        }
+
+        IncErrorRate();
+    }
+
+    private async Task ScheduleGracefulClose()
+    {
+        if (Client == null)
+            return;
+
+        await Task.Delay((int)WrapperPrm.GracefulCloseOnSwitchTimeout).ConfigureAwait(false);
+
+        Client.Close();
+    }
+
+    // restartIfUnhealthy checks healthy status of client and recreate it if status is unhealthy.
+    // Indicating if status was changed by this function call and returns error that caused unhealthy status.
+    internal async Task<bool> RestartIfUnhealthy(CallContext ctx)
+    {
+        bool wasHealthy;
+
+        try
+        {
+            var prmNodeInfo = new PrmNodeInfo(ctx);
+            var response = await Client!.GetNodeInfoAsync(prmNodeInfo).ConfigureAwait(false);
+            return false;
+        }
+        catch (RpcException)
+        {
+            wasHealthy = true;
+        }
+
+        // if connection is dialed before, to avoid routine/connection leak,
+        // pool has to close it and then initialize once again.
+        if (IsDialed())
+        {
+            await ScheduleGracefulClose().ConfigureAwait(false);
+        }
+
+        FrostFSClient? client = null;
+
+        try
+        {
+            client = new(WrapperPrm, sessionCache);
+
+            var dialCtx = new CallContext
+            {
+                Timeout = TimeSpan.FromTicks((long)WrapperPrm.DialTimeout),
+                CancellationToken = ctx.CancellationToken
+            };
+
+            var error = await client.Dial(ctx).ConfigureAwait(false);
+            if (!string.IsNullOrEmpty(error))
+            {
+                SetUnhealthyOnDial();
+                return wasHealthy;
+            }
+
+            lock (_lock)
+            {
+                Client = client;
+            }
+
+            client = null;
+        }
+        finally
+        {
+            client?.Dispose();
+        }
+
+        try
+        {
+            var prmNodeInfo = new PrmNodeInfo(ctx);
+            var res = await Client.GetNodeInfoAsync(prmNodeInfo).ConfigureAwait(false);
+        }
+        catch (FrostFsException)
+        {
+            SetUnhealthy();
+            return wasHealthy;
+        }
+
+        SetHealthy();
+        return !wasHealthy;
+    }
+
+    internal void IncRequests(ulong elapsed, MethodIndex method)
+    {
+        var methodStat = Methods[(int)method];
+
+        methodStat.IncRequests(elapsed);
+    }
+}
+

src/FrostFS.SDK.Client/Pool/HealthyStatus.cs

@@ -0,0 +1,18 @@
+namespace FrostFS.SDK.Client;
+
+// values for healthy status of clientStatusMonitor.
+public enum HealthyStatus
+{
+    // statusUnhealthyOnDial is set when dialing to the endpoint is failed,
+    // so there is no connection to the endpoint, and pool should not close it
+    // before re-establishing connection once again.
+    UnhealthyOnDial,
+
+    // statusUnhealthyOnRequest is set when communication after dialing to the
+    // endpoint is failed due to immediate or accumulated errors, connection is
+    // available and pool should close it before re-establishing connection once again.
+    UnhealthyOnRequest,
+
+    // statusHealthy is set when connection is ready to be used by the pool.
+    Healthy
+}

src/FrostFS.SDK.Client/Pool/IClientStatus.cs

@@ -0,0 +1,28 @@
+namespace FrostFS.SDK.Client;
+
+public interface IClientStatus
+{
+    // isHealthy checks if the connection can handle requests.
+    bool IsHealthy();
+
+    // isDialed checks if the connection was created.
+    bool IsDialed();
+
+    // setUnhealthy marks client as unhealthy.
+    void SetUnhealthy();
+
+    // address return address of endpoint.
+    string Address { get; }
+
+    // currentErrorRate returns current errors rate.
+    // After specific threshold connection is considered as unhealthy.
+    // Pool.startRebalance routine can make this connection healthy again.
+    uint GetCurrentErrorRate();
+
+    // overallErrorRate returns the number of all happened errors.
+    ulong GetOverallErrorRate();
+
+    // methodsStatus returns statistic for all used methods.
+    StatusSnapshot[] MethodsStatus();
+}
+

src/FrostFS.SDK.Client/Pool/InitParameters.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Security.Cryptography;
+
+using Grpc.Net.Client;
+
+using Microsoft.Extensions.Logging;
+
+namespace FrostFS.SDK.Client;
+
+// InitParameters contains values used to initialize connection Pool.
+public class InitParameters
+{
+    public ECDsa? Key { get; set; }
+
+    public ulong NodeDialTimeout { get; set; }
+
+    public ulong NodeStreamTimeout { get; set; }
+
+    public ulong HealthcheckTimeout { get; set; }
+
+    public ulong ClientRebalanceInterval { get; set; }
+
+    public ulong SessionExpirationDuration { get; set; }
+
+    public uint ErrorThreshold { get; set; }
+
+    public NodeParam[]? NodeParams { get; set; }
+
+    public GrpcChannelOptions[]? DialOptions { get; set; }
+
+    public Func<string, ClientWrapper>? ClientBuilder { get; set; }
+
+    public ulong GracefulCloseOnSwitchTimeout { get; set; }
+
+    public ILogger? Logger { get; set; }
+}

src/FrostFS.SDK.Client/Pool/InnerPool.cs

@@ -0,0 +1,47 @@
+using FrostFS.SDK.Client;
+
+internal sealed class InnerPool
+{
+    private readonly object _lock = new();
+
+    internal InnerPool(Sampler sampler, ClientWrapper[] clients)
+    {
+        Clients = clients;
+        Sampler = sampler;
+    }
+
+    internal Sampler Sampler { get; set; }
+
+    internal ClientWrapper[] Clients { get; }
+
+    internal ClientWrapper? Connection()
+    {
+        lock (_lock)
+        {
+            if (Clients.Length == 1)
+            {
+                var client = Clients[0];
+                if (client.IsHealthy())
+                {
+                    return client;
+                }
+            }
+            else
+            {
+                var attempts = 3 * Clients.Length;
+
+                for (int i = 0; i < attempts; i++)
+                {
+                    int index = Sampler.Next();
+
+                    if (Clients[index].IsHealthy())
+                    {
+                        return Clients[index];
+                    }
+                }
+            }
+
+            return null;
+        }
+    }
+}

src/FrostFS.SDK.Client/Pool/MethodIndex.cs

@@ -0,0 +1,24 @@
+namespace FrostFS.SDK.Client;
+
+public enum MethodIndex
+{
+    methodBalanceGet,
+    methodContainerPut,
+    methodContainerGet,
+    methodContainerList,
+    methodContainerDelete,
+    methodEndpointInfo,
+    methodNetworkInfo,
+    methodNetMapSnapshot,
+    methodObjectPut,
+    methodObjectDelete,
+    methodObjectGet,
+    methodObjectHead,
+    methodObjectRange,
+    methodObjectPatch,
+    methodSessionCreate,
+    methodAPEManagerAddChain,
+    methodAPEManagerRemoveChain,
+    methodAPEManagerListChains,
+    methodLast
+}

src/FrostFS.SDK.Client/Pool/MethodStatus.cs

@@ -0,0 +1,19 @@
+namespace FrostFS.SDK.Client;
+
+public class MethodStatus(string name)
+{
+    private readonly object _lock = new();
+
+    public string Name { get; } = name;
+
+    public StatusSnapshot Snapshot { get; set; } = new StatusSnapshot();
+
+    internal void IncRequests(ulong elapsed)
+    {
+        lock (_lock)
+        {
+            Snapshot.AllTime += elapsed;
+            Snapshot.AllRequests++;
+        }
+    }
+}

src/FrostFS.SDK.Client/Pool/NodeParam.cs

@@ -0,0 +1,12 @@
+namespace FrostFS.SDK.Client;
+
+// NodeParam groups parameters of remote node.
+[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>")]
+public readonly struct NodeParam(int priority, string address, float weight)
+{
+    public int Priority { get; } = priority;
+
+    public string Address { get; } = address;
+
+    public float Weight { get; } = weight;
+}

src/FrostFS.SDK.Client/Pool/NodeStatistic.cs

@@ -0,0 +1,12 @@
+namespace FrostFS.SDK.Client;
+
+public class NodeStatistic
+{
+    public string? Address { get; internal set; }
+
+    public StatusSnapshot[]? Methods { get; internal set; }
+
+    public ulong OverallErrors { get; internal set; }
+
+    public uint CurrentErrors { get; internal set; }
+}

src/FrostFS.SDK.Client/Pool/NodesParam.cs

@@ -0,0 +1,12 @@
+using System.Collections.ObjectModel;
+
+namespace FrostFS.SDK.Client;
+
+public class NodesParam(int priority)
+{
+    public int Priority { get; } = priority;
+
+    public Collection<string> Addresses { get; } = [];
+
+    public Collection<double> Weights { get; } = [];
+}

src/FrostFS.SDK.Client/Pool/Pool.cs

@@ -0,0 +1,831 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Frostfs.V2.Ape;
+
+using FrostFS.Refs;
+using FrostFS.SDK.Client.Interfaces;
+using FrostFS.SDK.Client.Mappers.GRPC;
+using FrostFS.SDK.Cryptography;
+
+using Grpc.Core;
+
+using Microsoft.Extensions.Logging;
+
+
+namespace FrostFS.SDK.Client;
+
+public partial class Pool : IFrostFSClient
+{
+    const int defaultSessionTokenExpirationDuration = 100; // in epochs
+
+    const int defaultErrorThreshold = 100;
+
+    const int defaultGracefulCloseOnSwitchTimeout = 10; //Seconds;
+    const int defaultRebalanceInterval = 15; //Seconds;
+    const int defaultHealthcheckTimeout = 4; //Seconds;
+    const int defaultDialTimeout = 5; //Seconds;
+    const int defaultStreamTimeout = 10; //Seconds;
+
+    private readonly object _lock = new();
+
+    private InnerPool[]? InnerPools { get; set; }
+
+    private ECDsa Key { get; set; }
+
+    private string PublicKey { get; }
+
+    private OwnerID? _ownerId;
+    private FrostFsOwner? _owner;
+
+    private FrostFsOwner Owner
+    {
+        get
+        {
+            _owner ??= new FrostFsOwner(Key.PublicKey().PublicKeyToAddress());
+            return _owner;
+        }
+    }
+
+    private OwnerID OwnerId
+    {
+        get
+        {
+            if (_ownerId == null)
+            {
+                _owner = new FrostFsOwner(Key.PublicKey().PublicKeyToAddress());
+                _ownerId = _owner.ToMessage();
+            }
+            return _ownerId;
+        }
+    }
+
+    internal CancellationTokenSource CancellationTokenSource { get; } = new CancellationTokenSource();
+
+    internal SessionCache SessionCache { get; set; }
+
+    private ulong SessionTokenDuration { get; set; }
+
+    private RebalanceParameters RebalanceParams { get; set; }
+
+    private Func<string, ClientWrapper> ClientBuilder;
+
+    private bool disposedValue;
+
+    private ILogger? logger { get; set; }
+
+    private ulong MaxObjectSize { get; set; }
+
+    public IClientStatus? ClientStatus { get; }
+
+    // NewPool creates connection pool using parameters.
+    public Pool(InitParameters options)
+    {
+        if (options is null)
+        {
+            throw new ArgumentNullException(nameof(options));
+        }
+
+        if (options.Key == null)
+        {
+            throw new ArgumentException($"Missed required parameter {nameof(options.Key)}");
+        }
+
+        var nodesParams = AdjustNodeParams(options.NodeParams);
+
+        var cache = new SessionCache(options.SessionExpirationDuration);
+
+        FillDefaultInitParams(options, this);
+
+        Key = options.Key;
+        PublicKey = $"{Key.PublicKey()}";
+
+        SessionCache = cache;
+        logger = options.Logger;
+        SessionTokenDuration = options.SessionExpirationDuration;
+
+        RebalanceParams = new RebalanceParameters(
+            nodesParams.ToArray(),
+            options.HealthcheckTimeout,
+            options.ClientRebalanceInterval,
+            options.SessionExpirationDuration);
+
+        ClientBuilder = options.ClientBuilder!;
+    }
+
+    private void SetupContext(CallContext ctx)
+    {
+        if (ctx == null)
+        {
+            throw new ArgumentNullException(nameof(ctx));
+        }
+
+        ctx.Key ??= Key;
+    }
+
+    // Dial establishes a connection to the servers from the FrostFS network.
+    // It also starts a routine that checks the health of the nodes and
+    // updates the weights of the nodes for balancing.
+    // Returns an error describing failure reason.
+    //
+    // If failed, the Pool SHOULD NOT be used.
+    //
+    // See also InitParameters.SetClientRebalanceInterval.
+    public async Task<string?> Dial(CallContext ctx)
+    {
+        SetupContext(ctx);
+
+        var inner = new InnerPool[RebalanceParams.NodesParams.Length];
+
+        bool atLeastOneHealthy = false;
+        int i = 0;
+        foreach (var nodeParams in RebalanceParams.NodesParams)
+        {
+            var clients = new ClientWrapper[nodeParams.Weights.Count];
+
+            for (int j = 0; j < nodeParams.Addresses.Count; j++)
+            {
+                ClientWrapper? client = null;
+                bool dialed = false;
+                try
+                {
+                    client = clients[j] = ClientBuilder(nodeParams.Addresses[j]);
+
+                    await client.Dial(ctx).ConfigureAwait(false);
+                    dialed = true;
+
+                    var token = await InitSessionForDuration(ctx, client, RebalanceParams.SessionExpirationDuration, Key, false)
+                        .ConfigureAwait(false);
+
+                    var key = FormCacheKey(nodeParams.Addresses[j], Key.PrivateKey().ToString());
+                    _ = SessionCache.Cache[key] = token;
+
+                    atLeastOneHealthy = true;
+                }
+                catch (RpcException ex)
+                {
+                    if (!dialed)
+                        client!.SetUnhealthyOnDial();
+                    else
+                        client!.SetUnhealthy();
+
+                    if (logger != null)
+                    {
+                        FrostFsMessages.SessionCreationError(logger, client!.WrapperPrm.Address, ex.Message);
+                    }
+                }
+                catch (FrostFsInvalidObjectException)
+                {
+                    break;
+                }
+            }
+
+            var sampler = new Sampler(nodeParams.Weights.ToArray());
+
+            inner[i] = new InnerPool(sampler, clients);
+
+            i++;
+        }
+
+        if (!atLeastOneHealthy)
+            return "At least one node must be healthy";
+
+        InnerPools = inner;
+
+        var res = await GetNetworkSettingsAsync(new PrmNetworkSettings(ctx)).ConfigureAwait(false);
+
+        MaxObjectSize = res.MaxObjectSize;
+
+        StartRebalance(ctx);
+
+        return null;
+    }
+
+    private static IEnumerable<NodesParam> AdjustNodeParams(NodeParam[]? nodeParams)
+    {
+        if (nodeParams == null || nodeParams.Length == 0)
+        {
+            throw new ArgumentException("No FrostFS peers configured");
+        }
+
+        Dictionary<int, NodesParam> nodesParamsDict = new(nodeParams.Length);
+        foreach (var nodeParam in nodeParams)
+        {
+            if (!nodesParamsDict.TryGetValue(nodeParam.Priority, out var nodes))
+            {
+                nodes = new NodesParam(nodeParam.Priority);
+                nodesParamsDict[nodeParam.Priority] = nodes;
+            }
+
+            nodes.Addresses.Add(nodeParam.Address);
+            nodes.Weights.Add(nodeParam.Weight);
+        }
+
+        var nodesParams = new List<NodesParam>(nodesParamsDict.Count);
+
+        foreach (var key in nodesParamsDict.Keys)
+        {
+            var nodes = nodesParamsDict[key];
+            var newWeights = AdjustWeights([.. nodes.Weights]);
+            nodes.Weights.Clear();
+            foreach (var weight in newWeights)
+            {
+                nodes.Weights.Add(weight);
+            }
+
+            nodesParams.Add(nodes);
+        }
+
+        return nodesParams.OrderBy(n => n.Priority);
+    }
+
+    private static double[] AdjustWeights(double[] weights)
+    {
+        var adjusted = new double[weights.Length];
+
+        var sum = weights.Sum();
+
+        if (sum > 0)
+        {
+            for (int i = 0; i < weights.Length; i++)
+            {
+                adjusted[i] = weights[i] / sum;
+            }
+        }
+
+        return adjusted;
+    }
+
+    private static void FillDefaultInitParams(InitParameters parameters, Pool pool)
+    {
+        if (parameters.SessionExpirationDuration == 0)
+            parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration;
+
+        if (parameters.ErrorThreshold == 0)
+            parameters.ErrorThreshold = defaultErrorThreshold;
+
+        if (parameters.ClientRebalanceInterval <= 0)
+            parameters.ClientRebalanceInterval = defaultRebalanceInterval;
+
+        if (parameters.GracefulCloseOnSwitchTimeout <= 0)
+            parameters.GracefulCloseOnSwitchTimeout = defaultGracefulCloseOnSwitchTimeout;
+
+        if (parameters.HealthcheckTimeout <= 0)
+            parameters.HealthcheckTimeout = defaultHealthcheckTimeout;
+
+        if (parameters.NodeDialTimeout <= 0)
+            parameters.NodeDialTimeout = defaultDialTimeout;
+
+        if (parameters.NodeStreamTimeout <= 0)
+            parameters.NodeStreamTimeout = defaultStreamTimeout;
+
+        if (parameters.SessionExpirationDuration == 0)
+            parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration;
+
+        parameters.ClientBuilder ??= new Func<string, ClientWrapper>((address) =>
+            {
+                var wrapperPrm = new WrapperPrm
+                {
+                    Address = address,
+                    Key = parameters.Key,
+                    Logger = parameters.Logger,
+                    DialTimeout = parameters.NodeDialTimeout,
+                    StreamTimeout = parameters.NodeStreamTimeout,
+                    ErrorThreshold = parameters.ErrorThreshold,
+                    GracefulCloseOnSwitchTimeout = parameters.GracefulCloseOnSwitchTimeout
+                };
+
+                return new ClientWrapper(wrapperPrm, pool);
+            }
+        );
+    }
+
+    private ClientWrapper Connection()
+    {
+        foreach (var pool in InnerPools!)
+        {
+            var client = pool.Connection();
+            if (client != null)
+            {
+                return client;
+            }
+        }
+
+        throw new FrostFsException("Cannot find alive client");
+    }
+
+    private static async Task<FrostFsSessionToken?> InitSessionForDuration(CallContext ctx, ClientWrapper cw, ulong duration, ECDsa key, bool clientCut)
+    {
+        var client = cw.Client;
+        var networkInfo = await client!.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx)).ConfigureAwait(false);
+
+        var epoch = networkInfo.Epoch;
+
+        ulong exp = ulong.MaxValue - epoch < duration
+            ? ulong.MaxValue
+            : epoch + duration;
+
+        var prmSessionCreate = new PrmSessionCreate(exp, ctx);
+
+        return await client.CreateSessionAsync(prmSessionCreate).ConfigureAwait(false);
+    }
+
+    internal static string FormCacheKey(string address, string key)
+    {
+        return $"{address}{key}";
+    }
+
+    public void Close()
+    {
+        CancellationTokenSource.Cancel();
+
+        if (InnerPools != null)
+        {
+            // close all clients
+            foreach (var innerPool in InnerPools)
+                foreach (var client in innerPool.Clients)
+                    if (client.IsDialed())
+                        client.Client?.Close();
+        }
+    }
+
+    // startRebalance runs loop to monitor connection healthy status.
+    internal void StartRebalance(CallContext ctx)
+    {
+        var buffers = new double[RebalanceParams.NodesParams.Length][];
+
+        for (int i = 0; i < RebalanceParams.NodesParams.Length; i++)
+        {
+            var parameters = RebalanceParams.NodesParams[i];
+            buffers[i] = new double[parameters.Weights.Count];
+
+            Task.Run(async () =>
+            {
+                await Task.Delay((int)RebalanceParams.ClientRebalanceInterval).ConfigureAwait(false);
+                UpdateNodesHealth(ctx, buffers);
+            });
+        }
+    }
+
+    private void UpdateNodesHealth(CallContext ctx, double[][] buffers)
+    {
+        var tasks = new Task[InnerPools!.Length];
+
+        for (int i = 0; i < InnerPools.Length; i++)
+        {
+            var bufferWeights = buffers[i];
+
+            tasks[i] = Task.Run(() => UpdateInnerNodesHealth(ctx, i, bufferWeights));
+        }
+
+        Task.WaitAll(tasks);
+    }
+
+    private async ValueTask UpdateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights)
+    {
+        if (poolIndex > InnerPools!.Length - 1)
+        {
+            return;
+        }
+
+        var pool = InnerPools[poolIndex];
+
+        var options = RebalanceParams;
+
+        int healthyChanged = 0;
+
+        var tasks = new Task[pool.Clients.Length];
+
+        for (int j = 0; j < pool.Clients.Length; j++)
+        {
+            var client = pool.Clients[j];
+            var healthy = false;
+            string? error = null;
+            var changed = false;
+
+            try
+            {
+                // check timeout settings
+                changed = await client!.RestartIfUnhealthy(ctx).ConfigureAwait(false);
+                healthy = true;
+                bufferWeights[j] = options.NodesParams[poolIndex].Weights[j];
+            }
+            // TODO: specify
+            catch (FrostFsException e)
+            {
+                error = e.Message;
+                bufferWeights[j] = 0;
+
+                SessionCache.DeleteByPrefix(client.Address);
+            }
+
+            if (changed)
+            {
+                if (!string.IsNullOrEmpty(error))
+                {
+                    if (logger != null)
+                    {
+                        FrostFsMessages.HealthChanged(logger, client.Address, healthy, error!);
+                    }
+
+                    Interlocked.Exchange(ref healthyChanged, 1);
+                }
+            }
+
+            await Task.WhenAll(tasks).ConfigureAwait(false);
+
+            if (Interlocked.CompareExchange(ref healthyChanged, -1, -1) == 1)
+            {
+                var probabilities = AdjustWeights(bufferWeights);
+
+                lock (_lock)
+                {
+                    pool.Sampler = new Sampler(probabilities);
+                }
+            }
+        }
+    }
+
+
+    // TODO: remove
+    private bool CheckSessionTokenErr(Exception error, string address)
+    {
+        if (error == null)
+        {
+            return false;
+        }
+
+        if (error is SessionNotFoundException || error is SessionExpiredException)
+        {
+            this.SessionCache.DeleteByPrefix(address);
+            return true;
+        }
+
+        return false;
+    }
+
+    public Statistic Statistic()
+    {
+        if (InnerPools == null)
+        {
+            throw new FrostFsInvalidObjectException(nameof(Pool));
+        }
+
+        var statistics = new Statistic();
+
+        foreach (var inner in InnerPools)
+        {
+            int valueIndex = 0;
+            var nodes = new string[inner.Clients.Length];
+
+            lock (_lock)
+            {
+                foreach (var client in inner.Clients)
+                {
+                    if (client.IsHealthy())
+                    {
+                        nodes[valueIndex] = client.Address;
+                    }
+
+                    var node = new NodeStatistic
+                    {
+                        Address = client.Address,
+                        Methods = client.MethodsStatus(),
+                        OverallErrors = client.GetOverallErrorRate(),
+                        CurrentErrors = client.GetCurrentErrorRate()
+                    };
+
+                    statistics.Nodes.Add(node);
+
+                    valueIndex++;
+
+                    statistics.OverallErrors += node.OverallErrors;
+                }
+
+                if (statistics.CurrentNodes == null || statistics.CurrentNodes.Length == 0)
+                {
+                    statistics.CurrentNodes = nodes;
+                }
+            }
+        }
+
+        return statistics;
+    }
+
+    public async Task<FrostFsNetmapSnapshot> GetNetmapSnapshotAsync(PrmNetmapSnapshot? args = null)
+    {
+        var client = Connection();
+
+        args ??= new();
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetNetmapSnapshotAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsNodeInfo> GetNodeInfoAsync(PrmNodeInfo? args = null)
+    {
+        var client = Connection();
+
+        args ??= new();
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetNodeInfoAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<NetworkSettings> GetNetworkSettingsAsync(PrmNetworkSettings? args = null)
+    {
+        var client = Connection();
+
+        args ??= new();
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetNetworkSettingsAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsSessionToken> CreateSessionAsync(PrmSessionCreate args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.CreateSessionAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<byte[]> AddChainAsync(PrmApeChainAdd args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.AddChainAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task RemoveChainAsync(PrmApeChainRemove args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        await client.Client!.RemoveChainAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<Chain[]> ListChainAsync(PrmApeChainList args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.ListChainAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsContainerInfo> GetContainerAsync(PrmContainerGet args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetContainerAsync(args).ConfigureAwait(false);
+    }
+
+    public IAsyncEnumerable<FrostFsContainerId> ListContainersAsync(PrmContainerGetAll? args = null)
+    {
+        var client = Connection();
+
+        args ??= new();
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return client.Client!.ListContainersAsync(args);
+    }
+
+    public async Task<FrostFsContainerId> CreateContainerAsync(PrmContainerCreate args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.CreateContainerAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task DeleteContainerAsync(PrmContainerDelete args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        await client.Client!.DeleteContainerAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsObjectHeader> GetObjectHeadAsync(PrmObjectHeadGet args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetObjectHeadAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsObject> GetObjectAsync(PrmObjectGet args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetObjectAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsObjectId> PutObjectAsync(PrmObjectPut args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.PutObjectAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsObjectId> PutSingleObjectAsync(PrmSingleObjectPut args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.PutSingleObjectAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsObjectId> PatchObjectAsync(PrmObjectPatch args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.PatchObjectAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<RangeReader> GetRangeAsync(PrmRangeGet args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetRangeAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<ReadOnlyMemory<byte>[]> GetRangeHashAsync(PrmRangeHashGet args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetRangeHashAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task<FrostFsObjectId> PatchAsync(PrmObjectPatch args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.PatchObjectAsync(args).ConfigureAwait(false);
+    }
+
+    public async Task DeleteObjectAsync(PrmObjectDelete args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        await client.Client!.DeleteObjectAsync(args).ConfigureAwait(false);
+    }
+
+    public IAsyncEnumerable<FrostFsObjectId> SearchObjectsAsync(PrmObjectSearch args)
+    {
+        if (args is null)
+        {
+            throw new ArgumentNullException(nameof(args));
+        }
+
+        var client = Connection();
+
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return client.Client!.SearchObjectsAsync(args);
+    }
+
+    public async Task<Accounting.Decimal> GetBalanceAsync(PrmBalance? args)
+    {
+        var client = Connection();
+
+        args ??= new();
+        args.Context.PoolErrorHandler = client.HandleError;
+
+        return await client.Client!.GetBalanceAsync(args).ConfigureAwait(false);
+    }
+
+    protected virtual void Dispose(bool disposing)
+    {
+        if (!disposedValue)
+        {
+            if (disposing)
+            {
+                Close();
+            }
+
+            disposedValue = true;
+        }
+    }
+
+    public void Dispose()
+    {
+        // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+
+    public FrostFsObjectId CalculateObjectId(FrostFsObjectHeader header, CallContext ctx)
+    {
+        throw new NotImplementedException();
+    }
+}

src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs

@@ -0,0 +1,16 @@
+namespace FrostFS.SDK.Client;
+
+public class RebalanceParameters(
+    NodesParam[] nodesParams,
+    ulong nodeRequestTimeout,
+    ulong clientRebalanceInterval,
+    ulong sessionExpirationDuration)
+{
+    public NodesParam[] NodesParams { get; set; } = nodesParams;
+
+    public ulong NodeRequestTimeout { get; set; } = nodeRequestTimeout;
+
+    public ulong ClientRebalanceInterval { get; set; } = clientRebalanceInterval;
+
+    public ulong SessionExpirationDuration { get; set; } = sessionExpirationDuration;
+}

src/FrostFS.SDK.Client/Pool/RequestInfo.cs

@@ -0,0 +1,14 @@
+using System;
+
+namespace FrostFS.SDK.Client;
+
+// RequestInfo groups info about pool request.
+struct RequestInfo
+{
+    public string Address { get; set; }
+
+    public MethodIndex MethodIndex { get; set; }
+
+    public TimeSpan Elapsed { get; set; }
+}
+

src/FrostFS.SDK.Client/Pool/Sampler.cs

@@ -0,0 +1,85 @@
+using System;
+
+namespace FrostFS.SDK.Client;
+
+internal sealed class Sampler
+{
+    private readonly object _lock = new();
+
+    private Random random = new();
+
+    internal double[] Probabilities { get; set; }
+    internal int[] Alias { get; set; }
+
+    internal Sampler(double[] probabilities)
+    {
+        var small = new WorkList();
+        var large = new WorkList();
+
+        var n = probabilities.Length;
+
+        //  sampler.randomGenerator = rand.New(source)
+        Probabilities = new double[n];
+        Alias = new int[n];
+
+        // Compute scaled probabilities.
+        var p = new double[n];
+
+        for (int i = 0; i < n; i++)
+        {
+            p[i] = probabilities[i] * n;
+            if (p[i] < 1)
+                small.Add(i);
+            else
+                large.Add(i);
+        }
+
+        while (small.Length > 0 && large.Length > 0)
+        {
+            var l = small.Remove();
+            var g = large.Remove();
+
+            Probabilities[l] = p[l];
+            Alias[l] = g;
+
+            p[g] = p[g] + p[l] - 1;
+
+            if (p[g] < 1)
+                small.Add(g);
+            else
+                large.Add(g);
+        }
+
+        while (large.Length > 0)
+        {
+            var g = large.Remove();
+            Probabilities[g] = 1;
+        }
+
+        while (small.Length > 0)
+        {
+            var l = small.Remove();
+            probabilities[l] = 1;
+        }
+    }
+
+    internal int Next()
+    {
+        var n = Alias.Length;
+
+        int i;
+        double f;
+        lock (_lock)
+        {
+            i = random.Next(0, n - 1);
+            f = random.NextDouble();
+        }
+
+        if (f < Probabilities[i])
+        {
+            return i;
+        }
+
+        return Alias[i];
+    }
+}

src/FrostFS.SDK.Client/Pool/SessionCache.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections;
+
+namespace FrostFS.SDK.Client;
+
+internal struct SessionCache(ulong sessionExpirationDuration)
+{
+    internal Hashtable Cache { get; } = [];
+
+    internal ulong CurrentEpoch { get; set; }
+
+    internal ulong TokenDuration { get; set; } = sessionExpirationDuration;
+
+    internal void DeleteByPrefix(string prefix)
+    {
+        foreach (var key in Cache.Keys)
+        {
+            if (((string)key).StartsWith(prefix, StringComparison.Ordinal))
+            {
+                Cache.Remove(key);
+            }
+        }
+    }
+}

src/FrostFS.SDK.Client/Pool/Statistic.cs

@@ -0,0 +1,12 @@
+using System.Collections.ObjectModel;
+
+namespace FrostFS.SDK.Client;
+
+public sealed class Statistic
+{
+    public ulong OverallErrors { get; internal set; }
+
+    public Collection<NodeStatistic> Nodes { get; } = [];
+
+    public string[]? CurrentNodes { get; internal set; }
+}

src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs

@@ -0,0 +1,8 @@
+namespace FrostFS.SDK.Client;
+
+public class StatusSnapshot()
+{
+    public ulong AllTime { get; internal set; }
+
+    public ulong AllRequests { get; internal set; }
+}

src/FrostFS.SDK.Client/Pool/WorkList.cs

@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace FrostFS.SDK.Client;
+
+internal sealed class WorkList
+{
+    private readonly List<int> elements = [];
+
+    internal int Length
+    {
+        get { return elements.Count; }
+    }
+
+    internal void Add(int element)
+    {
+        elements.Add(element);
+    }
+
+    internal int Remove()
+    {
+        int last = elements.LastOrDefault();
+        elements.RemoveAt(elements.Count - 1);
+        return last;
+    }
+}

src/FrostFS.SDK.Client/Pool/WrapperPrm.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Security.Cryptography;
+
+using Grpc.Net.Client;
+
+using Microsoft.Extensions.Logging;
+
+namespace FrostFS.SDK.Client;
+
+// wrapperPrm is params to create clientWrapper.
+[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>")]
+public struct WrapperPrm
+{
+    internal ILogger? Logger { get; set; }
+
+    internal string Address { get; set; }
+
+    internal ECDsa? Key { get; set; }
+
+    internal ulong DialTimeout { get; set; }
+
+    internal ulong StreamTimeout { get; set; }
+
+    internal uint ErrorThreshold { get; set; }
+
+    internal Action ResponseInfoCallback { get; set; }
+
+    internal Action PoolRequestInfoCallback { get; set; }
+
+    internal GrpcChannelOptions GrpcChannelOptions { get; set; }
+
+    internal ulong GracefulCloseOnSwitchTimeout { get; set; }
+}
+