potyarkin/frostfs-sdk-csharp
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-csharp
Code Pull requests Activity

[#43] Client: Memory optimization

Browse source 
Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2025-03-26 16:51:42 +03:00
parent 5e86f53b0e
commit 87fe8db674
76 changed files with 399 additions and 3668 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FrostFS.SDK.Client/ApeRules/Actions.cs
View file

@ -8,7 +8,7 @@ public struct Actions(bool inverted, string[] names) : System.IEquatable<Actions
public override readonly bool Equals(object obj)
{
if (obj == null || obj is not Actions)
if (obj == null || obj is not Actions)
return false;
return Equals((Actions)obj);

2
src/FrostFS.SDK.Client/ApeRules/Condition.cs
View file

@ -12,7 +12,7 @@ public struct Condition : System.IEquatable<Condition>
public override bool Equals(object obj)
{
if (obj == null || obj is not Condition)
if (obj == null || obj is not Condition)
return false;
return Equals((Condition)obj);

2
src/FrostFS.SDK.Client/ApeRules/Resources.cs
View file

@ -8,7 +8,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable<Resou
public override readonly bool Equals(object obj)
{
if (obj == null || obj is not Resources)
if (obj == null || obj is not Resources)
return false;
return Equals((Resources)obj);

4
src/FrostFS.SDK.Client/AssemblyInfo.cs
View file

@ -1,8 +1,8 @@
using System.Reflection;
[assembly: AssemblyCompany("FrostFS.SDK.Client")]
[assembly: AssemblyFileVersion("1.0.2.0")]
[assembly: AssemblyFileVersion("1.0.4.0")]
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
[assembly: AssemblyProduct("FrostFS.SDK.Client")]
[assembly: AssemblyTitle("FrostFS.SDK.Client")]
[assembly: AssemblyVersion("1.0.3")]
[assembly: AssemblyVersion("1.0.4")]

9
src/FrostFS.SDK.Client/Caches.cs
View file

@ -6,17 +6,8 @@ internal static class Caches
{
private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions
{
// TODO: get from options?
SizeLimit = 256
});
private static readonly IMemoryCache _containersCache = new MemoryCache(new MemoryCacheOptions
{
// TODO: get from options?
SizeLimit = 1024
});
internal static IMemoryCache Owners => _ownersCache;
internal static IMemoryCache Containers => _containersCache;
}

4
src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs
View file

@ -10,8 +10,8 @@ public class FrostFsResponseException : FrostFsException
{
}
public FrostFsResponseException(FrostFsResponseStatus status)
: base(status != null ? status.Message != null ? "" : "" : "")
public FrostFsResponseException(FrostFsResponseStatus status)
: base(status != null ? status.Message != null ? "" : "" : "")
{
Status = status;
}

72
src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs
View file

@ -1,14 +1,17 @@
using System;
using System.Security.Cryptography;
using Google.Protobuf;
namespace FrostFS.SDK.Cryptography;
public static class FrostFsExtensions
{
public static ByteString Sha256(this IMessage data)
public static byte[] Sha256(this IMessage data)
{
return ByteString.CopyFrom(data.ToByteArray().Sha256());
using var sha256 = SHA256.Create();
using HashStream stream = new(sha256);
data.WriteTo(stream);
return stream.Hash();
}
public static Guid ToUuid(this ByteString id)
@ -16,9 +19,18 @@ public static class FrostFsExtensions
if (id == null)
throw new ArgumentNullException(nameof(id));
var orderedBytes = GetGuidBytesDirectOrder(id.Span);
return new Guid(orderedBytes);
return new Guid(
(id[0] << 24) + (id[1] << 16) + (id[2] << 8) + id[3],
(short)((id[4] << 8) + id[5]),
(short)((id[6] << 8) + id[7]),
id[8],
id[9],
id[10],
id[11],
id[12],
id[13],
id[14],
id[15]);
}
/// <summary>
@ -26,37 +38,25 @@ public static class FrostFsExtensions
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public static byte[] ToBytes(this Guid id)
public unsafe static void ToBytes(this Guid id, Span<byte> span)
{
var bytes = id.ToByteArray();
var pGuid = (byte*)&id;
var orderedBytes = GetGuidBytesDirectOrder(bytes);
return orderedBytes;
}
private static byte[] GetGuidBytesDirectOrder(ReadOnlySpan<byte> source)
{
if (source.Length != 16)
throw new ArgumentException("Wrong uuid binary format");
return [
source[3],
source[2],
source[1],
source[0],
source[5],
source[4],
source[7],
source[6],
source[8],
source[9],
source[10],
source[11],
source[12],
source[13],
source[14],
source[15]
];
span[0] = pGuid[3];
span[1] = pGuid[2];
span[2] = pGuid[1];
span[3] = pGuid[0];
span[4] = pGuid[5];
span[5] = pGuid[4];
span[6] = pGuid[7];
span[7] = pGuid[6];
span[8] = pGuid[8];
span[9] = pGuid[9];
span[10] = pGuid[10];
span[11] = pGuid[11];
span[12] = pGuid[12];
span[13] = pGuid[13];
span[14] = pGuid[14];
span[15] = pGuid[15];
}
}

2
src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj
View file

@ -31,10 +31,10 @@
<PropertyGroup>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
<PrivateAssets>all</PrivateAssets>

35
src/FrostFS.SDK.Client/FrostFSClient.cs
View file

@ -162,25 +162,7 @@ public class FrostFSClient : IFrostFSClient
Callback = settings.Value.Callback,
Interceptors = settings.Value.Interceptors
};
// TODO: define timeout logic
// CheckFrostFsVersionSupport(new Context { Timeout = TimeSpan.FromSeconds(20) });
}
internal FrostFSClient(WrapperPrm prm, SessionCache cache)
{
ClientCtx = new ClientContext(
client: this,
key: new ClientKey(prm.Key),
owner: FrostFsOwner.FromKey(prm.Key!),
channel: prm.GrpcChannelFactory(prm.Address),
version: new FrostFsVersion(2, 13))
{
SessionCache = cache,
Interceptors = prm.Interceptors,
Callback = prm.Callback
};
}
}
#region ApeManagerImplementation
public Task<ReadOnlyMemory<byte>> AddChainAsync(PrmApeChainAdd args, CallContext ctx)
@ -447,18 +429,5 @@ public class FrostFSClient : IFrostFSClient
}
return ObjectServiceProvider;
}
public async Task<string?> Dial(CallContext ctx)
{
var service = GetAccouningService();
_ = await service.GetBallance(ctx).ConfigureAwait(false);
return null;
}
public bool RestartIfUnhealthy(CallContext ctx)
{
throw new NotImplementedException();
}
}
}

2
src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs
View file

@ -64,6 +64,4 @@ public interface IFrostFSClient
#region Account
Task<Accounting.Decimal> GetBalanceAsync(CallContext ctx);
#endregion
public Task<string?> Dial(CallContext ctx);
}

16
src/FrostFS.SDK.Client/Mappers/ContainerId.cs
View file

@ -5,16 +5,10 @@ using FrostFS.SDK.Cryptography;
using Google.Protobuf;
using Microsoft.Extensions.Caching.Memory;
namespace FrostFS.SDK.Client.Mappers.GRPC;
public static class ContainerIdMapper
{
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
.SetSlidingExpiration(TimeSpan.FromHours(1))
.SetSize(1);
public static ContainerID ToMessage(this FrostFsContainerId model)
{
if (model is null)
@ -24,15 +18,11 @@ public static class ContainerIdMapper
var containerId = model.GetValue() ?? throw new ArgumentNullException(nameof(model));
if (!Caches.Containers.TryGetValue(containerId, out ContainerID? message))
var message = new ContainerID
{
message = new ContainerID
{
Value = ByteString.CopyFrom(Base58.Decode(containerId))
};
Value = UnsafeByteOperations.UnsafeWrap(Base58.Decode(containerId))
};
Caches.Containers.Set(containerId, message, _oneHourExpiration);
}
return message!;
}

33
src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs
View file

@ -24,25 +24,14 @@ public static class ObjectHeaderMapper
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
};
FrostFsSplit? split = null;
if (header.Split != null)
{
var children = header.Split.Children.Count != 0 ? new ReadOnlyCollection<FrostFsObjectId>(
header.Split.Children.Select(x => x.ToModel()).ToList()) : null;
split = new FrostFsSplit(new SplitId(header.Split.SplitId.ToUuid()),
header.Split.Previous?.ToModel(),
header.Split.Parent?.ToModel(),
header.Split.ParentHeader?.ToModel(),
null,
children);
}
FrostFsSplit? split = header!.Split != null
? header.Split.ToModel()
: null;
var model = new FrostFsObjectHeader(
new FrostFsContainerId(Base58.Encode(header.ContainerId.Value.Span)),
objTypeName,
header.Attributes.Select(attribute => attribute.ToModel()).ToArray(),
[.. header.Attributes.Select(attribute => attribute.ToModel())],
split,
header.OwnerId.ToModel(),
header.Version.ToModel())
@ -52,4 +41,18 @@ public static class ObjectHeaderMapper
return model;
}
public static FrostFsSplit ToModel(this Header.Types.Split split)
{
var children = split!.Children.Count != 0
? new ReadOnlyCollection<FrostFsObjectId>([.. split.Children.Select(x => x.ToModel())])
: null;
return new FrostFsSplit(new SplitId(split.SplitId.ToUuid()),
split.Previous?.ToModel(),
split.Parent?.ToModel(),
split.ParentHeader?.ToModel(),
null,
children);
}
}

2
src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs
View file

@ -17,7 +17,7 @@ public static class ObjectIdMapper
return new ObjectID
{
Value = ByteString.CopyFrom(objectId.ToHash())
Value = UnsafeByteOperations.UnsafeWrap(objectId.ToHash())
};
}

2
src/FrostFS.SDK.Client/Mappers/OwnerId.cs
View file

@ -26,7 +26,7 @@ public static class OwnerIdMapper
{
message = new OwnerID
{
Value = ByteString.CopyFrom(model.ToHash())
Value = UnsafeByteOperations.UnsafeWrap(model.ToHash())
};
Caches.Owners.Set(model, message, _oneHourExpiration);

28
src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs
View file

@ -1,28 +0,0 @@
using System;
using Google.Protobuf;
namespace FrostFS.SDK.Client;
public static class SessionMapper
{
public static byte[] Serialize(this Session.SessionToken token)
{
if (token is null)
{
throw new ArgumentNullException(nameof(token));
}
byte[] bytes = new byte[token.CalculateSize()];
using CodedOutputStream stream = new(bytes);
token.WriteTo(stream);
return bytes;
}
public static Session.SessionToken Deserialize(this Session.SessionToken token, byte[] bytes)
{
token.MergeFrom(bytes);
return token;
}
}

4
src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs
View file

@ -23,9 +23,9 @@ public static class SignatureMapper
return new Refs.Signature
{
Key = ByteString.CopyFrom(signature.Key),
Key = UnsafeByteOperations.UnsafeWrap(signature.Key),
Scheme = scheme,
Sign = ByteString.CopyFrom(signature.Sign)
Sign = UnsafeByteOperations.UnsafeWrap(signature.Sign)
};
}
}

5
src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs
View file

@ -91,10 +91,13 @@ public class FrostFsContainerInfo
throw new ArgumentNullException("PlacementPolicy is null");
}
Span<byte> nonce = stackalloc byte[16];
Nonce.ToBytes(nonce);
this.container = new Container.Container()
{
PlacementPolicy = PlacementPolicy.Value.GetPolicy(),
Nonce = ByteString.CopyFrom(Nonce.ToBytes()),
Nonce = ByteString.CopyFrom(nonce),
OwnerId = Owner?.OwnerID,
Version = Version?.VersionID
};

2
src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs
View file

@ -11,7 +11,7 @@ public class CheckSum
public static CheckSum CreateCheckSum(byte[] content)
{
return new CheckSum { hash = content.Sha256() };
return new CheckSum { hash = DataHasher.Sha256(content.AsSpan()) };
}
public override string ToString()

2
src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs
View file

@ -353,7 +353,7 @@ internal struct Context
var start = hasPrefix ? likeWildcard.Length : 0;
var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length;
var str = f.Value.Substring(start, end-start);
var str = f.Value.Substring(start, end - start);
if (hasPrefix && hasSuffix)
return nodeInfo.Attributes[f.Key].Contains(str);

23
src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs
View file

@ -4,10 +4,6 @@ namespace FrostFS.SDK;
public class FrostFsObject
{
// private byte[]? _payloadBytes;
// private ReadOnlyMemory<byte> _payloadMemory;
// private bool _isInitPayloadMemory;
/// <summary>
/// Creates new instance from <c>ObjectHeader</c>
/// </summary>
@ -49,25 +45,6 @@ public class FrostFsObject
public ReadOnlyMemory<byte> SingleObjectPayload { get; set; }
//public ReadOnlyMemory<byte> SingleObjectPayloadMemory
//{
// get
// {
// if (!_isInitPayloadMemory)
// {
// _payloadMemory = _payloadBytes.AsMemory();
// _isInitPayloadMemory = true;
// }
// return _payloadMemory;
// }
// set
// {
// _payloadMemory = value;
// _isInitPayloadMemory = true;
// }
//}
/// <summary>
/// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic
/// </summary>

26
src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs
View file

@ -1,4 +1,7 @@
using System.Collections.ObjectModel;
using System.Linq;
using FrostFS.Object;
using FrostFS.SDK.Client.Mappers.GRPC;
namespace FrostFS.SDK;
@ -9,6 +12,8 @@ public class FrostFsSplit(SplitId splitId,
FrostFsSignature? parentSignature = null,
ReadOnlyCollection<FrostFsObjectId>? children = null)
{
private Header.Types.Split? _split;
public FrostFsSplit() : this(new SplitId())
{
}
@ -24,4 +29,25 @@ public class FrostFsSplit(SplitId splitId,
public FrostFsObjectHeader? ParentHeader { get; set; } = parentHeader;
public ReadOnlyCollection<FrostFsObjectId>? Children { get; } = children;
public Header.Types.Split GetSplit()
{
if (_split == null)
{
_split = new Header.Types.Split
{
SplitId = SplitId?.GetSplitId(),
Parent = Parent?.ToMessage(),
ParentHeader = ParentHeader?.GetHeader(),
ParentSignature = ParentSignature?.ToMessage()
};
if (Children != null)
{
_split.Children.AddRange(Children.Select(x => x.ToMessage()));
}
}
return _split;
}
}

13
src/FrostFS.SDK.Client/Models/Object/SplitId.cs
View file

@ -47,16 +47,11 @@ public class SplitId
return this.id.ToString();
}
public byte[]? ToBinary()
{
if (this.id == Guid.Empty)
return null;
return this.id.ToBytes();
}
public ByteString? GetSplitId()
{
return this.message ??= ByteString.CopyFrom(ToBinary());
Span<byte> span = stackalloc byte[16];
id.ToBytes(span);
return this.message ??= ByteString.CopyFrom(span);
}
}

4
src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs
View file

@ -4,9 +4,9 @@ public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = nul
{
public FrostFsStatusCode Code { get; set; } = code;
public string Message { get; set; } = message ?? string.Empty;
public string Details { get; set; } = details ?? string.Empty;
public bool IsSuccess => Code == FrostFsStatusCode.Success;
public override string ToString()

2
src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs
View file

@ -25,7 +25,7 @@ public readonly struct PrmApeChainRemove(
public readonly bool Equals(PrmApeChainRemove other)
{
return Target == other.Target
&& ChainId.Equals(other.ChainId)
&& ChainId.Equals(other.ChainId)
&& XHeaders == other.XHeaders;
}

163
src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs
View file

@ -1,163 +0,0 @@
using System;
using System.Threading;
using Microsoft.Extensions.Logging;
namespace FrostFS.SDK.Client;
// clientStatusMonitor count error rate and other statistics for connection.
public class ClientStatusMonitor : IClientStatus
{
private static readonly MethodIndex[] MethodIndexes =
[
MethodIndex.methodBalanceGet,
MethodIndex.methodContainerPut,
MethodIndex.methodContainerGet,
MethodIndex.methodContainerList,
MethodIndex.methodContainerDelete,
MethodIndex.methodEndpointInfo,
MethodIndex.methodNetworkInfo,
MethodIndex.methodNetMapSnapshot,
MethodIndex.methodObjectPut,
MethodIndex.methodObjectDelete,
MethodIndex.methodObjectGet,
MethodIndex.methodObjectHead,
MethodIndex.methodObjectRange,
MethodIndex.methodObjectPatch,
MethodIndex.methodSessionCreate,
MethodIndex.methodAPEManagerAddChain,
MethodIndex.methodAPEManagerRemoveChain,
MethodIndex.methodAPEManagerListChains
];
public static string GetMethodName(MethodIndex index)
{
return index switch
{
MethodIndex.methodBalanceGet => "BalanceGet",
MethodIndex.methodContainerPut => "ContainerPut",
MethodIndex.methodContainerGet => "ContainerGet",
MethodIndex.methodContainerList => "ContainerList",
MethodIndex.methodContainerDelete => "ContainerDelete",
MethodIndex.methodEndpointInfo => "EndpointInfo",
MethodIndex.methodNetworkInfo => "NetworkInfo",
MethodIndex.methodNetMapSnapshot => "NetMapSnapshot",
MethodIndex.methodObjectPut => "ObjectPut",
MethodIndex.methodObjectDelete => "ObjectDelete",
MethodIndex.methodObjectGet => "ObjectGet",
MethodIndex.methodObjectHead => "ObjectHead",
MethodIndex.methodObjectRange => "ObjectRange",
MethodIndex.methodObjectPatch => "ObjectPatch",
MethodIndex.methodSessionCreate => "SessionCreate",
MethodIndex.methodAPEManagerAddChain => "APEManagerAddChain",
MethodIndex.methodAPEManagerRemoveChain => "APEManagerRemoveChain",
MethodIndex.methodAPEManagerListChains => "APEManagerListChains",
_ => throw new ArgumentException("Unknown method", nameof(index)),
};
}
private readonly object _lock = new();
private readonly ILogger? logger;
private int healthy;
public ClientStatusMonitor(ILogger? logger, string address)
{
this.logger = logger;
healthy = (int)HealthyStatus.Healthy;
Address = address;
Methods = new MethodStatus[MethodIndexes.Length];
for (int i = 0; i < MethodIndexes.Length; i++)
{
Methods[i] = new MethodStatus(GetMethodName(MethodIndexes[i]));
}
}
public string Address { get; }
internal uint ErrorThreshold { get; set; }
public uint CurrentErrorCount { get; set; }
public ulong OverallErrorCount { get; set; }
public MethodStatus[] Methods { get; private set; }
public bool IsHealthy()
{
var res = Interlocked.CompareExchange(ref healthy, -1, -1) == (int)HealthyStatus.Healthy;
return res;
}
public bool IsDialed()
{
return Interlocked.CompareExchange(ref healthy, -1, -1) != (int)HealthyStatus.UnhealthyOnDial;
}
public void SetHealthy()
{
Interlocked.Exchange(ref healthy, (int)HealthyStatus.Healthy);
}
public void SetUnhealthy()
{
Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnRequest);
}
public void SetUnhealthyOnDial()
{
Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnDial);
}
public void IncErrorRate()
{
bool thresholdReached;
lock (_lock)
{
CurrentErrorCount++;
OverallErrorCount++;
thresholdReached = CurrentErrorCount >= ErrorThreshold;
if (thresholdReached)
{
SetUnhealthy();
CurrentErrorCount = 0;
}
}
if (thresholdReached && logger != null)
{
FrostFsMessages.ErrorЕhresholdReached(logger, Address, ErrorThreshold);
}
}
public uint GetCurrentErrorRate()
{
lock (_lock)
{
return CurrentErrorCount;
}
}
public ulong GetOverallErrorRate()
{
lock (_lock)
{
return OverallErrorCount;
}
}
public StatusSnapshot[] MethodsStatus()
{
var result = new StatusSnapshot[Methods.Length];
for (int i = 0; i < result.Length; i++)
{
result[i] = Methods[i].Snapshot!;
}
return result;
}
}

135
src/FrostFS.SDK.Client/Pool/ClientWrapper.cs
View file

@ -1,135 +0,0 @@
using System;
using System.Threading.Tasks;
using Grpc.Core;
namespace FrostFS.SDK.Client;
// clientWrapper is used by default, alternative implementations are intended for testing purposes only.
public class ClientWrapper : ClientStatusMonitor
{
private readonly object _lock = new();
private SessionCache sessionCache;
internal ClientWrapper(WrapperPrm wrapperPrm, Pool pool) : base(wrapperPrm.Logger, wrapperPrm.Address)
{
WrapperPrm = wrapperPrm;
ErrorThreshold = wrapperPrm.ErrorThreshold;
sessionCache = pool.SessionCache;
Client = new FrostFSClient(WrapperPrm, sessionCache);
}
internal FrostFSClient? Client { get; private set; }
internal WrapperPrm WrapperPrm { get; }
internal FrostFSClient? GetClient()
{
lock (_lock)
{
if (IsHealthy())
{
return Client;
}
return null;
}
}
// dial establishes a connection to the server from the FrostFS network.
// Returns an error describing failure reason. If failed, the client
// SHOULD NOT be used.
internal async Task Dial(CallContext ctx)
{
var client = GetClient() ?? throw new FrostFsInvalidObjectException("pool client unhealthy");
await client.Dial(ctx).ConfigureAwait(false);
}
internal void HandleError(Exception ex)
{
if (ex is FrostFsResponseException responseException && responseException.Status != null)
{
switch (responseException.Status.Code)
{
case FrostFsStatusCode.Internal:
case FrostFsStatusCode.WrongMagicNumber:
case FrostFsStatusCode.SignatureVerificationFailure:
case FrostFsStatusCode.NodeUnderMaintenance:
IncErrorRate();
return;
}
}
IncErrorRate();
}
private async Task ScheduleGracefulClose()
{
if (Client == null)
return;
await Task.Delay((int)WrapperPrm.GracefulCloseOnSwitchTimeout).ConfigureAwait(false);
}
// restartIfUnhealthy checks healthy status of client and recreate it if status is unhealthy.
// Indicating if status was changed by this function call and returns error that caused unhealthy status.
internal async Task<bool> RestartIfUnhealthy(CallContext ctx)
{
bool wasHealthy;
try
{
var response = await Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false);
return false;
}
catch (RpcException)
{
wasHealthy = true;
}
// if connection is dialed before, to avoid routine/connection leak,
// pool has to close it and then initialize once again.
if (IsDialed())
{
await ScheduleGracefulClose().ConfigureAwait(false);
}
FrostFSClient? client = new(WrapperPrm, sessionCache);
var error = await client.Dial(ctx).ConfigureAwait(false);
if (!string.IsNullOrEmpty(error))
{
SetUnhealthyOnDial();
return wasHealthy;
}
lock (_lock)
{
Client = client;
}
try
{
var res = await Client.GetNodeInfoAsync(ctx).ConfigureAwait(false);
}
catch (FrostFsException)
{
SetUnhealthy();
return wasHealthy;
}
SetHealthy();
return !wasHealthy;
}
internal void IncRequests(ulong elapsed, MethodIndex method)
{
var methodStat = Methods[(int)method];
methodStat.IncRequests(elapsed);
}
}

18
src/FrostFS.SDK.Client/Pool/HealthyStatus.cs
View file

@ -1,18 +0,0 @@
namespace FrostFS.SDK.Client;
// values for healthy status of clientStatusMonitor.
public enum HealthyStatus
{
// statusUnhealthyOnDial is set when dialing to the endpoint is failed,
// so there is no connection to the endpoint, and pool should not close it
// before re-establishing connection once again.
UnhealthyOnDial,
// statusUnhealthyOnRequest is set when communication after dialing to the
// endpoint is failed due to immediate or accumulated errors, connection is
// available and pool should close it before re-establishing connection once again.
UnhealthyOnRequest,
// statusHealthy is set when connection is ready to be used by the pool.
Healthy
}

28
src/FrostFS.SDK.Client/Pool/IClientStatus.cs
View file

@ -1,28 +0,0 @@
namespace FrostFS.SDK.Client;
public interface IClientStatus
{
// isHealthy checks if the connection can handle requests.
bool IsHealthy();
// isDialed checks if the connection was created.
bool IsDialed();
// setUnhealthy marks client as unhealthy.
void SetUnhealthy();
// address return address of endpoint.
string Address { get; }
// currentErrorRate returns current errors rate.
// After specific threshold connection is considered as unhealthy.
// Pool.startRebalance routine can make this connection healthy again.
uint GetCurrentErrorRate();
// overallErrorRate returns the number of all happened errors.
ulong GetOverallErrorRate();
// methodsStatus returns statistic for all used methods.
StatusSnapshot[] MethodsStatus();
}

45
src/FrostFS.SDK.Client/Pool/InitParameters.cs
View file

@ -1,45 +0,0 @@
using System;
using System.Collections.ObjectModel;
using System.Security.Cryptography;
using Grpc.Core;
using Grpc.Core.Interceptors;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
namespace FrostFS.SDK.Client;
// InitParameters contains values used to initialize connection Pool.
public class InitParameters(Func<string, ChannelBase> grpcChannelFactory)
{
public ECDsa? Key { get; set; }
public ulong NodeDialTimeout { get; set; }
public ulong NodeStreamTimeout { get; set; }
public ulong HealthcheckTimeout { get; set; }
public ulong ClientRebalanceInterval { get; set; }
public ulong SessionExpirationDuration { get; set; }
public uint ErrorThreshold { get; set; }
public NodeParam[]? NodeParams { get; set; }
public GrpcChannelOptions[]? DialOptions { get; set; }
public Func<string, ClientWrapper>? ClientBuilder { get; set; }
public ulong GracefulCloseOnSwitchTimeout { get; set; }
public ILogger? Logger { get; set; }
public Action<CallStatistics>? Callback { get; set; }
public Collection<Interceptor> Interceptors { get; } = [];
public Func<string, ChannelBase> GrpcChannelFactory { get; set; } = grpcChannelFactory;
}

47
src/FrostFS.SDK.Client/Pool/InnerPool.cs
View file

@ -1,47 +0,0 @@
using FrostFS.SDK.Client;
internal sealed class InnerPool
{
private readonly object _lock = new();
internal InnerPool(Sampler sampler, ClientWrapper[] clients)
{
Clients = clients;
Sampler = sampler;
}
internal Sampler Sampler { get; set; }
internal ClientWrapper[] Clients { get; }
internal ClientWrapper? Connection()
{
lock (_lock)
{
if (Clients.Length == 1)
{
var client = Clients[0];
if (client.IsHealthy())
{
return client;
}
}
else
{
var attempts = 3 * Clients.Length;
for (int i = 0; i < attempts; i++)
{
int index = Sampler.Next();
if (Clients[index].IsHealthy())
{
return Clients[index];
}
}
}
return null;
}
}
}

24
src/FrostFS.SDK.Client/Pool/MethodIndex.cs
View file

@ -1,24 +0,0 @@
namespace FrostFS.SDK.Client;
public enum MethodIndex
{
methodBalanceGet,
methodContainerPut,
methodContainerGet,
methodContainerList,
methodContainerDelete,
methodEndpointInfo,
methodNetworkInfo,
methodNetMapSnapshot,
methodObjectPut,
methodObjectDelete,
methodObjectGet,
methodObjectHead,
methodObjectRange,
methodObjectPatch,
methodSessionCreate,
methodAPEManagerAddChain,
methodAPEManagerRemoveChain,
methodAPEManagerListChains,
methodLast
}

19
src/FrostFS.SDK.Client/Pool/MethodStatus.cs
View file

@ -1,19 +0,0 @@
namespace FrostFS.SDK.Client;
public class MethodStatus(string name)
{
private readonly object _lock = new();
public string Name { get; } = name;
public StatusSnapshot Snapshot { get; set; } = new StatusSnapshot();
internal void IncRequests(ulong elapsed)
{
lock (_lock)
{
Snapshot.AllTime += elapsed;
Snapshot.AllRequests++;
}
}
}

12
src/FrostFS.SDK.Client/Pool/NodeParam.cs
View file

@ -1,12 +0,0 @@
namespace FrostFS.SDK.Client;
// NodeParam groups parameters of remote node.
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>")]
public readonly struct NodeParam(int priority, string address, float weight)
{
public int Priority { get; } = priority;
public string Address { get; } = address;
public float Weight { get; } = weight;
}

12
src/FrostFS.SDK.Client/Pool/NodeStatistic.cs
View file

@ -1,12 +0,0 @@
namespace FrostFS.SDK.Client;
public class NodeStatistic
{
public string? Address { get; internal set; }
public StatusSnapshot[]? Methods { get; internal set; }
public ulong OverallErrors { get; internal set; }
public uint CurrentErrors { get; internal set; }
}

12
src/FrostFS.SDK.Client/Pool/NodesParam.cs
View file

@ -1,12 +0,0 @@
using System.Collections.ObjectModel;
namespace FrostFS.SDK.Client;
public class NodesParam(int priority)
{
public int Priority { get; } = priority;
public Collection<string> Addresses { get; } = [];
public Collection<double> Weights { get; } = [];
}

677
src/FrostFS.SDK.Client/Pool/Pool.cs
View file

@ -1,677 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using FrostFS.Refs;
using FrostFS.SDK.Client.Interfaces;
using FrostFS.SDK.Client.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
using Grpc.Core;
using Microsoft.Extensions.Logging;
namespace FrostFS.SDK.Client;
public partial class Pool : IFrostFSClient
{
const int defaultSessionTokenExpirationDuration = 100; // in epochs
const int defaultErrorThreshold = 100;
const int defaultGracefulCloseOnSwitchTimeout = 10; //Seconds;
const int defaultRebalanceInterval = 15; //Seconds;
const int defaultHealthcheckTimeout = 4; //Seconds;
const int defaultDialTimeout = 5; //Seconds;
const int defaultStreamTimeout = 10; //Seconds;
private readonly object _lock = new();
private InnerPool[]? InnerPools { get; set; }
private ClientKey Key { get; set; }
private OwnerID? _ownerId;
private FrostFsVersion _version;
private FrostFsOwner? _owner;
private FrostFsOwner Owner
{
get
{
_owner ??= new FrostFsOwner(Key.ECDsaKey.PublicKey().PublicKeyToAddress());
return _owner;
}
}
private OwnerID OwnerId
{
get
{
if (_ownerId == null)
{
_owner = Key.Owner;
_ownerId = _owner.ToMessage();
}
return _ownerId;
}
}
internal CancellationTokenSource CancellationTokenSource { get; } = new CancellationTokenSource();
internal SessionCache SessionCache { get; set; }
private ulong SessionTokenDuration { get; set; }
private RebalanceParameters RebalanceParams { get; set; }
private Func<string, ClientWrapper> ClientBuilder;
private bool disposedValue;
private ILogger? logger { get; set; }
private ulong MaxObjectSize { get; set; }
public IClientStatus? ClientStatus { get; }
// NewPool creates connection pool using parameters.
public Pool(InitParameters options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (options.Key == null)
{
throw new ArgumentException($"Missed required parameter {nameof(options.Key)}");
}
_version = new FrostFsVersion(2, 13);
var nodesParams = AdjustNodeParams(options.NodeParams);
var cache = new SessionCache(options.SessionExpirationDuration);
FillDefaultInitParams(options, this);
Key = new ClientKey(options.Key);
SessionCache = cache;
logger = options.Logger;
SessionTokenDuration = options.SessionExpirationDuration;
RebalanceParams = new RebalanceParameters(
nodesParams.ToArray(),
options.HealthcheckTimeout,
options.ClientRebalanceInterval,
options.SessionExpirationDuration);
ClientBuilder = options.ClientBuilder!;
// ClientContext.PoolErrorHandler = client.HandleError;
}
// Dial establishes a connection to the servers from the FrostFS network.
// It also starts a routine that checks the health of the nodes and
// updates the weights of the nodes for balancing.
// Returns an error describing failure reason.
//
// If failed, the Pool SHOULD NOT be used.
//
// See also InitParameters.SetClientRebalanceInterval.
public async Task<string?> Dial(CallContext ctx)
{
var inner = new InnerPool[RebalanceParams.NodesParams.Length];
bool atLeastOneHealthy = false;
int i = 0;
foreach (var nodeParams in RebalanceParams.NodesParams)
{
var wrappers = new ClientWrapper[nodeParams.Weights.Count];
for (int j = 0; j < nodeParams.Addresses.Count; j++)
{
ClientWrapper? wrapper = null;
bool dialed = false;
try
{
wrapper = wrappers[j] = ClientBuilder(nodeParams.Addresses[j]);
await wrapper.Dial(ctx).ConfigureAwait(false);
dialed = true;
var token = await InitSessionForDuration(ctx, wrapper, RebalanceParams.SessionExpirationDuration, Key.ECDsaKey, false)
.ConfigureAwait(false);
var key = FormCacheKey(nodeParams.Addresses[j], Key.PublicKey);
SessionCache.SetValue(key, token);
atLeastOneHealthy = true;
}
catch (RpcException ex)
{
if (!dialed)
wrapper!.SetUnhealthyOnDial();
else
wrapper!.SetUnhealthy();
if (logger != null)
{
FrostFsMessages.SessionCreationError(logger, wrapper!.WrapperPrm.Address, ex.Message);
}
}
catch (FrostFsInvalidObjectException)
{
break;
}
}
var sampler = new Sampler(nodeParams.Weights.ToArray());
inner[i] = new InnerPool(sampler, wrappers);
i++;
}
if (!atLeastOneHealthy)
return "At least one node must be healthy";
InnerPools = inner;
var res = await GetNetworkSettingsAsync(default).ConfigureAwait(false);
MaxObjectSize = res.MaxObjectSize;
StartRebalance(ctx);
return null;
}
private static IEnumerable<NodesParam> AdjustNodeParams(NodeParam[]? nodeParams)
{
if (nodeParams == null || nodeParams.Length == 0)
{
throw new ArgumentException("No FrostFS peers configured");
}
Dictionary<int, NodesParam> nodesParamsDict = new(nodeParams.Length);
foreach (var nodeParam in nodeParams)
{
if (!nodesParamsDict.TryGetValue(nodeParam.Priority, out var nodes))
{
nodes = new NodesParam(nodeParam.Priority);
nodesParamsDict[nodeParam.Priority] = nodes;
}
nodes.Addresses.Add(nodeParam.Address);
nodes.Weights.Add(nodeParam.Weight);
}
var nodesParams = new List<NodesParam>(nodesParamsDict.Count);
foreach (var key in nodesParamsDict.Keys)
{
var nodes = nodesParamsDict[key];
var newWeights = AdjustWeights([.. nodes.Weights]);
nodes.Weights.Clear();
foreach (var weight in newWeights)
{
nodes.Weights.Add(weight);
}
nodesParams.Add(nodes);
}
return nodesParams.OrderBy(n => n.Priority);
}
private static double[] AdjustWeights(double[] weights)
{
var adjusted = new double[weights.Length];
var sum = weights.Sum();
if (sum > 0)
{
for (int i = 0; i < weights.Length; i++)
{
adjusted[i] = weights[i] / sum;
}
}
return adjusted;
}
private static void FillDefaultInitParams(InitParameters parameters, Pool pool)
{
if (parameters.SessionExpirationDuration == 0)
parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration;
if (parameters.ErrorThreshold == 0)
parameters.ErrorThreshold = defaultErrorThreshold;
if (parameters.ClientRebalanceInterval <= 0)
parameters.ClientRebalanceInterval = defaultRebalanceInterval;
if (parameters.GracefulCloseOnSwitchTimeout <= 0)
parameters.GracefulCloseOnSwitchTimeout = defaultGracefulCloseOnSwitchTimeout;
if (parameters.HealthcheckTimeout <= 0)
parameters.HealthcheckTimeout = defaultHealthcheckTimeout;
if (parameters.NodeDialTimeout <= 0)
parameters.NodeDialTimeout = defaultDialTimeout;
if (parameters.NodeStreamTimeout <= 0)
parameters.NodeStreamTimeout = defaultStreamTimeout;
if (parameters.SessionExpirationDuration == 0)
parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration;
parameters.ClientBuilder ??= new Func<string, ClientWrapper>((address) =>
{
var wrapperPrm = new WrapperPrm()
{
Address = address,
Key = parameters.Key!,
Logger = parameters.Logger,
DialTimeout = parameters.NodeDialTimeout,
StreamTimeout = parameters.NodeStreamTimeout,
ErrorThreshold = parameters.ErrorThreshold,
GracefulCloseOnSwitchTimeout = parameters.GracefulCloseOnSwitchTimeout,
Callback = parameters.Callback,
Interceptors = parameters.Interceptors,
GrpcChannelFactory = parameters.GrpcChannelFactory
};
return new ClientWrapper(wrapperPrm, pool);
}
);
}
private ClientWrapper Connection()
{
foreach (var pool in InnerPools!)
{
var client = pool.Connection();
if (client != null)
{
return client;
}
}
throw new FrostFsException("Cannot find alive client");
}
private static async Task<FrostFsSessionToken> InitSessionForDuration(CallContext ctx, ClientWrapper cw, ulong duration, ECDsa key, bool clientCut)
{
var client = cw.Client;
var networkInfo = await client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
var epoch = networkInfo.Epoch;
ulong exp = ulong.MaxValue - epoch < duration
? ulong.MaxValue
: epoch + duration;
var prmSessionCreate = new PrmSessionCreate(exp);
return await client.CreateSessionAsync(prmSessionCreate, ctx).ConfigureAwait(false);
}
internal static string FormCacheKey(string address, string key)
{
return $"{address}{key}";
}
public void Close()
{
CancellationTokenSource.Cancel();
//if (InnerPools != null)
//{
// // close all clients
// foreach (var innerPool in InnerPools)
// foreach (var client in innerPool.Clients)
// if (client.IsDialed())
// client.Client?.Close();
//}
}
// startRebalance runs loop to monitor connection healthy status.
internal void StartRebalance(CallContext ctx)
{
var buffers = new double[RebalanceParams.NodesParams.Length][];
for (int i = 0; i < RebalanceParams.NodesParams.Length; i++)
{
var parameters = RebalanceParams.NodesParams[i];
buffers[i] = new double[parameters.Weights.Count];
Task.Run(async () =>
{
await Task.Delay((int)RebalanceParams.ClientRebalanceInterval).ConfigureAwait(false);
UpdateNodesHealth(ctx, buffers);
});
}
}
private void UpdateNodesHealth(CallContext ctx, double[][] buffers)
{
var tasks = new Task[InnerPools!.Length];
for (int i = 0; i < InnerPools.Length; i++)
{
var bufferWeights = buffers[i];
tasks[i] = Task.Run(() => UpdateInnerNodesHealth(ctx, i, bufferWeights));
}
Task.WaitAll(tasks);
}
private async ValueTask UpdateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights)
{
if (poolIndex > InnerPools!.Length - 1)
{
return;
}
var pool = InnerPools[poolIndex];
var options = RebalanceParams;
int healthyChanged = 0;
var tasks = new Task[pool.Clients.Length];
for (int j = 0; j < pool.Clients.Length; j++)
{
var client = pool.Clients[j];
var healthy = false;
string? error = null;
var changed = false;
try
{
// check timeout settings
changed = await client!.RestartIfUnhealthy(ctx).ConfigureAwait(false);
healthy = true;
bufferWeights[j] = options.NodesParams[poolIndex].Weights[j];
}
// TODO: specify
catch (FrostFsException e)
{
error = e.Message;
bufferWeights[j] = 0;
SessionCache.DeleteByPrefix(client.Address);
}
if (changed)
{
if (!string.IsNullOrEmpty(error))
{
if (logger != null)
{
FrostFsMessages.HealthChanged(logger, client.Address, healthy, error!);
}
Interlocked.Exchange(ref healthyChanged, 1);
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
if (Interlocked.CompareExchange(ref healthyChanged, -1, -1) == 1)
{
var probabilities = AdjustWeights(bufferWeights);
lock (_lock)
{
pool.Sampler = new Sampler(probabilities);
}
}
}
}
// TODO: remove
private bool CheckSessionTokenErr(Exception error, string address)
{
if (error == null)
{
return false;
}
if (error is SessionNotFoundException || error is SessionExpiredException)
{
this.SessionCache.DeleteByPrefix(address);
return true;
}
return false;
}
public Statistic Statistic()
{
if (InnerPools == null)
{
throw new FrostFsInvalidObjectException(nameof(Pool));
}
var statistics = new Statistic();
foreach (var inner in InnerPools)
{
int valueIndex = 0;
var nodes = new string[inner.Clients.Length];
lock (_lock)
{
foreach (var client in inner.Clients)
{
if (client.IsHealthy())
{
nodes[valueIndex] = client.Address;
}
var node = new NodeStatistic
{
Address = client.Address,
Methods = client.MethodsStatus(),
OverallErrors = client.GetOverallErrorRate(),
CurrentErrors = client.GetCurrentErrorRate()
};
statistics.Nodes.Add(node);
valueIndex++;
statistics.OverallErrors += node.OverallErrors;
}
if (statistics.CurrentNodes == null || statistics.CurrentNodes.Length == 0)
{
statistics.CurrentNodes = nodes;
}
}
}
return statistics;
}
public async Task<FrostFsNetmapSnapshot> GetNetmapSnapshotAsync(CallContext ctx)
{
var client = Connection();
return await client.Client!.GetNetmapSnapshotAsync(ctx).ConfigureAwait(false);
}
public async Task<FrostFsNodeInfo> GetNodeInfoAsync(CallContext ctx)
{
var client = Connection();
return await client.Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false);
}
public async Task<NetworkSettings> GetNetworkSettingsAsync(CallContext ctx)
{
var client = Connection();
return await client.Client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
}
public async Task<FrostFsSessionToken> CreateSessionAsync(PrmSessionCreate args, CallContext ctx)
{
var client = Connection();
return await client.Client!.CreateSessionAsync(args, ctx).ConfigureAwait(false);
}
public async Task<ReadOnlyMemory<byte>> AddChainAsync(PrmApeChainAdd args, CallContext ctx)
{
var client = Connection();
return await client.Client!.AddChainAsync(args, ctx).ConfigureAwait(false);
}
public async Task RemoveChainAsync(PrmApeChainRemove args, CallContext ctx)
{
var client = Connection();
await client.Client!.RemoveChainAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsChain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
{
var client = Connection();
return await client.Client!.ListChainAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsContainerInfo> GetContainerAsync(PrmContainerGet args, CallContext ctx)
{
var client = Connection();
return await client.Client!.GetContainerAsync(args, ctx).ConfigureAwait(false);
}
public IAsyncEnumerable<FrostFsContainerId> ListContainersAsync(PrmContainerGetAll args, CallContext ctx)
{
var client = Connection();
return client.Client!.ListContainersAsync(args, ctx);
}
[Obsolete("Use PutContainerAsync method")]
public async Task<FrostFsContainerId> CreateContainerAsync(PrmContainerCreate args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsContainerId> PutContainerAsync(PrmContainerCreate args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false);
}
public async Task DeleteContainerAsync(PrmContainerDelete args, CallContext ctx)
{
var client = Connection();
await client.Client!.DeleteContainerAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsHeaderResult> GetObjectHeadAsync(PrmObjectHeadGet args, CallContext ctx)
{
var client = Connection();
return await client.Client!.GetObjectHeadAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsObject> GetObjectAsync(PrmObjectGet args, CallContext ctx)
{
var client = Connection();
return await client.Client!.GetObjectAsync(args, ctx).ConfigureAwait(false);
}
public async Task<IObjectWriter> PutObjectAsync(PrmObjectPut args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PutObjectAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PutClientCutObjectAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsObjectId> PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PutSingleObjectAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsObjectId> PatchObjectAsync(PrmObjectPatch args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false);
}
public async Task<RangeReader> GetRangeAsync(PrmRangeGet args, CallContext ctx)
{
var client = Connection();
return await client.Client!.GetRangeAsync(args, ctx).ConfigureAwait(false);
}
public async Task<ReadOnlyMemory<byte>[]> GetRangeHashAsync(PrmRangeHashGet args, CallContext ctx)
{
var client = Connection();
return await client.Client!.GetRangeHashAsync(args, ctx).ConfigureAwait(false);
}
public async Task<FrostFsObjectId> PatchAsync(PrmObjectPatch args, CallContext ctx)
{
var client = Connection();
return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false);
}
public async Task DeleteObjectAsync(PrmObjectDelete args, CallContext ctx)
{
var client = Connection();
await client.Client!.DeleteObjectAsync(args, ctx).ConfigureAwait(false);
}
public IAsyncEnumerable<FrostFsObjectId> SearchObjectsAsync(PrmObjectSearch args, CallContext ctx)
{
var client = Connection();
return client.Client!.SearchObjectsAsync(args, ctx);
}
public async Task<Accounting.Decimal> GetBalanceAsync(CallContext ctx)
{
var client = Connection();
return await client.Client!.GetBalanceAsync(ctx).ConfigureAwait(false);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
Close();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
}
}

16
src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs
View file

@ -1,16 +0,0 @@
namespace FrostFS.SDK.Client;
public class RebalanceParameters(
NodesParam[] nodesParams,
ulong nodeRequestTimeout,
ulong clientRebalanceInterval,
ulong sessionExpirationDuration)
{
public NodesParam[] NodesParams { get; set; } = nodesParams;
public ulong NodeRequestTimeout { get; set; } = nodeRequestTimeout;
public ulong ClientRebalanceInterval { get; set; } = clientRebalanceInterval;
public ulong SessionExpirationDuration { get; set; } = sessionExpirationDuration;
}

14
src/FrostFS.SDK.Client/Pool/RequestInfo.cs
View file

@ -1,14 +0,0 @@
using System;
namespace FrostFS.SDK.Client;
// RequestInfo groups info about pool request.
struct RequestInfo
{
public string Address { get; set; }
public MethodIndex MethodIndex { get; set; }
public TimeSpan Elapsed { get; set; }
}

85
src/FrostFS.SDK.Client/Pool/Sampler.cs
View file

@ -1,85 +0,0 @@
using System;
namespace FrostFS.SDK.Client;
internal sealed class Sampler
{
private readonly object _lock = new();
private Random random = new();
internal double[] Probabilities { get; set; }
internal int[] Alias { get; set; }
internal Sampler(double[] probabilities)
{
var small = new WorkList();
var large = new WorkList();
var n = probabilities.Length;
// sampler.randomGenerator = rand.New(source)
Probabilities = new double[n];
Alias = new int[n];
// Compute scaled probabilities.
var p = new double[n];
for (int i = 0; i < n; i++)
{
p[i] = probabilities[i] * n;
if (p[i] < 1)
small.Add(i);
else
large.Add(i);
}
while (small.Length > 0 && large.Length > 0)
{
var l = small.Remove();
var g = large.Remove();
Probabilities[l] = p[l];
Alias[l] = g;
p[g] = p[g] + p[l] - 1;
if (p[g] < 1)
small.Add(g);
else
large.Add(g);
}
while (large.Length > 0)
{
var g = large.Remove();
Probabilities[g] = 1;
}
while (small.Length > 0)
{
var l = small.Remove();
probabilities[l] = 1;
}
}
internal int Next()
{
var n = Alias.Length;
int i;
double f;
lock (_lock)
{
i = random.Next(0, n - 1);
f = random.NextDouble();
}
if (f < Probabilities[i])
{
return i;
}
return Alias[i];
}
}

12
src/FrostFS.SDK.Client/Pool/Statistic.cs
View file

@ -1,12 +0,0 @@
using System.Collections.ObjectModel;
namespace FrostFS.SDK.Client;
public sealed class Statistic
{
public ulong OverallErrors { get; internal set; }
public Collection<NodeStatistic> Nodes { get; } = [];
public string[]? CurrentNodes { get; internal set; }
}

8
src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs
View file

@ -1,8 +0,0 @@
namespace FrostFS.SDK.Client;
public class StatusSnapshot()
{
public ulong AllTime { get; internal set; }
public ulong AllRequests { get; internal set; }
}

26
src/FrostFS.SDK.Client/Pool/WorkList.cs
View file

@ -1,26 +0,0 @@
using System.Collections.Generic;
using System.Linq;
namespace FrostFS.SDK.Client;
internal sealed class WorkList
{
private readonly List<int> elements = [];
internal int Length
{
get { return elements.Count; }
}
internal void Add(int element)
{
elements.Add(element);
}
internal int Remove()
{
int last = elements.LastOrDefault();
elements.RemoveAt(elements.Count - 1);
return last;
}
}

39
src/FrostFS.SDK.Client/Pool/WrapperPrm.cs
View file

@ -1,39 +0,0 @@
using System;
using System.Collections.ObjectModel;
using System.Security.Cryptography;
using Grpc.Core;
using Grpc.Core.Interceptors;
using Microsoft.Extensions.Logging;
namespace FrostFS.SDK.Client;
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>")]
public struct WrapperPrm
{
internal ILogger? Logger { get; set; }
internal string Address { get; set; }
internal ECDsa Key { get; set; }
internal ulong DialTimeout { get; set; }
internal ulong StreamTimeout { get; set; }
internal uint ErrorThreshold { get; set; }
internal Action ResponseInfoCallback { get; set; }
internal Action PoolRequestInfoCallback { get; set; }
internal Func<string, ChannelBase> GrpcChannelFactory { get; set; }
internal ulong GracefulCloseOnSwitchTimeout { get; set; }
internal Action<CallStatistics>? Callback { get; set; }
internal Collection<Interceptor>? Interceptors { get; set; }
}

2
src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs
View file

@ -20,8 +20,6 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
{
var binary = RuleSerializer.Serialize(args.Chain);
var base64 = Convert.ToBase64String(binary);
AddChainRequest request = new()
{
Body = new()

7
src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs
View file

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Threading.Tasks;
using FrostFS.Container;
@ -83,7 +82,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
Body = new PutRequest.Types.Body
{
Container = grpcContainer,
Signature = ClientContext.Key.ECDsaKey.SignRFC6979(grpcContainer)
Signature = ClientContext.Key.SignRFC6979(grpcContainer)
}
};
@ -113,8 +112,8 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
{
Body = new DeleteRequest.Types.Body
{
ContainerId = args.ContainerId.ToMessage(),
Signature = ClientContext.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value)
ContainerId = args.ContainerId.GetContainerID(),
Signature = ClientContext.Key.SignRFC6979(args.ContainerId.GetContainerID().Value)
}
};

25
src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs
View file

@ -96,7 +96,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
Address = new Address
{
ContainerId = args.ContainerId.ToMessage(),
ContainerId = args.ContainerId.GetContainerID(),
ObjectId = args.ObjectId.ToMessage()
}
}
@ -124,7 +124,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
Address = new Address
{
ContainerId = args.ContainerId.ToMessage(),
ContainerId = args.ContainerId.GetContainerID(),
ObjectId = args.ObjectId.ToMessage()
},
Range = new Object.Range
@ -159,7 +159,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
Address = new Address
{
ContainerId = args.ContainerId.ToMessage(),
ContainerId = args.ContainerId.GetContainerID(),
ObjectId = args.ObjectId.ToMessage()
},
Type = ChecksumType.Sha256,
@ -204,7 +204,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
Address = new Address
{
ContainerId = args.ContainerId.ToMessage(),
ContainerId = args.ContainerId.GetContainerID(),
ObjectId = args.ObjectId.ToMessage()
}
}
@ -231,7 +231,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
Body = new SearchRequest.Types.Body
{
ContainerId = args.ContainerId.ToMessage(),
ContainerId = args.ContainerId.GetContainerID(),
Version = 1 // TODO: clarify this param
}
};
@ -296,18 +296,17 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
var chunkSize = args.MaxChunkLength;
Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null");
var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken);
byte[]? chunkBuffer = null;
try
{
// common
chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
bool isFirstChunk = true;
ulong currentPos = args.Range.Offset;
var address = new Address
{
ObjectId = args.Address.ObjectId,
@ -327,11 +326,11 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
{
Body = new()
{
Address = address,
Address = address,
Patch = new PatchRequest.Types.Body.Types.Patch
{
Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)),
SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount }
SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount }
}
}
};
@ -385,7 +384,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
return response.Body.ObjectId.ToModel();
}
internal async Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
{
var stream = args.Payload!;
@ -567,7 +566,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
break;
sentBytes += bytesCount;
var chunkRequest = new PutRequest
{
Body = new PutRequest.Types.Body
@ -622,7 +621,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
var initRequest = new PutRequest
{
Body = new PutRequest.Types.Body
{
{
Init = new PutRequest.Types.Body.Types.Init
{
Header = grpcHeader,

14
src/FrostFS.SDK.Client/Pool/SessionCache.cs → src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs
View file

@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Concurrent;
namespace FrostFS.SDK.Client;
@ -36,15 +35,4 @@ internal sealed class SessionCache(ulong sessionExpirationDuration)
_cache[key] = value;
}
}
internal void DeleteByPrefix(string prefix)
{
foreach (var key in _cache.Keys)
{
if (key.StartsWith(prefix, StringComparison.Ordinal))
{
_cache.TryRemove(key, out var _);
}
}
}
}

2
src/FrostFS.SDK.Client/Tools/ClientContext.cs
View file

@ -39,7 +39,7 @@ public class ClientContext(FrostFSClient client, ClientKey key, FrostFsOwner own
{
if (sessionKey == null && Key != null && Address != null)
{
sessionKey = Pool.FormCacheKey(Address, Key.PublicKey);
sessionKey = $"{Address}{Key}";
}
return sessionKey;

25
src/FrostFS.SDK.Client/Tools/ObjectTools.cs
View file

@ -1,6 +1,6 @@
using System;
using System.Linq;
using System.Security.Cryptography;
using FrostFS.Object;
using FrostFS.Refs;
using FrostFS.SDK.Client.Mappers.GRPC;
@ -44,7 +44,11 @@ public static class ObjectTools
if (header.Split != null)
SetSplitValues(grpcHeader, header.Split, owner, version, key);
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
using var sha256 = SHA256.Create();
using HashStream stream = new(sha256);
grpcHeader.WriteTo(stream);
return new FrostFsObjectId(Base58.Encode(stream.Hash()));
}
internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx)
@ -75,7 +79,7 @@ public static class ObjectTools
var obj = new Object.Object
{
Header = grpcHeader,
ObjectId = new ObjectID { Value = grpcHeader.Sha256() },
ObjectId = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcHeader.Sha256()) },
Payload = UnsafeByteOperations.UnsafeWrap(@object.SingleObjectPayload)
};
@ -117,9 +121,9 @@ public static class ObjectTools
if (split.ParentHeader is not null)
{
var grpcParentHeader = CreateHeader(split.ParentHeader, Array.Empty<byte>().Sha256(), owner, version);
var grpcParentHeader = CreateHeader(split.ParentHeader, DataHasher.Sha256([]), owner, version);
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
grpcHeader.Split.Parent = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcParentHeader.Sha256()) };
grpcHeader.Split.ParentHeader = grpcParentHeader;
grpcHeader.Split.ParentSignature = new Signature
{
@ -147,21 +151,12 @@ public static class ObjectTools
return grpcHeader;
}
internal static Checksum Sha256Checksum(byte[] data)
{
return new Checksum
{
Type = ChecksumType.Sha256,
Sum = ByteString.CopyFrom(data.Sha256())
};
}
internal static Checksum Sha256Checksum(ReadOnlyMemory<byte> data)
{
return new Checksum
{
Type = ChecksumType.Sha256,
Sum = ByteString.CopyFrom(data.Sha256())
Sum = UnsafeByteOperations.UnsafeWrap(DataHasher.Sha256(data))
};
}

22
src/FrostFS.SDK.Client/Tools/RequestSigner.cs
View file

@ -33,6 +33,7 @@ public static class RequestSigner
var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N);
var privateKey = new ECPrivateKeyParameters(new BigInteger(1, key.PrivateKey()), ecParameters);
var signer = new ECDsaSigner(new HMacDsaKCalculator(digest));
var hash = new byte[digest.GetDigestSize()];
digest.BlockUpdate(data, 0, data.Length);
@ -54,21 +55,21 @@ public static class RequestSigner
return ByteString.CopyFrom(signature);
}
internal static SignatureRFC6979 SignRFC6979(this ECDsa key, IMessage message)
internal static SignatureRFC6979 SignRFC6979(this ClientKey key, IMessage message)
{
return new SignatureRFC6979
{
Key = ByteString.CopyFrom(key.PublicKey()),
Sign = key.SignRFC6979(message.ToByteArray()),
Key = key.PublicKeyProto,
Sign = key.ECDsaKey.SignRFC6979(message.ToByteArray()),
};
}
internal static SignatureRFC6979 SignRFC6979(this ECDsa key, ByteString data)
internal static SignatureRFC6979 SignRFC6979(this ClientKey key, ByteString data)
{
return new SignatureRFC6979
{
Key = ByteString.CopyFrom(key.PublicKey()),
Sign = key.SignRFC6979(data.ToByteArray()),
Key = key.PublicKeyProto,
Sign = key.ECDsaKey.SignRFC6979(data.ToByteArray()),
};
}
@ -82,11 +83,11 @@ public static class RequestSigner
Span<byte> result = stackalloc byte[65];
result[0] = 0x04;
key.SignHash(data.Sha512()).AsSpan().CopyTo(result.Slice(1));
key.SignHash(DataHasher.Sha512(data)).AsSpan().CopyTo(result.Slice(1));
return ByteString.CopyFrom(result);
}
public static ByteString SignDataByHash(this ECDsa key, byte[] hash)
{
if (key is null)
@ -112,8 +113,9 @@ public static class RequestSigner
Sign = key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty),
};
}
using HashStream stream = new();
using var sha512 = SHA512.Create();
using HashStream stream = new(sha512);
data.WriteTo(stream);
var sig = new Signature

66
src/FrostFS.SDK.Client/Tools/Verifier.cs
View file

@ -9,61 +9,13 @@ using FrostFS.Session;
using Google.Protobuf;
using Org.BouncyCastle.Asn1.Sec;
using Org.BouncyCastle.Crypto.Digests;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.Crypto.Signers;
using Org.BouncyCastle.Math;
namespace FrostFS.SDK.Client;
public static class Verifier
{
public const int RFC6979SignatureSize = 64;
private static BigInteger[] DecodeSignature(byte[] sig)
{
if (sig.Length != RFC6979SignatureSize)
throw new FormatException($"Wrong signature size, expect={RFC6979SignatureSize}, actual={sig.Length}");
var rs = new BigInteger[2];
rs[0] = new BigInteger(1, sig.AsSpan(0, 32).ToArray());
rs[1] = new BigInteger(1, sig.AsSpan(32).ToArray());
return rs;
}
public static bool VerifyRFC6979(this byte[] publicKey, byte[] data, byte[] sig)
{
if (publicKey is null || data is null || sig is null)
return false;
var rs = DecodeSignature(sig);
var digest = new Sha256Digest();
var signer = new ECDsaSigner(new HMacDsaKCalculator(digest));
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N);
var bcPublicKey = new ECPublicKeyParameters(secp256R1.Curve.DecodePoint(publicKey), ecParameters);
var hash = new byte[digest.GetDigestSize()];
digest.BlockUpdate(data, 0, data.Length);
digest.DoFinal(hash, 0);
signer.Init(false, bcPublicKey);
return signer.VerifySignature(hash, rs[0], rs[1]);
}
public static bool VerifyRFC6979(this SignatureRFC6979 signature, IMessage message)
{
if (signature is null)
{
throw new ArgumentNullException(nameof(signature));
}
return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray());
}
public static bool VerifyData(this ECDsa key, ReadOnlyMemory<byte> data, byte[] sig)
public static bool VerifyData(this ECDsa key, IMessage data, ByteString sig)
{
if (key is null)
throw new ArgumentNullException(nameof(key));
@ -71,7 +23,18 @@ public static class Verifier
if (sig is null)
throw new ArgumentNullException(nameof(sig));
return key.VerifyHash(data.Sha512(), sig.AsSpan(1).ToArray());
var signature = sig.Span.Slice(1).ToArray();
using var sha = SHA512.Create();
if (data is null)
{
return key.VerifyHash(DataHasher.Sha512(new Span<byte>([])), signature);
}
using var stream = new HashStream(sha);
data.WriteTo(stream);
return key.VerifyHash(stream.Hash(), signature);
}
public static bool VerifyMessagePart(this Signature sig, IMessage data)
@ -80,9 +43,8 @@ public static class Verifier
return false;
using var key = sig.Key.ToByteArray().LoadPublicKey();
var data2Verify = data is null ? [] : data.ToByteArray();
return key.VerifyData(data2Verify, sig.Sign.ToByteArray());
return key.VerifyData(data, sig.Sign);
}
internal static bool VerifyMatryoskaLevel(IMessage body, IMetaHeader meta, IVerificationHeader verification)

14
src/FrostFS.SDK.Cryptography/ArrayHelper.cs
View file

@ -21,4 +21,18 @@ internal static class ArrayHelper
return dst;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void GetRevertedArray(ReadOnlySpan<byte> source, byte[] data)
{
if (source.Length != 0)
{
int i = 0;
int j = source.Length - 1;
while (i < source.Length)
{
data[i++] = source[j--];
}
}
}
}

4
src/FrostFS.SDK.Cryptography/AssemblyInfo.cs
View file

@ -1,8 +1,8 @@
using System.Reflection;
[assembly: AssemblyCompany("FrostFS.SDK.Cryptography")]
[assembly: AssemblyFileVersion("1.0.2.0")]
[assembly: AssemblyFileVersion("1.0.4.0")]
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
[assembly: AssemblyProduct("FrostFS.SDK.Cryptography")]
[assembly: AssemblyTitle("FrostFS.SDK.Cryptography")]
[assembly: AssemblyVersion("1.0.3.0")]
[assembly: AssemblyVersion("1.0.4.0")]

32
src/FrostFS.SDK.Cryptography/Base58.cs
View file

@ -19,19 +19,20 @@ public static class Base58
if (buffer.Length < 4)
throw new FormatException();
var check = buffer.AsSpan(0, buffer.Length - 4).ToArray();
byte[] checksum = check.Sha256().Sha256();
var check = buffer.AsSpan(0, buffer.Length - 4);
byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(check).AsSpan());
if (!buffer.AsSpan(buffer.Length - 4).SequenceEqual(checksum.AsSpan(0, 4)))
throw new FormatException();
var result = check.ToArray();
Array.Clear(buffer, 0, buffer.Length);
return check;
return result;
}
public static string Base58CheckEncode(this ReadOnlySpan<byte> data)
public static string Base58CheckEncode(this Span<byte> data)
{
byte[] checksum = data.ToArray().Sha256().Sha256();
byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan()); ;
Span<byte> buffer = stackalloc byte[data.Length + 4];
data.CopyTo(buffer);
@ -59,10 +60,9 @@ public static class Base58
}
int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count();
var leadingZeros = new byte[leadingZeroCount];
if (bi.IsZero)
return leadingZeros;
return new byte[leadingZeroCount];
var bytesBigEndian = bi.ToByteArray().Reverse().ToArray();
@ -73,12 +73,26 @@ public static class Base58
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();
return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros);
var result = new byte[leadingZeroCount + bytesBigEndian.Length - firstNonZeroIndex];
int p = 0;
while (p < leadingZeroCount)
result[p++] = 0;
for (int j = firstNonZeroIndex; j < bytesBigEndian.Length; j++)
result[p++] = bytesBigEndian[j];
return result;
}
public static string Encode(ReadOnlySpan<byte> input)
{
var data = input.ToArray().Reverse().Concat(new byte[] { 0 }).ToArray();
var data = new byte[input.Length + 1];
ArrayHelper.GetRevertedArray(input, data);
data[input.Length] = 0;
BigInteger value = new(data);
// Encode BigInteger to Base58 string

121
src/FrostFS.SDK.Cryptography/DataHasher.cs Normal file
View file

@ -0,0 +1,121 @@
using System;
using System.Buffers;
using System.Security.Cryptography;
namespace FrostFS.SDK.Cryptography;
public static class DataHasher
{
private const int LargeBlockSize = 1024 * 1024;
private const int SmallBlockSize = 256;
public static byte[] Hash(this ReadOnlyMemory<byte> bytes, HashAlgorithm algorithm)
{
if (algorithm is null)
{
throw new ArgumentNullException(nameof(algorithm));
}
if (bytes.Length == 0)
{
return algorithm.ComputeHash([]);
}
int rest, pos = 0;
var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize;
byte[] buffer = ArrayPool<byte>.Shared.Rent(blockSize);
try
{
while ((rest = bytes.Length - pos) > 0)
{
var size = Math.Min(rest, blockSize);
bytes.Slice(pos, size).CopyTo(buffer);
algorithm.TransformBlock(buffer, 0, size, buffer, 0);
pos += size;
}
algorithm.TransformFinalBlock([], 0, 0);
return algorithm.Hash;
}
finally
{
if (buffer != null)
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
}
public static byte[] Hash(this ReadOnlySpan<byte> bytes, HashAlgorithm algorithm)
{
if (algorithm is null)
{
throw new ArgumentNullException(nameof(algorithm));
}
if (bytes.Length == 0)
{
return algorithm.ComputeHash([]);
}
int rest, pos = 0;
var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize;
byte[] buffer = ArrayPool<byte>.Shared.Rent(blockSize);
try
{
while ((rest = bytes.Length - pos) > 0)
{
var size = Math.Min(rest, blockSize);
bytes.Slice(pos, size).CopyTo(buffer);
algorithm.TransformBlock(buffer, 0, size, buffer, 0);
pos += size;
}
algorithm.TransformFinalBlock([], 0, 0);
return algorithm.Hash;
}
finally
{
if (buffer != null)
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
}
public static byte[] Sha256(ReadOnlyMemory<byte> value)
{
using SHA256 sha = SHA256.Create();
return Hash(value, sha);
}
public static byte[] Sha256(ReadOnlySpan<byte> value)
{
using SHA256 sha = SHA256.Create();
return Hash(value, sha);
}
public static byte[] Sha512(ReadOnlyMemory<byte> value)
{
using SHA512 sha = SHA512.Create();
return Hash(value, sha);
}
public static byte[] Sha512(ReadOnlySpan<byte> value)
{
using SHA512 sha = SHA512.Create();
return Hash(value, sha);
}
}

79
src/FrostFS.SDK.Cryptography/Extentions.cs
View file

@ -1,20 +1,9 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using CommunityToolkit.HighPerformance;
using Org.BouncyCastle.Crypto.Digests;
namespace FrostFS.SDK.Cryptography;
public static class Extentions
{
private static readonly SHA256 _sha256 = SHA256.Create();
private static SpinLock _spinlockSha256;
private static readonly SHA512 _sha512 = SHA512.Create();
private static SpinLock _spinlockSha512;
internal static byte[] RIPEMD160(this byte[] value)
{
var hash = new byte[20];
@ -24,72 +13,4 @@ public static class Extentions
digest.DoFinal(hash, 0);
return hash;
}
public static byte[] Sha256(this byte[] value)
{
bool lockTaken = false;
try
{
_spinlockSha256.Enter(ref lockTaken);
return _sha256.ComputeHash(value);
}
finally
{
if (lockTaken)
{
_spinlockSha256.Exit(false);
}
}
}
public static byte[] Sha256(this ReadOnlyMemory<byte> value)
{
bool lockTaken = false;
try
{
_spinlockSha256.Enter(ref lockTaken);
return _sha256.ComputeHash(value.AsStream());
}
finally
{
if (lockTaken)
{
_spinlockSha256.Exit(false);
}
}
}
public static byte[] Sha512(this ReadOnlyMemory<byte> value)
{
bool lockTaken = false;
try
{
_spinlockSha512.Enter(ref lockTaken);
return _sha512.ComputeHash(value.AsStream());
}
finally
{
if (lockTaken)
_spinlockSha512.Exit(false);
}
}
public static byte[] Sha512(this Stream stream)
{
bool lockTaken = false;
try
{
_spinlockSha512.Enter(ref lockTaken);
return _sha512.ComputeHash(stream);
}
finally
{
if (lockTaken)
_spinlockSha512.Exit(false);
}
}
}

1
src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj
View file

@ -30,7 +30,6 @@
<ItemGroup>
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
<PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

4
src/FrostFS.SDK.Cryptography/HashStream.cs
View file

@ -3,11 +3,11 @@ using System.Security.Cryptography;
namespace FrostFS.SDK.Cryptography;
public sealed class HashStream() : Stream
public sealed class HashStream(HashAlgorithm algorithm) : Stream
{
private long position;
private readonly SHA512 _hash = SHA512.Create();
private readonly HashAlgorithm _hash = algorithm;
public override bool CanRead => false;

36
src/FrostFS.SDK.Cryptography/Key.cs
View file

@ -16,7 +16,7 @@ public static class KeyExtension
private const int UncompressedPublicKeyLength = 65;
private static readonly uint CheckSigDescriptor =
BinaryPrimitives.ReadUInt32LittleEndian(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").Sha256());
BinaryPrimitives.ReadUInt32LittleEndian(DataHasher.Sha256(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").AsSpan()));
public static byte[] Compress(this byte[] publicKey)
{
@ -60,10 +60,15 @@ public static class KeyExtension
$"expected length={CompressedPublicKeyLength}, actual={publicKey.Length}"
);
var script = new byte[] { 0x0c, CompressedPublicKeyLength }; //PUSHDATA1 33
script = ArrayHelper.Concat(script, publicKey);
script = ArrayHelper.Concat(script, [0x41]); //SYSCALL
script = ArrayHelper.Concat(script, BitConverter.GetBytes(CheckSigDescriptor)); //Neo_Crypto_CheckSig
var signDescriptor = BitConverter.GetBytes(CheckSigDescriptor);
var script = new byte[3 + publicKey.Length + signDescriptor.Length];
script[0] = 0x0c;
script[1] = CompressedPublicKeyLength; //PUSHDATA1 33
Buffer.BlockCopy(publicKey, 0, script, 2, publicKey.Length);
script[publicKey.Length + 2] = 0x41; //SYSCALL
Buffer.BlockCopy(signDescriptor, 0, script, publicKey.Length + 3, signDescriptor.Length);
return script;
}
@ -74,7 +79,7 @@ public static class KeyExtension
throw new ArgumentNullException(nameof(publicKey));
var script = publicKey.CreateSignatureRedeemScript();
return script.Sha256().RIPEMD160();
return DataHasher.Sha256(script.AsSpan()).RIPEMD160();
}
private static string ToAddress(this byte[] scriptHash, byte version)
@ -102,11 +107,6 @@ public static class KeyExtension
return privateKey;
}
public static string Address(this ECDsa key)
{
return key.PublicKey().PublicKeyToAddress();
}
public static string PublicKeyToAddress(this byte[] publicKey)
{
if (publicKey == null)
@ -132,10 +132,12 @@ public static class KeyExtension
var pos = 33 - param.Q.X.Length;
param.Q.X.CopyTo(pubkey, pos);
if (new BigInteger(param.Q.Y.Reverse().Concat(new byte[] { 0x0 }).ToArray()).IsEven)
pubkey[0] = 0x2;
else
pubkey[0] = 0x3;
var y = new byte[33];
ArrayHelper.GetRevertedArray(param.Q.Y, y);
y[32] = 0;
pubkey[0] = new BigInteger(y).IsEven ? (byte)0x2 : (byte)0x3;
return pubkey;
}
@ -152,7 +154,9 @@ public static class KeyExtension
{
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
.GetEncoded(false).Skip(1).ToArray();
.GetEncoded(false)
.Skip(1)
.ToArray();
var key = ECDsa.Create(new ECParameters
{

4
src/FrostFS.SDK.Protos/AssemblyInfo.cs
View file

@ -1,8 +1,8 @@
using System.Reflection;
[assembly: AssemblyCompany("FrostFS.SDK.Protos")]
[assembly: AssemblyFileVersion("1.0.2.0")]
[assembly: AssemblyFileVersion("1.0.4.0")]
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
[assembly: AssemblyProduct("FrostFS.SDK.Protos")]
[assembly: AssemblyTitle("FrostFS.SDK.Protos")]
[assembly: AssemblyVersion("1.0.3.0")]
[assembly: AssemblyVersion("1.0.4.0")]

3
src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs
View file

@ -3,7 +3,6 @@ using System.Security.Cryptography;
using FrostFS.Object;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
using FrostFS.Session;
using Google.Protobuf;
@ -43,7 +42,7 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader)
Signature = new Refs.Signature
{
Key = Key.PublicKeyProto,
Sign = Key.ECDsaKey. SignData(header.ToByteArray()),
Sign = Key.ECDsaKey.SignData(header.ToByteArray()),
}
}
},

2
src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs
View file

@ -1,5 +1,3 @@
using System.Security.Cryptography;
using FrostFS.Container;
using FrostFS.Object;
using FrostFS.SDK.Client;

7
src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs
View file

@ -23,13 +23,16 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
var grpcVersion = Version.ToMessage();
Span<byte> ContainerGuidSpan = stackalloc byte[16];
ContainerGuid.ToBytes(ContainerGuidSpan);
PutResponse putResponse = new()
{
Body = new PutResponse.Types.Body
{
ContainerId = new ContainerID
{
Value = ByteString.CopyFrom(ContainerGuid.ToBytes())
Value = ByteString.CopyFrom(ContainerGuidSpan)
}
},
MetaHeader = new ResponseMetaHeader
@ -69,7 +72,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
Container = new Container.Container
{
Version = grpcVersion,
Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()),
Nonce = ByteString.CopyFrom(ContainerGuidSpan),
PlacementPolicy = PlacementPolicy.GetPolicy()
}
},

1
src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs
View file

@ -4,7 +4,6 @@ using System.Security.Cryptography;
using FrostFS.Object;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
using Google.Protobuf;

41
src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs
View file

@ -40,9 +40,9 @@ public class ContainerTests : SmokeTestsBase
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
await Cleanup(client);
client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
FrostFsContainerId containerId = await CreateContainer(client,
ctx: default,
token: null,
@ -50,22 +50,25 @@ public class ContainerTests : SmokeTestsBase
backupFactor: 1,
selectors: [],
filter: [],
containerAttributes: [new ("testKey1", "testValue1")],
containerAttributes: [new("testKey1", "testValue1")],
new FrostFsReplica(3));
Assert.NotNull(containerId);
var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default);
Assert.NotNull(container);
Assert.NotNull(container.Attributes);
Assert.Equal(2, container.Attributes.Count);
Assert.Equal("testKey1", container.Attributes[0].Key);
Assert.Equal("testValue1", container.Attributes[0].Value);
Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key);
Assert.Equal("true", container.Attributes[1].Value);
//Assert.Equal("true", container.Attributes[1].Value);
// for cluster
//Assert.Equal(2, container.Attributes.Count);
//Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key);
Assert.True(container.PlacementPolicy.HasValue);
Assert.Equal(1u, container.PlacementPolicy.Value.BackupFactor);
@ -82,7 +85,7 @@ public class ContainerTests : SmokeTestsBase
Assert.Equal(OwnerId!.ToString(), container.Owner!.Value);
Assert.NotNull(container.Version);
Assert.Equal(Version!.Major, container.Version.Major);
Assert.Equal(Version.Minor, container.Version.Minor);
}
@ -101,8 +104,8 @@ public class ContainerTests : SmokeTestsBase
new ("filter2", "filterKey2", 2, "testValue2", [new ("subFilter2", "subFilterKey2", 3, "testValue3",[])])
];
Collection<FrostFsSelector> selectors = [
new ("selector1") {
Collection<FrostFsSelector> selectors = [
new ("selector1") {
Count = 1,
Clause = 1,
Attribute = "attribute1",
@ -129,19 +132,19 @@ public class ContainerTests : SmokeTestsBase
Assert.NotNull(containerId);
var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default);
Assert.NotNull(container);
Assert.NotNull(container.Attributes);
Assert.Single(container.Attributes);
Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key);
Assert.Equal("true", container.Attributes[0].Value);
//Assert.Single(container.Attributes);
//Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key);
//Assert.Equal("true", container.Attributes[0].Value);
Assert.True(container.PlacementPolicy.HasValue);
Assert.Equal(2u, container.PlacementPolicy.Value.BackupFactor);
Assert.False(container.PlacementPolicy.Value.Unique);
Assert.NotEmpty(container.PlacementPolicy.Value.Selectors);
Assert.Equal(2, container.PlacementPolicy.Value.Selectors.Count);
@ -196,7 +199,7 @@ public class ContainerTests : SmokeTestsBase
Assert.Equal(OwnerId!.ToString(), container.Owner!.Value);
Assert.NotNull(container.Version);
Assert.Equal(Version!.Major, container.Version.Major);
Assert.Equal(Version.Minor, container.Version.Minor);
}
@ -233,12 +236,12 @@ public class ContainerTests : SmokeTestsBase
});
}
#pragma warning disable xUnit1031 // Timeout is used
#pragma warning disable xUnit1031 // Timeout is used
if (!Task.WaitAll(tasks, 20000))
{
Assert.Fail("cannot create containers");
}
#pragma warning restore xUnit1031
#pragma warning restore xUnit1031
var containers = client.ListContainersAsync(new PrmContainerGetAll(), default);

10
src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs
View file

@ -32,8 +32,10 @@ public class SmokeClientTests : SmokeTestsBase
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Equal(2, result.Addresses.Count);
Assert.Equal(11, result.Attributes.Count);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
//Assert.Equal(2, result.Addresses.Count);
//Assert.Equal(11, result.Attributes.Count);
}
[Fact]
@ -65,13 +67,13 @@ public class SmokeClientTests : SmokeTestsBase
Assert.True(result.HomomorphicHashingDisabled);
Assert.True(result.MaintenanceModeAllowed);
Assert.True(0u < result.MagicNumber);
Assert.Equal(0u, result.AuditFee);
Assert.Equal(0u, result.BasicIncomeRate);
Assert.Equal(0u, result.ContainerAliasFee);
Assert.Equal(0u, result.ContainerFee);
Assert.Equal(75u, result.EpochDuration);
Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee);
Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee);
Assert.Equal(12u, result.MaxECDataCount);
Assert.Equal(4u, result.MaxECParityCount);
Assert.Equal(5242880u, result.MaxObjectSize);

6
src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs
View file

@ -58,7 +58,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
int[] objectSizes = [1, 257, 5 * 1024 * 1024, 20 * 1024 * 1024];
string[] objectTypes = [clientCut, serverCut, singleObject];
foreach (var objectSize in objectSizes)
{
_testOutputHelper.WriteLine($"test set for object size {objectSize}");
@ -157,7 +157,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
Assert.NotNull(x);
Assert.True(x.Length > 0);
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
}
}
@ -201,7 +201,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
{
if (bytes.Length < 1024 + 64 || bytes.Length > 5900)
if (bytes.Length < 1024 + 64 || bytes.Length > 5900)
return;
var patch = new byte[1024];

42
src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs
View file

@ -1,42 +0,0 @@
using System.Security.Cryptography;
using FrostFS.SDK.Client;
using FrostFS.SDK.Cryptography;
namespace FrostFS.SDK.Tests.Smoke;
public abstract class MultiThreadTestsBase
{
private TestNodeInfo[] nodes;
protected CallContext? Ctx { get; }
protected MultiThreadTestsBase()
{
nodes = new TestNodeInfo[4];
nodes[0] = new(new Uri(""), "");
nodes[1] = new(new Uri(""), "");
nodes[2] = new(new Uri(""), "");
nodes[3] = new(new Uri(""), "");
}
}
public class TestNodeInfo
{
internal Uri Uri;
protected ECDsa? Key { get; }
protected FrostFsOwner? OwnerId { get; }
protected FrostFsVersion? Version { get; }
public TestNodeInfo(Uri uri, string keyString)
{
Uri = uri;
Key = keyString.LoadWif();
OwnerId = FrostFsOwner.FromKey(Key);
Version = new FrostFsVersion(2, 13);
}
}

544
src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs
View file

@ -1,544 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
using FrostFS.SDK.Client;
using FrostFS.SDK.Cryptography;
namespace FrostFS.SDK.Tests.Smoke;
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
public class MultithreadPoolSmokeTests : SmokeTestsBase
{
private InitParameters GetDefaultParams()
{
return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
{
Key = keyString.LoadWif(),
NodeParams = [new(1, url, 100.0f)],
ClientBuilder = null,
GracefulCloseOnSwitchTimeout = 30_000_000,
Logger = null
};
}
[Fact]
public async void NetworkMapTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
Assert.Null(error);
var result = await pool.GetNetmapSnapshotAsync(default);
Assert.True(result.Epoch > 0);
Assert.Single(result.NodeInfoCollection);
var item = result.NodeInfoCollection[0];
Assert.Equal(2, item.Version.Major);
Assert.Equal(13, item.Version.Minor);
Assert.Equal(NodeState.Online, item.State);
Assert.True(item.PublicKey.Length > 0);
Assert.Single(item.Addresses);
Assert.Equal(9, item.Attributes.Count);
}
[Fact]
public async void NodeInfoTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
Assert.Null(error);
var result = await pool.GetNodeInfoAsync(default);
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
}
[Fact]
public async void NodeInfoStatisticsTwoNodesTest()
{
var callbackText = string.Empty;
var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
{
Key = keyString.LoadWif(),
NodeParams = [
new(1, url, 100.0f),
new(2, url.Replace('0', '1'), 100.0f)
],
ClientBuilder = null,
GracefulCloseOnSwitchTimeout = 30_000_000,
Logger = null,
Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"
};
var pool = new Pool(options);
var ctx = new CallContext(TimeSpan.Zero);
var error = await pool.Dial(ctx).ConfigureAwait(true);
Assert.Null(error);
var result = await pool.GetNodeInfoAsync(default);
var statistics = pool.Statistic();
Assert.False(string.IsNullOrEmpty(callbackText));
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
}
[Fact]
public async void NodeInfoStatisticsTest()
{
var options = GetDefaultParams();
var callbackText = string.Empty;
options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
var pool = new Pool(options);
var ctx = new CallContext(TimeSpan.Zero);
var error = await pool.Dial(ctx).ConfigureAwait(true);
Assert.Null(error);
var result = await pool.GetNodeInfoAsync(default);
Assert.False(string.IsNullOrEmpty(callbackText));
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
}
[Fact]
public async void GetSessionTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
var prm = new PrmSessionCreate(100);
var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true);
var ownerHash = Base58.Decode(OwnerId!.Value);
Assert.NotNull(token);
Assert.NotEqual(Guid.Empty, token.Id);
Assert.Equal(33, token.SessionKey.Length);
}
[Fact]
public async void CreateObjectWithSessionToken()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["key1", "value1"]);
var containerId = await pool.PutContainerAsync(createContainerParam, default);
var bytes = GetRandomBytes(1024);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
sessionToken: token);
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(pool);
}
[Fact]
public async void FilterTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
lightWait);
var containerId = await pool.PutContainerAsync(createContainerParam, default);
var bytes = new byte[] { 1, 2, 3 };
var ParentHeader = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular)
{
PayloadLength = 3
};
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")],
new FrostFsSplit()));
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId, false), default);
var ecdsaKey = keyString.LoadWif();
var networkInfo = await pool.GetNetmapSnapshotAsync(default);
await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId));
await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
var checkSum = CheckSum.CreateCheckSum(bytes);
await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
await CheckFilter(pool, containerId, new FilterByPhysicallyStored());
}
private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter)
{
var resultObjectsCount = 0;
PrmObjectSearch searchParam = new(containerId, null, [], filter);
await foreach (var objId in pool.SearchObjectsAsync(searchParam, default))
{
resultObjectsCount++;
var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
}
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioTest(int objectSize)
{
bool callbackInvoked = false;
var options = GetDefaultParams();
options.Callback = new((cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
});
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["testKey", "testValue"]);
var createdContainer = await pool.PutContainerAsync(createContainerParam, default);
var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default);
Assert.NotNull(container);
Assert.True(callbackInvoked);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]));
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default))
{
hasObject = true;
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes!);
Assert.Equal("fileName", objHeader.Attributes!.First().Key);
Assert.Equal("test", objHeader.Attributes!.First().Value);
}
Assert.True(hasObject);
var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(pool);
await foreach (var _ in pool.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioWithSessionTest(int objectSize)
{
var options = GetDefaultParams();
var pool = new Pool(options);
options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0));
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
await Cleanup(pool);
var ctx = new CallContext(TimeSpan.FromSeconds(20));
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams);
var container = await pool.PutContainerAsync(createContainerParam, ctx);
var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx);
Assert.NotNull(containerInfo);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: container,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
sessionToken: token);
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default))
{
hasObject = true;
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(pool);
await foreach (var _ in pool.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
[InlineData(64 * 1024 * 1024 - 1)]
[InlineData(64 * 1024 * 1024 + 1)]
[InlineData(2 * 64 * 1024 * 1024 + 256)]
[InlineData(200)]
public async void ClientCutScenarioTest(int objectSize)
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
lightWait);
var containerId = await pool.PutContainerAsync(createContainerParam, default);
var ctx = new CallContext(TimeSpan.FromSeconds(10));
// ctx.Interceptors.Add(new CallbackInterceptor());
var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx);
Assert.NotNull(container);
byte[] bytes = GetRandomBytes(objectSize);
var param = new PrmObjectClientCutPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
payload: new MemoryStream(bytes));
var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default))
{
hasObject = true;
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes[0].Key);
Assert.Equal("test", objHeader.Attributes[0].Value);
}
Assert.True(hasObject);
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await CheckFilter(pool, containerId, new FilterByRootObject());
await Cleanup(pool);
await foreach (var cid in pool.ListContainersAsync(default, default))
{
Assert.Fail($"Container {cid.GetValue()} exist");
}
}
}

684
src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs
View file

@ -1,684 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Interfaces;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.SmokeTests;
namespace FrostFS.SDK.Tests.Smoke;
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
public class MultithreadSmokeClientTests : SmokeTestsBase
{
[Fact]
public async void AccountTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
var result = await client.GetBalanceAsync(default);
Assert.NotNull(result);
Assert.True(result.Value == 0);
}
[Fact]
public async void NetworkMapTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
var result = await client.GetNetmapSnapshotAsync(default);
Assert.True(result.Epoch > 0);
Assert.Single(result.NodeInfoCollection);
var item = result.NodeInfoCollection[0];
Assert.Equal(2, item.Version.Major);
Assert.Equal(13, item.Version.Minor);
Assert.Equal(NodeState.Online, item.State);
Assert.True(item.PublicKey.Length > 0);
Assert.Single(item.Addresses);
Assert.Equal(9, item.Attributes.Count);
}
[Fact]
public async void NodeInfoTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
var result = await client.GetNodeInfoAsync(default);
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
}
[Fact]
public async void NodeInfoStatisticsTest()
{
var options = ClientOptions;
var callbackContent = string.Empty;
options.Value.Callback = (cs) => callbackContent = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
var client = FrostFSClient.GetInstance(options, GrpcChannel);
var result = await client.GetNodeInfoAsync(default);
Assert.NotEmpty(callbackContent);
}
[Fact]
public async void GetSessionTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
var token = await client.CreateSessionAsync(new(100), default);
Assert.NotNull(token);
Assert.NotEqual(Guid.Empty, token.Id);
Assert.Equal(33, token.SessionKey.Length);
}
[Fact]
public async void CreateObjectWithSessionToken()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
await Cleanup(client);
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["key1", "value1"]);
var containerId = await client.PutContainerAsync(createContainerParam, default);
var bytes = GetRandomBytes(1024);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
sessionToken: token);
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default)
.ConfigureAwait(true);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk().ConfigureAwait(true)) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(client).ConfigureAwait(true);
}
[Fact]
public async void FilterTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
lightWait);
var containerId = await client.PutContainerAsync(createContainerParam, default);
var bytes = new byte[] { 1, 2, 3 };
var ParentHeader = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular)
{
PayloadLength = 3
};
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")],
new FrostFsSplit()));
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var head = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
var ecdsaKey = keyString.LoadWif();
var networkInfo = await client.GetNetmapSnapshotAsync(default);
await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId));
await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
var checkSum = CheckSum.CreateCheckSum(bytes);
await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
await CheckFilter(client, containerId, new FilterByPhysicallyStored());
}
private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter)
{
var resultObjectsCount = 0;
PrmObjectSearch searchParam = new(containerId, null, [], filter);
await foreach (var objId in client.SearchObjectsAsync(searchParam, default))
{
resultObjectsCount++;
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
}
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioTest(int objectSize)
{
bool callbackInvoked = false;
var options = ClientOptions;
options.Value.Callback = new((cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
});
var client = FrostFSClient.GetInstance(options, GrpcChannel);
await Cleanup(client);
var ctx = new CallContext(TimeSpan.FromSeconds(20));
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["testKey", "testValue"]);
var createdContainer = await client.PutContainerAsync(createContainerParam, ctx);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
Assert.NotNull(container);
Assert.True(callbackInvoked);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]));
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default))
{
hasObject = true;
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Fact]
public async void PatchTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["testKey", "testValue"]);
var createdContainer = await client.PutContainerAsync(createContainerParam, default);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
Assert.NotNull(container);
var bytes = new byte[1024];
for (int i = 0; i < 1024; i++)
{
bytes[i] = 31;
}
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]));
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var patch = new byte[16];
for (int i = 0; i < 16; i++)
{
patch[i] = 32;
}
var range = new FrostFsRange(8, (ulong)patch.Length);
var patchParams = new PrmObjectPatch(
new FrostFsAddress(createdContainer, objectId),
payload: new MemoryStream(patch),
maxChunkLength: 32,
range: range);
var newIbjId = await client.PatchObjectAsync(patchParams, default);
var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
for (int i = 0; i < (int)range.Offset; i++)
Assert.Equal(downloadedBytes[i], bytes[i]);
var rangeEnd = range.Offset + range.Length;
for (int i = (int)range.Offset; i < (int)rangeEnd; i++)
Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]);
for (int i = (int)rangeEnd; i < bytes.Length; i++)
Assert.Equal(downloadedBytes[i], bytes[i]);
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Fact]
public async void RangeTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["testKey", "testValue"]);
var createdContainer = await client.PutContainerAsync(createContainerParam, default);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
Assert.NotNull(container);
var bytes = new byte[256];
for (int i = 0; i < 256; i++)
{
bytes[i] = (byte)i;
}
var param = new PrmObjectPut(
new FrostFsObjectHeader(containerId: createdContainer, type: FrostFsObjectType.Regular));
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64));
var rangeReader = await client.GetRangeAsync(rangeParam, default);
var downloadedBytes = new byte[rangeParam.Range.Length];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await rangeReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Fact]
public async void RangeHashTest()
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["testKey", "testValue"]);
var createdContainer = await client.PutContainerAsync(createContainerParam, default);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
Assert.NotNull(container);
var bytes = new byte[256];
for (int i = 0; i < 256; i++)
{
bytes[i] = (byte)i;
}
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular));
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [new FrostFsRange(100, 64)], bytes);
var hashes = await client.GetRangeHashAsync(rangeParam, default);
foreach (var hash in hashes)
{
var x = hash[..32].ToArray();
}
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioWithSessionTest(int objectSize)
{
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
//Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
await Cleanup(client);
var ctx = new CallContext(TimeSpan.FromSeconds(20));
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams);
var container = await client.PutContainerAsync(createContainerParam, ctx);
var containerInfo = await client.GetContainerAsync(new PrmContainerGet(container), ctx);
Assert.NotNull(containerInfo);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: container,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
sessionToken: token);
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(
new PrmObjectSearch(container, token, [], filter), default))
{
hasObject = true;
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmObjectGet(container, objectId, token), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
[InlineData(64 * 1024 * 1024 - 1)]
[InlineData(64 * 1024 * 1024 + 1)]
[InlineData(2 * 64 * 1024 * 1024 + 256)]
[InlineData(200)]
public async void ClientCutScenarioTest(int objectSize)
{
var options = ClientOptions;
options.Value.Interceptors.Add(new CallbackInterceptor());
var client = FrostFSClient.GetInstance(options, GrpcChannel);
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
lightWait);
var containerId = await client.PutContainerAsync(createContainerParam, default);
var ctx = new CallContext(TimeSpan.FromSeconds(10));
var container = await client.GetContainerAsync(new PrmContainerGet(containerId), ctx);
Assert.NotNull(container);
byte[] bytes = GetRandomBytes(objectSize);
var param = new PrmObjectClientCutPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
payload: new MemoryStream(bytes));
var objectId = await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default))
{
hasObject = true;
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes[0].Key);
Assert.Equal("test", objHeader.Attributes[0].Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await CheckFilter(client, containerId, new FilterByRootObject());
await Cleanup(client);
var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5));
IAsyncEnumerator<FrostFsContainerId>? enumerator = null;
do
{
if (deadline <= DateTime.UtcNow)
{
Assert.Fail("Containers exist");
break;
}
enumerator = client.ListContainersAsync(default, default).GetAsyncEnumerator();
await Task.Delay(500);
}
while (await enumerator!.MoveNextAsync());
}
[Fact]
public async void NodeInfoCallbackAndInterceptorTest()
{
bool callbackInvoked = false;
bool intercepterInvoked = false;
var options = ClientOptions;
options.Value.Callback = (cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
};
options.Value.Interceptors.Add(new CallbackInterceptor(s => intercepterInvoked = true));
var client = FrostFSClient.GetInstance(options, GrpcChannel);
var result = await client.GetNodeInfoAsync(default);
Assert.True(callbackInvoked);
Assert.True(intercepterInvoked);
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
}
}

546
src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs
View file

@ -1,546 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
using FrostFS.SDK.Client;
using FrostFS.SDK.Cryptography;
namespace FrostFS.SDK.Tests.Smoke;
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
public class PoolSmokeTests : SmokeTestsBase
{
private InitParameters GetDefaultParams()
{
return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
{
Key = keyString.LoadWif(),
NodeParams = [new(1, url, 100.0f)],
ClientBuilder = null,
GracefulCloseOnSwitchTimeout = 30_000_000,
Logger = null
};
}
[Fact]
public async void NetworkMapTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
Assert.Null(error);
var result = await pool.GetNetmapSnapshotAsync(default);
Assert.True(result.Epoch > 0);
Assert.Single(result.NodeInfoCollection);
var item = result.NodeInfoCollection[0];
Assert.Equal(2, item.Version.Major);
Assert.Equal(13, item.Version.Minor);
Assert.Equal(NodeState.Online, item.State);
Assert.True(item.PublicKey.Length > 0);
Assert.Single(item.Addresses);
Assert.Equal(9, item.Attributes.Count);
}
[Fact]
public async void NodeInfoTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
Assert.Null(error);
var result = await pool.GetNodeInfoAsync(default);
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
}
[Fact]
public async void NodeInfoStatisticsTwoNodesTest()
{
var callbackText = string.Empty;
var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
{
Key = keyString.LoadWif(),
NodeParams = [
new(1, url, 100.0f),
new(2, url.Replace('0', '1'), 100.0f)
],
ClientBuilder = null,
GracefulCloseOnSwitchTimeout = 30_000_000,
Logger = null,
Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"
};
var pool = new Pool(options);
var ctx = new CallContext(TimeSpan.Zero);
var error = await pool.Dial(ctx).ConfigureAwait(true);
Assert.Null(error);
var result = await pool.GetNodeInfoAsync(default);
var statistics = pool.Statistic();
Assert.False(string.IsNullOrEmpty(callbackText));
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
}
[Fact]
public async void NodeInfoStatisticsTest()
{
var options = GetDefaultParams();
var callbackText = string.Empty;
options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
var pool = new Pool(options);
var ctx = new CallContext(TimeSpan.Zero);
var error = await pool.Dial(ctx).ConfigureAwait(true);
Assert.Null(error);
var result = await pool.GetNodeInfoAsync(default);
Assert.False(string.IsNullOrEmpty(callbackText));
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
}
[Fact]
public async void GetSessionTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
var prm = new PrmSessionCreate(100);
var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true);
var ownerHash = Base58.Decode(OwnerId!.Value);
Assert.NotNull(token);
Assert.NotEqual(Guid.Empty, token.Id);
Assert.Equal(33, token.SessionKey.Length);
}
[Fact]
public async void CreateObjectWithSessionToken()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["key1", "value1"]);
var containerId = await pool.PutContainerAsync(createContainerParam, default);
var bytes = GetRandomBytes(1024);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
sessionToken: token);
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(pool);
}
[Fact]
public async void FilterTest()
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
lightWait);
var containerId = await pool.PutContainerAsync(createContainerParam, default);
var bytes = new byte[] { 1, 2, 3 };
var ParentHeader = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular)
{
PayloadLength = 3
};
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")],
new FrostFsSplit()));
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
var ecdsaKey = keyString.LoadWif();
var networkInfo = await pool.GetNetmapSnapshotAsync(default);
await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId));
await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
var checkSum = CheckSum.CreateCheckSum(bytes);
await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
await CheckFilter(pool, containerId, new FilterByPhysicallyStored());
}
private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter)
{
var resultObjectsCount = 0;
PrmObjectSearch searchParam = new(containerId, null, [], filter);
await foreach (var objId in pool.SearchObjectsAsync(searchParam, default))
{
resultObjectsCount++;
var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
}
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioTest(int objectSize)
{
bool callbackInvoked = false;
var options = GetDefaultParams();
options.Callback = new((cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
});
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var ctx = new CallContext(TimeSpan.Zero);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams,
xheaders: ["testKey", "testValue"]);
var createdContainer = await pool.PutContainerAsync(createContainerParam, ctx);
var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default);
Assert.NotNull(container);
Assert.True(callbackInvoked);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]));
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default))
{
hasObject = true;
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes!);
Assert.Equal("fileName", objHeader.Attributes!.First().Key);
Assert.Equal("test", objHeader.Attributes!.First().Value);
}
Assert.True(hasObject);
var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(pool);
await foreach (var _ in pool.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioWithSessionTest(int objectSize)
{
var options = GetDefaultParams();
var pool = new Pool(options);
options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0));
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
await Cleanup(pool);
var ctx = new CallContext(TimeSpan.FromSeconds(20));
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
PrmWait.DefaultParams);
var container = await pool.PutContainerAsync(createContainerParam, ctx);
var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx);
Assert.NotNull(containerInfo);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: container,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
sessionToken: token);
var stream = await pool.PutObjectAsync(param, new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
var objs = pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default);
await foreach (var objId in objs)
{
hasObject = true;
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(pool);
await foreach (var _ in pool.ListContainersAsync(default, default))
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
[InlineData(64 * 1024 * 1024 - 1)]
[InlineData(64 * 1024 * 1024 + 1)]
[InlineData(2 * 64 * 1024 * 1024 + 256)]
[InlineData(200)]
public async void ClientCutScenarioTest(int objectSize)
{
var options = GetDefaultParams();
var pool = new Pool(options);
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
Assert.Null(error);
await Cleanup(pool);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
lightWait);
var containerId = await pool.PutContainerAsync(createContainerParam, default);
var ctx = new CallContext(TimeSpan.FromSeconds(10));
// ctx.Interceptors.Add(new CallbackInterceptor());
var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx);
Assert.NotNull(container);
byte[] bytes = GetRandomBytes(objectSize);
var param = new PrmObjectClientCutPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
payload: new MemoryStream(bytes));
var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default))
{
hasObject = true;
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader); Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes[0].Key);
Assert.Equal("test", objHeader.Attributes[0].Value);
}
Assert.True(hasObject);
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await CheckFilter(pool, containerId, new FilterByRootObject());
await Cleanup(pool);
await foreach (var cid in pool.ListContainersAsync(default, default))
{
Assert.Fail($"Container {cid.GetValue()} exist");
}
}
}

16
src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs
View file

@ -16,19 +16,13 @@ namespace FrostFS.SDK.Tests.Smoke;
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
public abstract class SmokeTestsBase
{
// cluster Ori
// internal readonly string url = "http://10.78.128.207:8080";
// internal readonly string keyString = "L4JWLdedUd4b21sriRHtCPGkjG2Mryz2AWLiVqTBSNyxxyAUcc7s";
// cluster
internal readonly string url = "http://10.78.128.190:8080";
internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba";
// internal readonly string url = "http://10.78.128.190:8080";
// internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba";
// WSL2
// internal readonly string url = "http://172.29.238.97:8080";
// internal readonly string keyString = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; // "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK";
//"KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq";
internal readonly string url = "http://172.29.238.97:8080"; // "http://172.20.8.23:8080";
internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK";
protected ECDsa? Key { get; }
@ -77,7 +71,7 @@ public abstract class SmokeTestsBase
if (networkSettings.HomomorphicHashingDisabled)
attributes.Add(new("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", "true"));
var containerInfo = new FrostFsContainerInfo(
new FrostFsPlacementPolicy(unique, backupFactor, selectors, filter, replicas),
[.. attributes]);

10
src/FrostFS.SDK.Tests/Unit/ContainerTest.cs
View file

@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
@ -20,13 +19,16 @@ public class ContainerTest : ContainerTestsBase
Assert.NotNull(result);
Assert.NotNull(result.GetValue());
Assert.True(Base58.Encode(Mocker.ContainerGuid.ToBytes()) == result.GetValue());
var bytes = Mocker.ContainerGuid.ToByteArray(true);
Assert.True(Base58.Encode(new Span<byte>(bytes)) == result.GetValue());
}
[Fact]
public async void GetContainerTest()
{
var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes()));
var cid = new FrostFsContainerId(Base58.Encode(new Span<byte>(Mocker.ContainerGuid.ToByteArray(true))));
var result = await GetClient().GetContainerAsync(new PrmContainerGet(cid), default);
@ -61,7 +63,7 @@ public class ContainerTest : ContainerTestsBase
public async void DeleteContainerAsyncTest()
{
Mocker.ReturnContainerRemoved = true;
var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes()));
var cid = new FrostFsContainerId(Base58.Encode(new Span<byte>(Mocker.ContainerGuid.ToByteArray(true))));
await GetClient().DeleteContainerAsync(new PrmContainerDelete(cid, PrmWait.DefaultParams), default);

4
src/FrostFS.SDK.Tests/Unit/ObjectTest.cs
View file

@ -88,7 +88,7 @@ public class ObjectTest : ObjectTestsBase
// PART1
Assert.Equal(blockSize, objects[0].Payload.Length);
Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload));
Assert.NotNull(objects[0].Header.Split.SplitId);
Assert.Null(objects[0].Header.Split.Previous);
Assert.True(objects[0].Header.Attributes.Count == 0);
@ -104,7 +104,7 @@ public class ObjectTest : ObjectTestsBase
// last part
Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length);
Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload));
Assert.True(bytes.AsMemory(2 * blockSize).ToArray().SequenceEqual(objects[2].Payload));
Assert.NotNull(objects[3].Header.Split.Parent);
Assert.NotNull(objects[3].Header.Split.ParentHeader);

2
src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs
View file

@ -35,7 +35,7 @@ public abstract class ObjectTestsBase
ContainerGuid = Guid.NewGuid()
};
ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes()));
ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToByteArray(true)));
Mocker.ObjectHeader = new(
ContainerId,

3
src/FrostFS.SDK.Tests/Unit/SessionTests.cs
View file

@ -2,7 +2,6 @@ using System.Diagnostics.CodeAnalysis;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
namespace FrostFS.SDK.Tests.Unit;
@ -38,7 +37,7 @@ public class SessionTest : SessionTestsBase
Assert.NotNull(result);
Assert.NotEqual(Guid.Empty, result.Id);
Assert.Equal(Mocker.SessionId, result.Id.ToBytes());
Assert.Equal(Mocker.SessionId, result.Id.ToByteArray(true));
Assert.Equal(Mocker.SessionKey, result.SessionKey.ToArray());
//Assert.Equal(OwnerId.ToMessage(), result.Token.Body.OwnerId);