TrueCloudLab/frostfs-sdk-csharp
18
0
Fork 4
Code Issues 5 Pull requests Projects Releases Packages 3 Activity Actions

[#40] Client: Add memory optimization for hash

Browse source 
Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2025-03-11 22:56:28 +03:00
parent 32a7e64538
commit 809bd90352
17 changed files with 170 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj
View file

@ -4,7 +4,7 @@
<TargetFramework>netstandard2.0</TargetFramework>
<LangVersion>12.0</LangVersion>
<Nullable>enable</Nullable>
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
</PropertyGroup>
<PropertyGroup>

4
src/FrostFS.SDK.Client/Models/Session/FrostFsSessionToken.cs
View file

@ -81,7 +81,7 @@ public class FrostFsSessionToken
}
sessionToken.Body.SessionKey = key.PublicKeyProto;
sessionToken.Signature = key.ECDsaKey.SignMessagePart(sessionToken.Body);
sessionToken.Signature = key.SignMessagePart(sessionToken.Body);
return sessionToken;
}
@ -116,7 +116,7 @@ public class FrostFsSessionToken
Verb = verb
};
sessionToken.Signature = key.ECDsaKey.SignMessagePart(sessionToken.Body);
sessionToken.Signature = key.SignMessagePart(sessionToken.Body);
return sessionToken;
}

2
src/FrostFS.SDK.Client/ObjectWriter.cs
View file

@ -34,7 +34,7 @@ namespace FrostFS.SDK.Client
chunkRequest.AddMetaHeader(args.XHeaders);
chunkRequest.Sign(this.ctx.Key.ECDsaKey);
chunkRequest.Sign(this.ctx.Key);
await streamer.Write(chunkRequest).ConfigureAwait(false);
}

2
src/FrostFS.SDK.Client/Services/AccountingServiceProvider.cs
View file

@ -27,7 +27,7 @@ internal sealed class AccountingServiceProvider : ContextAccessor
};
request.AddMetaHeader([]);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await _accountingServiceClient!.BalanceAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);

6
src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs
View file

@ -32,7 +32,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
};
request.AddMetaHeader(args.XHeaders);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await _apeManagerServiceClient!.AddChainAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -53,7 +53,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
};
request.AddMetaHeader(args.XHeaders);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await _apeManagerServiceClient!.RemoveChainAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -71,7 +71,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
};
request.AddMetaHeader(args.XHeaders);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await _apeManagerServiceClient!.ListChainsAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);

12
src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs
View file

@ -39,7 +39,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
internal async Task<FrostFsContainerInfo> GetContainerAsync(PrmContainerGet args, CallContext ctx)
{
GetRequest request = GetContainerRequest(args.Container.GetContainerID(), args.XHeaders, ClientContext.Key.ECDsaKey);
GetRequest request = GetContainerRequest(args.Container.GetContainerID(), args.XHeaders, ClientContext.Key);
var response = await service.GetAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -59,7 +59,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
};
request.AddMetaHeader(args.XHeaders);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await service.ListAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -96,7 +96,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await service.PutAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -127,7 +127,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await service.DeleteAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -139,7 +139,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
Verifier.CheckResponse(response);
}
private static GetRequest GetContainerRequest(ContainerID id, string[] xHeaders, ECDsa key)
private static GetRequest GetContainerRequest(ContainerID id, string[] xHeaders, ClientKey key)
{
var request = new GetRequest
{
@ -163,7 +163,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
private async Task WaitForContainer(WaitExpects expect, ContainerID id, PrmWait waitParams, CallContext ctx)
{
var request = GetContainerRequest(id, [], ClientContext.Key.ECDsaKey);
var request = GetContainerRequest(id, [], ClientContext.Key);
async Task action()
{

6
src/FrostFS.SDK.Client/Services/NetmapServiceProvider.cs
View file

@ -50,7 +50,7 @@ internal sealed class NetmapServiceProvider : ContextAccessor
};
request.AddMetaHeader([]);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -64,7 +64,7 @@ internal sealed class NetmapServiceProvider : ContextAccessor
var request = new NetworkInfoRequest();
request.AddMetaHeader([]);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken)
.ConfigureAwait(false);
@ -79,7 +79,7 @@ internal sealed class NetmapServiceProvider : ContextAccessor
var request = new NetmapSnapshotRequest();
request.AddMetaHeader([]);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);

27
src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs
View file

@ -67,7 +67,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await client!.HeadAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken).ConfigureAwait(false);
@ -111,7 +111,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
return await GetObject(request, ctx).ConfigureAwait(false);
}
@ -145,7 +145,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var call = client.GetRange(request, null, ctx.GetDeadline(), ctx.CancellationToken);
return new RangeReader(call);
@ -185,7 +185,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await client.GetRangeHashAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -218,7 +218,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
ClientContext.Key);
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await client.DeleteAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
@ -247,7 +247,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
using var stream = GetSearchReader(request, ctx);
@ -283,7 +283,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders, protoToken);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
var response = await client.PutSingleAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken).ConfigureAwait(false);
@ -363,7 +363,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
request.AddMetaHeader(args.XHeaders);
}
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
await call.RequestStream.WriteAsync(request).ConfigureAwait(false);
@ -402,11 +402,11 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
throw new ArgumentException("The stream has zero length");
var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
args.PutObjectContext.MaxObjectSizeCache = (int)networkSettings.MaxObjectSize;
var partSize = (int)networkSettings.MaxObjectSize;
var restBytes = args.PutObjectContext.FullLength;
var objectSize = (int)Math.Min((ulong)args.PutObjectContext.MaxObjectSizeCache, restBytes);
var objectSize = (int)Math.Min((ulong)partSize, restBytes);
// define collection capacity
var objectsCount = (int)(restBytes / (ulong)objectSize) + ((restBytes % (ulong)objectSize) > 0 ? 1 : 0);
@ -414,6 +414,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
// if the object fits one part, it can be loaded as non-complex object
if (objectsCount == 1)
{
args.PutObjectContext.MaxObjectSizeCache = partSize;
var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false);
return singlePartResult.ObjectId;
}
@ -422,8 +423,6 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
SplitId splitId = new();
var partSize = args.PutObjectContext.MaxObjectSizeCache;
// keep attributes for the large object
var attributes = args.Header!.Attributes.ToArray();
header.Attributes = null;
@ -578,7 +577,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
};
chunkRequest.AddMetaHeader(args.XHeaders);
chunkRequest.Sign(ClientContext.Key.ECDsaKey);
chunkRequest.Sign(ClientContext.Key);
await stream.Write(chunkRequest).ConfigureAwait(false);
}
@ -640,7 +639,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
initRequest.AddMetaHeader(args.XHeaders, protoToken);
initRequest.Sign(ClientContext.Key.ECDsaKey);
initRequest.Sign(ClientContext.Key);
return await PutObjectInit(initRequest, ctx).ConfigureAwait(false);
}

2
src/FrostFS.SDK.Client/Services/SessionServiceProvider.cs
View file

@ -26,7 +26,7 @@ internal sealed class SessionServiceProvider : ContextAccessor
};
request.AddMetaHeader(args.XHeaders);
request.Sign(ClientContext.Key.ECDsaKey);
request.Sign(ClientContext.Key);
return await CreateSession(request, ctx).ConfigureAwait(false);
}

57
src/FrostFS.SDK.Client/Tools/RequestSigner.cs
View file

@ -1,4 +1,5 @@
using System;
using System.IO;
using System.Security.Cryptography;
using FrostFS.Refs;
@ -13,7 +14,7 @@ using Org.BouncyCastle.Crypto.Digests;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.Crypto.Signers;
using Org.BouncyCastle.Math;
using Org.BouncyCastle.Utilities;
using Signature = FrostFS.Refs.Signature;
namespace FrostFS.SDK.Client;
@ -74,7 +75,7 @@ public static class RequestSigner
};
}
public static ByteString SignData(this ECDsa key, byte[] data)
public static ByteString SignData(this ECDsa key, ReadOnlyMemory<byte> data)
{
if (key is null)
{
@ -84,27 +85,61 @@ public static class RequestSigner
Span<byte> result = stackalloc byte[65];
result[0] = 0x04;
//var hash = new byte[65];
//hash[0] = 0x04;
key.SignHash(data.Sha512()).AsSpan().CopyTo(result[1..]);
return ByteString.CopyFrom(result);
}
internal static Signature SignMessagePart(this ECDsa key, IMessage? data)
public static ByteString SignDataByHash(this ECDsa key, byte[] hash)
{
var data2Sign = data is null ? [] : data.ToByteArray();
if (key is null)
{
throw new ArgumentNullException(nameof(key));
}
Span<byte> result = stackalloc byte[65];
result[0] = 0x04;
key.SignHash(hash).AsSpan().CopyTo(result[1..]);
return ByteString.CopyFrom(result);
}
internal static Signature SignMessagePart(this ClientKey key, IMessage? data)
{
if (data is null)
{
return new Signature
{
Key = key.PublicKeyProto,
Sign = key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty),
};
}
var size = data.CalculateSize();
if (size == 0)
{
return new Signature
{
Key = key.PublicKeyProto,
Sign = key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty),
};
}
using HashStream stream = new();
data.WriteTo(stream);
var sig = new Signature
{
Key = ByteString.CopyFrom(key.PublicKey()),
Sign = key.SignData(data2Sign),
Key = key.PublicKeyProto,
Sign = key.ECDsaKey.SignDataByHash(stream.Hash())
};
return sig;
}
internal static void Sign(this IVerifiableMessage message, ECDsa key)
internal static void Sign(this IVerifiableMessage message, ClientKey key)
{
var meta = message.GetMetaHeader();
IVerificationHeader verify = message switch

5
src/FrostFS.SDK.Client/Tools/Verifier.cs
View file

@ -63,14 +63,11 @@ public static class Verifier
return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray());
}
public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig)
public static bool VerifyData(this ECDsa key, ReadOnlyMemory<byte> data, byte[] sig)
{
if (key is null)
throw new ArgumentNullException(nameof(key));
if (data is null)
throw new ArgumentNullException(nameof(data));
if (sig is null)
throw new ArgumentNullException(nameof(sig));

21
src/FrostFS.SDK.Cryptography/Extentions.cs
View file

@ -1,4 +1,5 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using CommunityToolkit.HighPerformance;
@ -60,14 +61,30 @@ public static class Extentions
}
}
public static byte[] Sha512(this byte[] value)
public static byte[] Sha512(this ReadOnlyMemory<byte> value)
{
bool lockTaken = false;
try
{
_spinlockSha512.Enter(ref lockTaken);
return _sha512.ComputeHash(value);
return _sha512.ComputeHash(value.AsStream());
}
finally
{
if (lockTaken)
_spinlockSha512.Exit(false);
}
}
public static byte[] Sha512(this Stream stream)
{
bool lockTaken = false;
try
{
_spinlockSha512.Enter(ref lockTaken);
return _sha512.ComputeHash(stream);
}
finally
{

58
src/FrostFS.SDK.Cryptography/HashStream.cs Normal file
View file

@ -0,0 +1,58 @@
using System.IO;
using System.Security.Cryptography;
namespace FrostFS.SDK.Cryptography;
public sealed class HashStream() : Stream
{
private long position;
private readonly SHA512 _hash = SHA512.Create();
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => long.MaxValue;
public override long Position
{
get { return position; }
set { position = value; }
}
public override void Flush()
{ }
public override int Read(byte[] buffer, int offset, int count)
{
return 0;
}
public override long Seek(long offset, SeekOrigin origin)
{
return 0;
}
public override void SetLength(long value)
{ }
public override void Write(byte[] buffer, int offset, int count)
{
_hash.TransformBlock(buffer, offset, count, buffer, offset);
}
public byte[] Hash()
{
_hash.TransformFinalBlock([], 0, 0);
return _hash.Hash;
}
protected override void Dispose(bool disposing)
{
_hash?.Dispose();
base.Dispose(disposing);
}
}

4
src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs
View file

@ -42,8 +42,8 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader)
ObjectId = new Refs.ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Array.Empty<byte>())) },
Signature = new Refs.Signature
{
Key = ByteString.CopyFrom(Key.PublicKey()),
Sign = Key.SignData(header.ToByteArray()),
Key = Key.PublicKeyProto,
Sign = Key.ECDsaKey. SignData(header.ToByteArray()),
}
}
},

14
src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs
View file

@ -18,7 +18,7 @@ namespace FrostFS.SDK.Tests;
public abstract class ServiceBase(string key)
{
public string StringKey { get; private set; } = key;
public ECDsa Key { get; private set; } = key.LoadWif();
public ClientKey Key { get; private set; } = new ClientKey(key.LoadWif());
public FrostFsVersion Version { get; set; } = DefaultVersion;
public FrostFsPlacementPolicy PlacementPolicy { get; set; } = DefaultPlacementPolicy;
@ -44,21 +44,21 @@ public abstract class ServiceBase(string key)
{
MetaSignature = new Refs.Signature
{
Key = ByteString.CopyFrom(Key.PublicKey()),
Key = Key.PublicKeyProto,
Scheme = Refs.SignatureScheme.EcdsaRfc6979Sha256,
Sign = Key.SignData(response.MetaHeader.ToByteArray())
Sign = Key.ECDsaKey.SignData(response.MetaHeader.ToByteArray())
},
BodySignature = new Refs.Signature
{
Key = ByteString.CopyFrom(Key.PublicKey()),
Key = Key.PublicKeyProto,
Scheme = Refs.SignatureScheme.EcdsaRfc6979Sha256,
Sign = Key.SignData(response.GetBody().ToByteArray())
Sign = Key.ECDsaKey.SignData(response.GetBody().ToByteArray())
},
OriginSignature = new Refs.Signature
{
Key = ByteString.CopyFrom(Key.PublicKey()),
Key = Key.PublicKeyProto,
Scheme = Refs.SignatureScheme.EcdsaRfc6979Sha256,
Sign = Key.SignData([])
Sign = Key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty)
}
};

4
src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs
View file

@ -92,8 +92,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
headResponse.Body.Header.Signature = new Refs.Signature
{
Key = ByteString.CopyFrom(Key.PublicKey()),
Sign = Key.SignData(headResponse.Body.Header.ToByteArray()),
Key = Key.PublicKeyProto,
Sign = Key.ECDsaKey.SignData(headResponse.Body.Header.ToByteArray()),
};
headResponse.VerifyHeader = GetResponseVerificationHeader(headResponse);

8
src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs
View file

@ -55,10 +55,10 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
private async Task RunSuite(IFrostFSClient client, FrostFsContainerId containerId)
{
int[] objectSizes = [1, 257, 6 * 1024, 20 * 1024];
int[] objectSizes = [1, 257, 5 * 1024 * 1024, 20 * 1024 * 1024];
string[] objectTypes = [clientCut, serverCut, singleObject];
foreach (var objectSize in objectSizes)
{
_testOutputHelper.WriteLine($"test set for object size {objectSize}");
@ -77,13 +77,13 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
break;
case clientCut:
objectId = await CreateObjectClientCut(client, containerId, bytes);
_testOutputHelper.WriteLine($"\tclient side cut");
_testOutputHelper.WriteLine($"\tclient side cut");
break;
case singleObject:
if (objectSize > 1 * 1024 * 1024)
continue;
objectId = await PutSingleObject(client, containerId, bytes);
_testOutputHelper.WriteLine($"\tput single object");
_testOutputHelper.WriteLine($"\tput single object");
break;
default: