[#40] Client: Add memory optimization for hash

Signed-off-by: Pavel Gross <p.gross@yadro.com>

src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj

@@ -4,7 +4,7 @@
     <TargetFramework>netstandard2.0</TargetFramework>
     <LangVersion>12.0</LangVersion>
     <Nullable>enable</Nullable>
-	<AnalysisMode>AllEnabledByDefault</AnalysisMode>
+	  <AnalysisMode>AllEnabledByDefault</AnalysisMode>
   </PropertyGroup>
 
   <PropertyGroup>

src/FrostFS.SDK.Client/Models/Session/FrostFsSessionToken.cs

@@ -81,7 +81,7 @@ public class FrostFsSessionToken
         }
 
         sessionToken.Body.SessionKey = key.PublicKeyProto;
-        sessionToken.Signature = key.ECDsaKey.SignMessagePart(sessionToken.Body);
+        sessionToken.Signature = key.SignMessagePart(sessionToken.Body);
 
         return sessionToken;
     }
@@ -116,7 +116,7 @@ public class FrostFsSessionToken
             Verb = verb
         };
 
-        sessionToken.Signature = key.ECDsaKey.SignMessagePart(sessionToken.Body);
+        sessionToken.Signature = key.SignMessagePart(sessionToken.Body);
 
         return sessionToken;
     }

src/FrostFS.SDK.Client/ObjectWriter.cs

@@ -34,7 +34,7 @@ namespace FrostFS.SDK.Client
 
             chunkRequest.AddMetaHeader(args.XHeaders);
 
-            chunkRequest.Sign(this.ctx.Key.ECDsaKey);
+            chunkRequest.Sign(this.ctx.Key);
 
             await streamer.Write(chunkRequest).ConfigureAwait(false);
         }

src/FrostFS.SDK.Client/Services/AccountingServiceProvider.cs

@@ -27,7 +27,7 @@ internal sealed class AccountingServiceProvider : ContextAccessor
         };
 
         request.AddMetaHeader([]);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await _accountingServiceClient!.BalanceAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 

src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs

@@ -32,7 +32,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
         };
 
         request.AddMetaHeader(args.XHeaders);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await _apeManagerServiceClient!.AddChainAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -53,7 +53,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
         };
 
         request.AddMetaHeader(args.XHeaders);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await _apeManagerServiceClient!.RemoveChainAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -71,7 +71,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
         };
 
         request.AddMetaHeader(args.XHeaders);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await _apeManagerServiceClient!.ListChainsAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 

src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs

@@ -39,7 +39,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
 
     internal async Task<FrostFsContainerInfo> GetContainerAsync(PrmContainerGet args, CallContext ctx)
     {
-        GetRequest request = GetContainerRequest(args.Container.GetContainerID(), args.XHeaders, ClientContext.Key.ECDsaKey);
+        GetRequest request = GetContainerRequest(args.Container.GetContainerID(), args.XHeaders, ClientContext.Key);
 
         var response = await service.GetAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -59,7 +59,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
         };
 
         request.AddMetaHeader(args.XHeaders);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await service.ListAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -96,7 +96,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await service.PutAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -127,7 +127,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await service.DeleteAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -139,7 +139,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
         Verifier.CheckResponse(response);
     }
 
-    private static GetRequest GetContainerRequest(ContainerID id, string[] xHeaders, ECDsa key)
+    private static GetRequest GetContainerRequest(ContainerID id, string[] xHeaders, ClientKey key)
     {
         var request = new GetRequest
         {
@@ -163,7 +163,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
 
     private async Task WaitForContainer(WaitExpects expect, ContainerID id, PrmWait waitParams, CallContext ctx)
     {
-        var request = GetContainerRequest(id, [], ClientContext.Key.ECDsaKey);
+        var request = GetContainerRequest(id, [], ClientContext.Key);
 
         async Task action()
         {

src/FrostFS.SDK.Client/Services/NetmapServiceProvider.cs

@@ -50,7 +50,7 @@ internal sealed class NetmapServiceProvider : ContextAccessor
         };
 
         request.AddMetaHeader([]);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -64,7 +64,7 @@ internal sealed class NetmapServiceProvider : ContextAccessor
         var request = new NetworkInfoRequest();
 
         request.AddMetaHeader([]);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken)
             .ConfigureAwait(false);
@@ -79,7 +79,7 @@ internal sealed class NetmapServiceProvider : ContextAccessor
         var request = new NetmapSnapshotRequest();
 
         request.AddMetaHeader([]);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 

src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs

@@ -67,7 +67,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await client!.HeadAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken).ConfigureAwait(false);
 
@@ -111,7 +111,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         return await GetObject(request, ctx).ConfigureAwait(false);
     }
@@ -145,7 +145,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var call = client.GetRange(request, null, ctx.GetDeadline(), ctx.CancellationToken);
         return new RangeReader(call);
@@ -185,7 +185,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await client.GetRangeHashAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -218,7 +218,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
             ClientContext.Key);
 
         request.AddMetaHeader(args.XHeaders, protoToken);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await client.DeleteAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken);
 
@@ -247,7 +247,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         using var stream = GetSearchReader(request, ctx);
 
@@ -283,7 +283,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         request.AddMetaHeader(args.XHeaders, protoToken);
 
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         var response = await client.PutSingleAsync(request, null, ctx.GetDeadline(), ctx.CancellationToken).ConfigureAwait(false);
 
@@ -363,7 +363,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
                     request.AddMetaHeader(args.XHeaders);
                 }
 
-                request.Sign(ClientContext.Key.ECDsaKey);
+                request.Sign(ClientContext.Key);
 
                 await call.RequestStream.WriteAsync(request).ConfigureAwait(false);
 
@@ -402,11 +402,11 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
             throw new ArgumentException("The stream has zero length");
 
         var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
-        args.PutObjectContext.MaxObjectSizeCache = (int)networkSettings.MaxObjectSize;
+        var partSize = (int)networkSettings.MaxObjectSize;
 
         var restBytes = args.PutObjectContext.FullLength;
 
-        var objectSize = (int)Math.Min((ulong)args.PutObjectContext.MaxObjectSizeCache, restBytes);
+        var objectSize = (int)Math.Min((ulong)partSize, restBytes);
 
         // define collection capacity
         var objectsCount = (int)(restBytes / (ulong)objectSize) + ((restBytes % (ulong)objectSize) > 0 ? 1 : 0);
@@ -414,6 +414,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
         // if the object fits one part, it can be loaded as non-complex object
         if (objectsCount == 1)
         {
+            args.PutObjectContext.MaxObjectSizeCache = partSize;
             var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false);
             return singlePartResult.ObjectId;
         }
@@ -422,8 +423,6 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         SplitId splitId = new();
 
-        var partSize = args.PutObjectContext.MaxObjectSizeCache;
-
         // keep attributes for the large object
         var attributes = args.Header!.Attributes.ToArray();
         header.Attributes = null;
@@ -578,7 +577,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
                 };
 
                 chunkRequest.AddMetaHeader(args.XHeaders);
-                chunkRequest.Sign(ClientContext.Key.ECDsaKey);
+                chunkRequest.Sign(ClientContext.Key);
 
                 await stream.Write(chunkRequest).ConfigureAwait(false);
             }
@@ -640,7 +639,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
         initRequest.AddMetaHeader(args.XHeaders, protoToken);
 
-        initRequest.Sign(ClientContext.Key.ECDsaKey);
+        initRequest.Sign(ClientContext.Key);
 
         return await PutObjectInit(initRequest, ctx).ConfigureAwait(false);
     }

src/FrostFS.SDK.Client/Services/SessionServiceProvider.cs

@@ -26,7 +26,7 @@ internal sealed class SessionServiceProvider : ContextAccessor
         };
 
         request.AddMetaHeader(args.XHeaders);
-        request.Sign(ClientContext.Key.ECDsaKey);
+        request.Sign(ClientContext.Key);
 
         return await CreateSession(request, ctx).ConfigureAwait(false);
     }

src/FrostFS.SDK.Client/Tools/RequestSigner.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.Security.Cryptography;
 
 using FrostFS.Refs;
@@ -13,7 +14,7 @@ using Org.BouncyCastle.Crypto.Digests;
 using Org.BouncyCastle.Crypto.Parameters;
 using Org.BouncyCastle.Crypto.Signers;
 using Org.BouncyCastle.Math;
-
+using Org.BouncyCastle.Utilities;
 using Signature = FrostFS.Refs.Signature;
 
 namespace FrostFS.SDK.Client;
@@ -74,7 +75,7 @@ public static class RequestSigner
         };
     }
 
-    public static ByteString SignData(this ECDsa key, byte[] data)
+    public static ByteString SignData(this ECDsa key, ReadOnlyMemory<byte> data)
     {
         if (key is null)
         {
@@ -84,27 +85,61 @@ public static class RequestSigner
         Span<byte> result = stackalloc byte[65];
         result[0] = 0x04;
 
-        //var hash = new byte[65];
-        //hash[0] = 0x04;
-
         key.SignHash(data.Sha512()).AsSpan().CopyTo(result[1..]);
 
         return ByteString.CopyFrom(result);
     }
-
+    
-    internal static Signature SignMessagePart(this ECDsa key, IMessage? data)
+    public static ByteString SignDataByHash(this ECDsa key, byte[] hash)
     {
-        var data2Sign = data is null ? [] : data.ToByteArray();
+        if (key is null)
+        {
+            throw new ArgumentNullException(nameof(key));
+        }
+
+        Span<byte> result = stackalloc byte[65];
+        result[0] = 0x04;
+
+        key.SignHash(hash).AsSpan().CopyTo(result[1..]);
+
+        return ByteString.CopyFrom(result);
+    }
+
+    internal static Signature SignMessagePart(this ClientKey key, IMessage? data)
+    {
+        if (data is null)
+        {
+            return new Signature
+            {
+                Key = key.PublicKeyProto,
+                Sign = key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty),
+            };
+        }
+
+        var size = data.CalculateSize();
+
+        if (size == 0)
+        {
+            return new Signature
+            {
+                Key = key.PublicKeyProto,
+                Sign = key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty),
+            };
+        }
+       
+        using HashStream stream = new();
+        data.WriteTo(stream);
+
         var sig = new Signature
         {
-            Key = ByteString.CopyFrom(key.PublicKey()),
+            Key = key.PublicKeyProto,
-            Sign = key.SignData(data2Sign),
+            Sign = key.ECDsaKey.SignDataByHash(stream.Hash())
         };
 
         return sig;
     }
 
-    internal static void Sign(this IVerifiableMessage message, ECDsa key)
+    internal static void Sign(this IVerifiableMessage message, ClientKey key)
     {
         var meta = message.GetMetaHeader();
         IVerificationHeader verify = message switch

src/FrostFS.SDK.Client/Tools/Verifier.cs

@@ -63,14 +63,11 @@ public static class Verifier
         return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray());
     }
 
-    public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig)
+    public static bool VerifyData(this ECDsa key, ReadOnlyMemory<byte> data, byte[] sig)
     {
         if (key is null)
             throw new ArgumentNullException(nameof(key));
 
-        if (data is null)
-            throw new ArgumentNullException(nameof(data));
-
         if (sig is null)
             throw new ArgumentNullException(nameof(sig));
 

src/FrostFS.SDK.Cryptography/Extentions.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.Security.Cryptography;
 using System.Threading;
 using CommunityToolkit.HighPerformance;
@@ -60,14 +61,30 @@ public static class Extentions
         }
     }
 
-    public static byte[] Sha512(this byte[] value)
+    public static byte[] Sha512(this ReadOnlyMemory<byte> value)
     {
         bool lockTaken = false;
         try
         {
             _spinlockSha512.Enter(ref lockTaken);
 
-            return _sha512.ComputeHash(value);
+            return _sha512.ComputeHash(value.AsStream());
+        }
+        finally
+        {
+            if (lockTaken)
+                _spinlockSha512.Exit(false);
+        }
+    }
+
+    public static byte[] Sha512(this Stream stream)
+    {
+        bool lockTaken = false;
+        try
+        {
+            _spinlockSha512.Enter(ref lockTaken);
+
+            return _sha512.ComputeHash(stream);
         }
         finally
         {

src/FrostFS.SDK.Cryptography/HashStream.cs

@@ -0,0 +1,58 @@
+using System.IO;
+using System.Security.Cryptography;
+
+namespace FrostFS.SDK.Cryptography;
+
+public sealed class HashStream() : Stream
+{
+    private long position;
+
+    private readonly SHA512 _hash = SHA512.Create();
+
+    public override bool CanRead => false;
+
+    public override bool CanSeek => false;
+
+    public override bool CanWrite => true;
+
+    public override long Length => long.MaxValue;
+
+    public override long Position
+    {
+        get { return position; }
+        set { position = value; }
+    }
+
+    public override void Flush()
+    { }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        return 0;
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        return 0;
+    }
+
+    public override void SetLength(long value)
+    { }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        _hash.TransformBlock(buffer, offset, count, buffer, offset);
+    }
+
+    public byte[] Hash()
+    {
+        _hash.TransformFinalBlock([], 0, 0);
+        return _hash.Hash;
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        _hash?.Dispose();
+        base.Dispose(disposing);
+    }
+}

src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs

@@ -42,8 +42,8 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader)
                         ObjectId = new Refs.ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Array.Empty<byte>())) },
                         Signature = new Refs.Signature
                         {
-                            Key = ByteString.CopyFrom(Key.PublicKey()),
+                            Key = Key.PublicKeyProto,
-                            Sign = Key.SignData(header.ToByteArray()),
+                            Sign = Key.ECDsaKey. SignData(header.ToByteArray()),
                         }
                     }
                 },

src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs

@@ -18,7 +18,7 @@ namespace FrostFS.SDK.Tests;
 public abstract class ServiceBase(string key)
 {
     public string StringKey { get; private set; } = key;
-    public ECDsa Key { get; private set; } = key.LoadWif();
+    public ClientKey Key { get; private set; } = new ClientKey(key.LoadWif());
     public FrostFsVersion Version { get; set; } = DefaultVersion;
     public FrostFsPlacementPolicy PlacementPolicy { get; set; } = DefaultPlacementPolicy;
 
@@ -44,21 +44,21 @@ public abstract class ServiceBase(string key)
         {
             MetaSignature = new Refs.Signature
             {
-                Key = ByteString.CopyFrom(Key.PublicKey()),
+                Key = Key.PublicKeyProto,
                 Scheme = Refs.SignatureScheme.EcdsaRfc6979Sha256,
-                Sign = Key.SignData(response.MetaHeader.ToByteArray())
+                Sign = Key.ECDsaKey.SignData(response.MetaHeader.ToByteArray())
             },
             BodySignature = new Refs.Signature
             {
-                Key = ByteString.CopyFrom(Key.PublicKey()),
+                Key = Key.PublicKeyProto,
                 Scheme = Refs.SignatureScheme.EcdsaRfc6979Sha256,
-                Sign = Key.SignData(response.GetBody().ToByteArray())
+                Sign = Key.ECDsaKey.SignData(response.GetBody().ToByteArray())
             },
             OriginSignature = new Refs.Signature
             {
-                Key = ByteString.CopyFrom(Key.PublicKey()),
+                Key = Key.PublicKeyProto,
                 Scheme = Refs.SignatureScheme.EcdsaRfc6979Sha256,
-                Sign = Key.SignData([])
+                Sign = Key.ECDsaKey.SignData(ReadOnlyMemory<byte>.Empty)
             }
         };
 

src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs

@@ -92,8 +92,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
 
             headResponse.Body.Header.Signature = new Refs.Signature
             {
-                Key = ByteString.CopyFrom(Key.PublicKey()),
+                Key = Key.PublicKeyProto,
-                Sign = Key.SignData(headResponse.Body.Header.ToByteArray()),
+                Sign = Key.ECDsaKey.SignData(headResponse.Body.Header.ToByteArray()),
             };
 
             headResponse.VerifyHeader = GetResponseVerificationHeader(headResponse);

src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs

@@ -55,10 +55,10 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
 
     private async Task RunSuite(IFrostFSClient client, FrostFsContainerId containerId)
     {
-        int[] objectSizes = [1, 257, 6 * 1024, 20 * 1024];
+        int[] objectSizes = [1, 257, 5 * 1024 * 1024, 20 * 1024 * 1024];
 
         string[] objectTypes = [clientCut, serverCut, singleObject];
-
+        
         foreach (var objectSize in objectSizes)
         {
             _testOutputHelper.WriteLine($"test set for object size {objectSize}");
@@ -77,13 +77,13 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
                         break;
                     case clientCut:
                         objectId = await CreateObjectClientCut(client, containerId, bytes);
-                         _testOutputHelper.WriteLine($"\tclient side cut");
+                        _testOutputHelper.WriteLine($"\tclient side cut");
                         break;
                     case singleObject:
                         if (objectSize > 1 * 1024 * 1024)
                             continue;
                         objectId = await PutSingleObject(client, containerId, bytes);
-                         _testOutputHelper.WriteLine($"\tput single object");
+                        _testOutputHelper.WriteLine($"\tput single object");
 
                         break;
                     default: