[#39] Client: add memory usage optimizations

Signed-off-by: Pavel Gross <p.gross@yadro.com>

src/FrostFS.SDK.Client/ApeRules/Resources.cs

@@ -6,7 +6,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable<Resou
 
     public string[] Names { get; set; } = names;
 
-    public override bool Equals(object obj)
+    public override readonly bool Equals(object obj)
     {
          if (obj == null || obj is not Resources)
             return false;

src/FrostFS.SDK.Client/AssemblyInfo.cs

@@ -1,11 +1,8 @@
 using System.Reflection;
-using System.Runtime.InteropServices;
 
 [assembly: AssemblyCompany("FrostFS.SDK.Client")]
+[assembly: AssemblyFileVersion("1.0.2.0")]
+[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
 [assembly: AssemblyProduct("FrostFS.SDK.Client")]
 [assembly: AssemblyTitle("FrostFS.SDK.Client")]
-
+[assembly: AssemblyVersion("1.0.2")]
-[assembly: AssemblyVersion("1.0.1")]
-[assembly: AssemblyFileVersion("1.0.1")]
-
-[assembly: ComVisible(false)]

src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj

@@ -32,6 +32,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
       <PrivateAssets>all</PrivateAssets>

src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs

@@ -14,7 +14,7 @@ public struct FrostFsPlacementPolicy(bool unique,
     params FrostFsReplica[] replicas)
     : IEquatable<FrostFsPlacementPolicy>
 {
-    private PlacementPolicy policy;
+    private PlacementPolicy? policy;
 
     public FrostFsReplica[] Replicas { get; } = replicas;
 

src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs

@@ -229,7 +229,7 @@ internal struct Context
         return [.. result];
     }
 
-    static double CalcBucketWeight(List<FrostFsNodeInfo> ns, IAggregator a, Func<FrostFsNodeInfo, double> wf)
+    static double CalcBucketWeight(List<FrostFsNodeInfo> ns, MeanIQRAgg a, Func<FrostFsNodeInfo, double> wf)
     {
         foreach (var node in ns)
         {

src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs

@@ -4,7 +4,9 @@ namespace FrostFS.SDK;
 
 public class FrostFsObject
 {
-    private byte[]? bytes;
+  //  private byte[]? _payloadBytes;
+  //  private ReadOnlyMemory<byte> _payloadMemory;
+  //  private bool _isInitPayloadMemory;
 
     /// <summary>
     /// Creates new instance from <c>ObjectHeader</c>
@@ -45,21 +47,31 @@ public class FrostFsObject
     /// <value>Reader for received data</value>
     public IObjectReader? ObjectReader { get; set; }
 
-    public byte[] SingleObjectPayload
+    public ReadOnlyMemory<byte> SingleObjectPayload { get; set; }
-    {
+
-        get { return bytes ?? []; }
+    //public ReadOnlyMemory<byte> SingleObjectPayloadMemory
-        set { bytes = value; }
+    //{
-    }
+    //    get
+    //    {
+    //        if (!_isInitPayloadMemory)
+    //        {
+    //            _payloadMemory = _payloadBytes.AsMemory();
+    //            _isInitPayloadMemory = true;
+    //        }
+
+    //        return _payloadMemory;
+    //    }
+    //    set
+    //    {
+    //        _payloadMemory = value;
+    //        _isInitPayloadMemory = true;
+    //    }
+    //}
 
     /// <summary>
-    /// The size of payload cannot exceed <c>MaxObjectSize</c> value from <c>NetworkSettings</c>
+    /// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic
-    /// Used only for PutSingleObject method
     /// </summary>
-    /// <value>Buffer for output data</value>
+    public byte[]? PayloadHash { get; set; }
-    public void SetSingleObjectPayload(byte[] bytes)
-    {
-        this.bytes = bytes;
-    }
 
     /// <summary>
     /// Applied only for the last Object in chain in case of manual multipart uploading

src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs

@@ -327,10 +327,10 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
                 {
                     Body = new()
                     {
-                        Address = address,
+                        Address = address,  
                         Patch = new PatchRequest.Types.Body.Types.Patch
                         {
-                            Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount),
+                            Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)),
                             SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount }
                         }
                     }
@@ -481,7 +481,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
 
                 var obj = new FrostFsObject(partHeader)
                 {
-                    SingleObjectPayload = buffer.Length == size ? buffer : buffer[..size]
+                    SingleObjectPayload = buffer.AsMemory(0, size)
                 };
 
                 var prm = new PrmSingleObjectPut(obj);

src/FrostFS.SDK.Client/Tools/ObjectTools.cs

@@ -55,7 +55,15 @@ public static class ObjectTools
         var grpcHeader = @object.Header.GetHeader();
 
         grpcHeader.PayloadLength = (ulong)@object.SingleObjectPayload.Length;
-        grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload);
+
+        if (@object.PayloadHash != null)
+        {
+            grpcHeader.PayloadHash = ChecksumFromSha256(@object.PayloadHash);
+        }
+        else
+        {
+            grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload);
+        }
 
         var split = @object.Header.Split;
 
@@ -148,6 +156,15 @@ public static class ObjectTools
         };
     }
 
+    internal static Checksum Sha256Checksum(ReadOnlyMemory<byte> data)
+    {
+        return new Checksum
+        {
+            Type = ChecksumType.Sha256,
+            Sum = ByteString.CopyFrom(data.Sha256())
+        };
+    }
+
     internal static Checksum ChecksumFromSha256(ReadOnlyMemory<byte> dataHash)
     {
         return new Checksum

src/FrostFS.SDK.Cryptography/AssemblyInfo.cs

@@ -1,11 +1,8 @@
 using System.Reflection;
-using System.Runtime.InteropServices;
 
 [assembly: AssemblyCompany("FrostFS.SDK.Cryptography")]
+[assembly: AssemblyFileVersion("1.0.2.0")]
+[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
 [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")]
 [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")]
-
+[assembly: AssemblyVersion("1.0.2.0")]
-[assembly: AssemblyVersion("1.0.1")]
-[assembly: AssemblyFileVersion("1.0.1")]
-
-[assembly: ComVisible(false)]

src/FrostFS.SDK.Cryptography/Extentions.cs

@@ -1,6 +1,7 @@
+using System;
 using System.Security.Cryptography;
 using System.Threading;
-
+using CommunityToolkit.HighPerformance;
 using Org.BouncyCastle.Crypto.Digests;
 
 namespace FrostFS.SDK.Cryptography;
@@ -35,7 +36,27 @@ public static class Extentions
         finally
         {
             if (lockTaken)
+            {
                 _spinlockSha256.Exit(false);
+            }
+        }
+    }
+
+    public static byte[] Sha256(this ReadOnlyMemory<byte> value)
+    {
+        bool lockTaken = false;
+        try
+        {
+            _spinlockSha256.Enter(ref lockTaken);
+
+            return _sha256.ComputeHash(value.AsStream());
+        }
+        finally
+        {
+            if (lockTaken)
+            {
+                _spinlockSha256.Exit(false);
+            }
         }
     }
 

src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj

@@ -28,6 +28,7 @@
 
   <ItemGroup>
     <PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
+    <PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/FrostFS.SDK.Cryptography/Key.cs

@@ -152,7 +152,7 @@ public static class KeyExtension
     {
         var secp256R1 = SecNamedCurves.GetByName("secp256r1");
         var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
-            .GetEncoded(false).Skip(1);
+            .GetEncoded(false).Skip(1).ToArray();
 
         var key = ECDsa.Create(new ECParameters
         {
@@ -177,7 +177,7 @@ public static class KeyExtension
 
     public static ECDsa LoadPublicKey(this byte[] publicKey)
     {
-        var publicKeyFull = publicKey.Decompress().Skip(1);
+        var publicKeyFull = publicKey.Decompress().Skip(1).ToArray();
         var key = ECDsa.Create(new ECParameters
         {
             Curve = ECCurve.NamedCurves.nistP256,

src/FrostFS.SDK.Protos/AssemblyInfo.cs

@@ -1,11 +1,8 @@
 using System.Reflection;
-using System.Runtime.InteropServices;
 
 [assembly: AssemblyCompany("FrostFS.SDK.Protos")]
+[assembly: AssemblyFileVersion("1.0.2.0")]
+[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
 [assembly: AssemblyProduct("FrostFS.SDK.Protos")]
 [assembly: AssemblyTitle("FrostFS.SDK.Protos")]
-
+[assembly: AssemblyVersion("1.0.2.0")]
-[assembly: AssemblyVersion("1.0.1")]
-[assembly: AssemblyFileVersion("1.0.1")]
-
-[assembly: ComVisible(false)]

src/FrostFS.SDK.Tests/Unit/ObjectTest.cs

@@ -77,68 +77,53 @@ public class ObjectTest : ObjectTestsBase
         var singleObjects = Mocker.PutSingleRequests.ToArray();
 
         Assert.NotNull(Mocker.ClientStreamWriter?.Messages);
-        var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray();
 
-        Assert.Single(singleObjects);
+        var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray();
 
-        Assert.Equal(11, streamObjects.Length);
+        Assert.Equal(4, objects.Length);
 
-        var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray();
+        // linked object
+        Assert.Equal(0, objects[3].Payload.Length);
 
-        Assert.Equal(3, bodies.Count(b => b.Init != null));
+        // PART1        
-        Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024));
+        Assert.Equal(blockSize, objects[0].Payload.Length);
+        Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload));
+      
+        Assert.NotNull(objects[0].Header.Split.SplitId);
+        Assert.Null(objects[0].Header.Split.Previous);
+        Assert.True(objects[0].Header.Attributes.Count == 0);
+        Assert.Null(objects[0].Header.Split.Parent);
 
-        Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length);
+        // PART2
+        Assert.Equal(blockSize, objects[1].Payload.Length);
+        Assert.True(bytes.AsMemory(blockSize, blockSize).ToArray().SequenceEqual(objects[1].Payload));
 
-        var linkObject = singleObjects[0].Body.Object;
+        Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId);
-        var header1 = bodies[0].Init.Header;
+        Assert.True(objects[1].Header.Attributes.Count == 0);
-        var header2 = bodies[4].Init.Header;
+        Assert.Null(objects[1].Header.Split.Parent);
-        var header3 = bodies[8].Init.Header;
-
-        var payload1 = bodies[1].Chunk
-            .Concat(bodies[2].Chunk)
-            .Concat(bodies[3].Chunk)
-            .ToArray();
-
-        var payload2 = bodies[5].Chunk
-           .Concat(bodies[6].Chunk)
-           .Concat(bodies[7].Chunk)
-           .ToArray();
-
-        var payload3 = bodies[9].Chunk
-           .Concat(bodies[10].Chunk)
-           .ToArray();
-
-        Assert.NotNull(header1.Split.SplitId);
-        Assert.Null(header1.Split.Previous);
-        Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(0, blockSize).ToArray()), SHA256.HashData(payload1));
-        Assert.True(header1.Attributes.Count == 0);
-
-        Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
-        Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
-        Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(blockSize, blockSize).ToArray()), SHA256.HashData(payload2));
-        Assert.True(header2.Attributes.Count == 0);
 
         // last part
-        Assert.NotNull(header3.Split.Parent);
+        Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length);
-        Assert.NotNull(header3.Split.ParentHeader);
+        Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload));
-        Assert.NotNull(header3.Split.ParentSignature);
-        Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
-        Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(fileLength - fileLength % blockSize, fileLength % blockSize).ToArray()), SHA256.HashData(payload3));
-        Assert.True(header3.Attributes.Count == 0);
 
-        //link object
+        Assert.NotNull(objects[3].Header.Split.Parent);
-        Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent);
+        Assert.NotNull(objects[3].Header.Split.ParentHeader);
-        Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader);
+        Assert.NotNull(objects[3].Header.Split.ParentSignature);
-        Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId);
+        Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId);
-        Assert.Equal(0, (int)linkObject.Header.PayloadLength);
+        Assert.True(objects[2].Header.Attributes.Count == 0);
-        Assert.True(header3.Attributes.Count == 0);
 
-        Assert.Single(linkObject.Header.Split.ParentHeader.Attributes);
+        // link object
-        Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key);
+        Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent);
-        Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value);
+        Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader);
+        Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId);
+        Assert.Equal(0, (int)objects[3].Header.PayloadLength);
+        Assert.True(objects[3].Header.Attributes.Count == 0);
 
-        var modelObjId = FrostFsObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray());
+        Assert.Single(objects[3].Header.Split.ParentHeader.Attributes);
+        Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key);
+        Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value);
+
+        var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray());
 
         Assert.Equal(result.Value, modelObjId.ToString());
     }