From 32a7e645381e67a8fa8892dc42bcce87810fb5db Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Fri, 7 Mar 2025 15:29:45 +0300 Subject: [PATCH] [#39] Client: add memory usage optimizations Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/ApeRules/Resources.cs | 2 +- src/FrostFS.SDK.Client/AssemblyInfo.cs | 9 +- .../FrostFS.SDK.Client.csproj | 1 + .../Models/Netmap/FrostFsPlacementPolicy.cs | 2 +- .../Models/Netmap/Placement/Context.cs | 2 +- .../Models/Object/FrostFsObject.cs | 38 +++++--- .../Services/ObjectServiceProvider.cs | 6 +- src/FrostFS.SDK.Client/Tools/ObjectTools.cs | 19 +++- src/FrostFS.SDK.Cryptography/AssemblyInfo.cs | 9 +- src/FrostFS.SDK.Cryptography/Extentions.cs | 23 ++++- .../FrostFS.SDK.Cryptography.csproj | 1 + src/FrostFS.SDK.Cryptography/Key.cs | 4 +- src/FrostFS.SDK.Protos/AssemblyInfo.cs | 9 +- src/FrostFS.SDK.Tests/Unit/ObjectTest.cs | 87 ++++++++----------- 14 files changed, 120 insertions(+), 92 deletions(-) diff --git a/src/FrostFS.SDK.Client/ApeRules/Resources.cs b/src/FrostFS.SDK.Client/ApeRules/Resources.cs index 4f431a6..ef06b4b 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Resources.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Resources.cs @@ -6,7 +6,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable + all diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs index abfce6c..61bdb82 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs @@ -14,7 +14,7 @@ public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replicas) : IEquatable { - private PlacementPolicy policy; + private PlacementPolicy? policy; public FrostFsReplica[] Replicas { get; } = replicas; diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs index ed4ad82..cc5c20a 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs @@ -229,7 +229,7 @@ internal struct Context return [.. result]; } - static double CalcBucketWeight(List ns, IAggregator a, Func wf) + static double CalcBucketWeight(List ns, MeanIQRAgg a, Func wf) { foreach (var node in ns) { diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs index dfd2a22..c39a587 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs @@ -4,7 +4,9 @@ namespace FrostFS.SDK; public class FrostFsObject { - private byte[]? bytes; + // private byte[]? _payloadBytes; + // private ReadOnlyMemory _payloadMemory; + // private bool _isInitPayloadMemory; /// /// Creates new instance from ObjectHeader @@ -45,21 +47,31 @@ public class FrostFsObject /// Reader for received data public IObjectReader? ObjectReader { get; set; } - public byte[] SingleObjectPayload - { - get { return bytes ?? []; } - set { bytes = value; } - } + public ReadOnlyMemory SingleObjectPayload { get; set; } + + //public ReadOnlyMemory SingleObjectPayloadMemory + //{ + // get + // { + // if (!_isInitPayloadMemory) + // { + // _payloadMemory = _payloadBytes.AsMemory(); + // _isInitPayloadMemory = true; + // } + + // return _payloadMemory; + // } + // set + // { + // _payloadMemory = value; + // _isInitPayloadMemory = true; + // } + //} /// - /// The size of payload cannot exceed MaxObjectSize value from NetworkSettings - /// Used only for PutSingleObject method + /// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic /// - /// Buffer for output data - public void SetSingleObjectPayload(byte[] bytes) - { - this.bytes = bytes; - } + public byte[]? PayloadHash { get; set; } /// /// Applied only for the last Object in chain in case of manual multipart uploading diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index b5b5bde..8992733 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -327,10 +327,10 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Body = new() { - Address = address, + Address = address, Patch = new PatchRequest.Types.Body.Types.Patch { - Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount), + Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } } } @@ -481,7 +481,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var obj = new FrostFsObject(partHeader) { - SingleObjectPayload = buffer.Length == size ? buffer : buffer[..size] + SingleObjectPayload = buffer.AsMemory(0, size) }; var prm = new PrmSingleObjectPut(obj); diff --git a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs index ffc33a3..ac5d019 100644 --- a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs +++ b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs @@ -55,7 +55,15 @@ public static class ObjectTools var grpcHeader = @object.Header.GetHeader(); grpcHeader.PayloadLength = (ulong)@object.SingleObjectPayload.Length; - grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload); + + if (@object.PayloadHash != null) + { + grpcHeader.PayloadHash = ChecksumFromSha256(@object.PayloadHash); + } + else + { + grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload); + } var split = @object.Header.Split; @@ -148,6 +156,15 @@ public static class ObjectTools }; } + internal static Checksum Sha256Checksum(ReadOnlyMemory data) + { + return new Checksum + { + Type = ChecksumType.Sha256, + Sum = ByteString.CopyFrom(data.Sha256()) + }; + } + internal static Checksum ChecksumFromSha256(ReadOnlyMemory dataHash) { return new Checksum diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index 7e34e5b..addd4ad 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,11 +1,8 @@ using System.Reflection; -using System.Runtime.InteropServices; [assembly: AssemblyCompany("FrostFS.SDK.Cryptography")] +[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] - -[assembly: AssemblyVersion("1.0.1")] -[assembly: AssemblyFileVersion("1.0.1")] - -[assembly: ComVisible(false)] +[assembly: AssemblyVersion("1.0.2.0")] diff --git a/src/FrostFS.SDK.Cryptography/Extentions.cs b/src/FrostFS.SDK.Cryptography/Extentions.cs index a3f051f..fc51c10 100644 --- a/src/FrostFS.SDK.Cryptography/Extentions.cs +++ b/src/FrostFS.SDK.Cryptography/Extentions.cs @@ -1,6 +1,7 @@ +using System; using System.Security.Cryptography; using System.Threading; - +using CommunityToolkit.HighPerformance; using Org.BouncyCastle.Crypto.Digests; namespace FrostFS.SDK.Cryptography; @@ -35,7 +36,27 @@ public static class Extentions finally { if (lockTaken) + { _spinlockSha256.Exit(false); + } + } + } + + public static byte[] Sha256(this ReadOnlyMemory value) + { + bool lockTaken = false; + try + { + _spinlockSha256.Enter(ref lockTaken); + + return _sha256.ComputeHash(value.AsStream()); + } + finally + { + if (lockTaken) + { + _spinlockSha256.Exit(false); + } } } diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 484bc1a..c9a51d3 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -28,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/FrostFS.SDK.Cryptography/Key.cs b/src/FrostFS.SDK.Cryptography/Key.cs index cf9c3bd..072f155 100644 --- a/src/FrostFS.SDK.Cryptography/Key.cs +++ b/src/FrostFS.SDK.Cryptography/Key.cs @@ -152,7 +152,7 @@ public static class KeyExtension { var secp256R1 = SecNamedCurves.GetByName("secp256r1"); var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)) - .GetEncoded(false).Skip(1); + .GetEncoded(false).Skip(1).ToArray(); var key = ECDsa.Create(new ECParameters { @@ -177,7 +177,7 @@ public static class KeyExtension public static ECDsa LoadPublicKey(this byte[] publicKey) { - var publicKeyFull = publicKey.Decompress().Skip(1); + var publicKeyFull = publicKey.Decompress().Skip(1).ToArray(); var key = ECDsa.Create(new ECParameters { Curve = ECCurve.NamedCurves.nistP256, diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index bc2beee..eb6f873 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,11 +1,8 @@ using System.Reflection; -using System.Runtime.InteropServices; [assembly: AssemblyCompany("FrostFS.SDK.Protos")] +[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] - -[assembly: AssemblyVersion("1.0.1")] -[assembly: AssemblyFileVersion("1.0.1")] - -[assembly: ComVisible(false)] +[assembly: AssemblyVersion("1.0.2.0")] diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs index 564627d..479c6ba 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs @@ -77,68 +77,53 @@ public class ObjectTest : ObjectTestsBase var singleObjects = Mocker.PutSingleRequests.ToArray(); Assert.NotNull(Mocker.ClientStreamWriter?.Messages); - var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray(); - Assert.Single(singleObjects); + var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); - Assert.Equal(11, streamObjects.Length); + Assert.Equal(4, objects.Length); - var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray(); + // linked object + Assert.Equal(0, objects[3].Payload.Length); - Assert.Equal(3, bodies.Count(b => b.Init != null)); - Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024)); + // PART1 + Assert.Equal(blockSize, objects[0].Payload.Length); + Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload)); + + Assert.NotNull(objects[0].Header.Split.SplitId); + Assert.Null(objects[0].Header.Split.Previous); + Assert.True(objects[0].Header.Attributes.Count == 0); + Assert.Null(objects[0].Header.Split.Parent); - Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length); + // PART2 + Assert.Equal(blockSize, objects[1].Payload.Length); + Assert.True(bytes.AsMemory(blockSize, blockSize).ToArray().SequenceEqual(objects[1].Payload)); - var linkObject = singleObjects[0].Body.Object; - var header1 = bodies[0].Init.Header; - var header2 = bodies[4].Init.Header; - var header3 = bodies[8].Init.Header; - - var payload1 = bodies[1].Chunk - .Concat(bodies[2].Chunk) - .Concat(bodies[3].Chunk) - .ToArray(); - - var payload2 = bodies[5].Chunk - .Concat(bodies[6].Chunk) - .Concat(bodies[7].Chunk) - .ToArray(); - - var payload3 = bodies[9].Chunk - .Concat(bodies[10].Chunk) - .ToArray(); - - Assert.NotNull(header1.Split.SplitId); - Assert.Null(header1.Split.Previous); - Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(0, blockSize).ToArray()), SHA256.HashData(payload1)); - Assert.True(header1.Attributes.Count == 0); - - Assert.Equal(header1.Split.SplitId, header2.Split.SplitId); - Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value); - Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(blockSize, blockSize).ToArray()), SHA256.HashData(payload2)); - Assert.True(header2.Attributes.Count == 0); + Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); + Assert.True(objects[1].Header.Attributes.Count == 0); + Assert.Null(objects[1].Header.Split.Parent); // last part - Assert.NotNull(header3.Split.Parent); - Assert.NotNull(header3.Split.ParentHeader); - Assert.NotNull(header3.Split.ParentSignature); - Assert.Equal(header2.Split.SplitId, header3.Split.SplitId); - Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(fileLength - fileLength % blockSize, fileLength % blockSize).ToArray()), SHA256.HashData(payload3)); - Assert.True(header3.Attributes.Count == 0); + Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); + Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload)); - //link object - Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent); - Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader); - Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId); - Assert.Equal(0, (int)linkObject.Header.PayloadLength); - Assert.True(header3.Attributes.Count == 0); + Assert.NotNull(objects[3].Header.Split.Parent); + Assert.NotNull(objects[3].Header.Split.ParentHeader); + Assert.NotNull(objects[3].Header.Split.ParentSignature); + Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); + Assert.True(objects[2].Header.Attributes.Count == 0); - Assert.Single(linkObject.Header.Split.ParentHeader.Attributes); - Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key); - Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value); + // link object + Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent); + Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader); + Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); + Assert.Equal(0, (int)objects[3].Header.PayloadLength); + Assert.True(objects[3].Header.Attributes.Count == 0); - var modelObjId = FrostFsObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray()); + Assert.Single(objects[3].Header.Split.ParentHeader.Attributes); + Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key); + Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value); + + var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray()); Assert.Equal(result.Value, modelObjId.ToString()); }