TrueCloudLab/frostfs-sdk-csharp
18
0
Fork 5
Code Issues 3 Pull requests Projects Releases Packages 3 Activity Actions

[#39] Client: add memory usage optimizations
All checks were successful
DCO / DCO (pull_request) Successful in 24s
Details
lint-build / dotnet8.0 (pull_request) Successful in 41s
Details
lint-build / dotnet8.0 (push) Successful in 55s
Details

Browse source 
Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2025-03-07 15:29:45 +03:00
parent d6fe034453
commit 32a7e64538
14 changed files with 120 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FrostFS.SDK.Client/ApeRules/Resources.cs
View file

@ -6,7 +6,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable<Resou
public string[] Names { get; set; } = names;
public override bool Equals(object obj)
public override readonly bool Equals(object obj)
{
if (obj == null || obj is not Resources)
return false;

9
src/FrostFS.SDK.Client/AssemblyInfo.cs
View file

@ -1,11 +1,8 @@
using System.Reflection;
using System.Runtime.InteropServices;
[assembly: AssemblyCompany("FrostFS.SDK.Client")]
[assembly: AssemblyFileVersion("1.0.2.0")]
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
[assembly: AssemblyProduct("FrostFS.SDK.Client")]
[assembly: AssemblyTitle("FrostFS.SDK.Client")]
[assembly: AssemblyVersion("1.0.1")]
[assembly: AssemblyFileVersion("1.0.1")]
[assembly: ComVisible(false)]
[assembly: AssemblyVersion("1.0.2")]

1
src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj
View file

@ -32,6 +32,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
<PrivateAssets>all</PrivateAssets>

2
src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs
View file

@ -14,7 +14,7 @@ public struct FrostFsPlacementPolicy(bool unique,
params FrostFsReplica[] replicas)
: IEquatable<FrostFsPlacementPolicy>
{
private PlacementPolicy policy;
private PlacementPolicy? policy;
public FrostFsReplica[] Replicas { get; } = replicas;

2
src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs
View file

@ -229,7 +229,7 @@ internal struct Context
return [.. result];
}
static double CalcBucketWeight(List<FrostFsNodeInfo> ns, IAggregator a, Func<FrostFsNodeInfo, double> wf)
static double CalcBucketWeight(List<FrostFsNodeInfo> ns, MeanIQRAgg a, Func<FrostFsNodeInfo, double> wf)
{
foreach (var node in ns)
{

38
src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs
View file

@ -4,7 +4,9 @@ namespace FrostFS.SDK;
public class FrostFsObject
{
private byte[]? bytes;
// private byte[]? _payloadBytes;
// private ReadOnlyMemory<byte> _payloadMemory;
// private bool _isInitPayloadMemory;
/// <summary>
/// Creates new instance from <c>ObjectHeader</c>
@ -45,21 +47,31 @@ public class FrostFsObject
/// <value>Reader for received data</value>
public IObjectReader? ObjectReader { get; set; }
public byte[] SingleObjectPayload
{
get { return bytes ?? []; }
set { bytes = value; }
}
public ReadOnlyMemory<byte> SingleObjectPayload { get; set; }
//public ReadOnlyMemory<byte> SingleObjectPayloadMemory
//{
// get
// {
// if (!_isInitPayloadMemory)
// {
// _payloadMemory = _payloadBytes.AsMemory();
// _isInitPayloadMemory = true;
// }
// return _payloadMemory;
// }
// set
// {
// _payloadMemory = value;
// _isInitPayloadMemory = true;
// }
//}
/// <summary>
/// The size of payload cannot exceed <c>MaxObjectSize</c> value from <c>NetworkSettings</c>
/// Used only for PutSingleObject method
/// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic
/// </summary>
/// <value>Buffer for output data</value>
public void SetSingleObjectPayload(byte[] bytes)
{
this.bytes = bytes;
}
public byte[]? PayloadHash { get; set; }
/// <summary>
/// Applied only for the last Object in chain in case of manual multipart uploading

4
src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs
View file

@ -330,7 +330,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
Address = address,
Patch = new PatchRequest.Types.Body.Types.Patch
{
Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount),
Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)),
SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount }
}
}
@ -481,7 +481,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
var obj = new FrostFsObject(partHeader)
{
SingleObjectPayload = buffer.Length == size ? buffer : buffer[..size]
SingleObjectPayload = buffer.AsMemory(0, size)
};
var prm = new PrmSingleObjectPut(obj);

17
src/FrostFS.SDK.Client/Tools/ObjectTools.cs
View file

@ -55,7 +55,15 @@ public static class ObjectTools
var grpcHeader = @object.Header.GetHeader();
grpcHeader.PayloadLength = (ulong)@object.SingleObjectPayload.Length;
if (@object.PayloadHash != null)
{
grpcHeader.PayloadHash = ChecksumFromSha256(@object.PayloadHash);
}
else
{
grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload);
}
var split = @object.Header.Split;
@ -148,6 +156,15 @@ public static class ObjectTools
};
}
internal static Checksum Sha256Checksum(ReadOnlyMemory<byte> data)
{
return new Checksum
{
Type = ChecksumType.Sha256,
Sum = ByteString.CopyFrom(data.Sha256())
};
}
internal static Checksum ChecksumFromSha256(ReadOnlyMemory<byte> dataHash)
{
return new Checksum

9
src/FrostFS.SDK.Cryptography/AssemblyInfo.cs
View file

@ -1,11 +1,8 @@
using System.Reflection;
using System.Runtime.InteropServices;
[assembly: AssemblyCompany("FrostFS.SDK.Cryptography")]
[assembly: AssemblyFileVersion("1.0.2.0")]
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
[assembly: AssemblyProduct("FrostFS.SDK.Cryptography")]
[assembly: AssemblyTitle("FrostFS.SDK.Cryptography")]
[assembly: AssemblyVersion("1.0.1")]
[assembly: AssemblyFileVersion("1.0.1")]
[assembly: ComVisible(false)]
[assembly: AssemblyVersion("1.0.2.0")]

23
src/FrostFS.SDK.Cryptography/Extentions.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Security.Cryptography;
using System.Threading;
using CommunityToolkit.HighPerformance;
using Org.BouncyCastle.Crypto.Digests;
namespace FrostFS.SDK.Cryptography;
@ -35,9 +36,29 @@ public static class Extentions
finally
{
if (lockTaken)
{
_spinlockSha256.Exit(false);
}
}
}
public static byte[] Sha256(this ReadOnlyMemory<byte> value)
{
bool lockTaken = false;
try
{
_spinlockSha256.Enter(ref lockTaken);
return _sha256.ComputeHash(value.AsStream());
}
finally
{
if (lockTaken)
{
_spinlockSha256.Exit(false);
}
}
}
public static byte[] Sha512(this byte[] value)
{

1
src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj
View file

@ -28,6 +28,7 @@
<ItemGroup>
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
<PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

4
src/FrostFS.SDK.Cryptography/Key.cs
View file

@ -152,7 +152,7 @@ public static class KeyExtension
{
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
.GetEncoded(false).Skip(1);
.GetEncoded(false).Skip(1).ToArray();
var key = ECDsa.Create(new ECParameters
{
@ -177,7 +177,7 @@ public static class KeyExtension
public static ECDsa LoadPublicKey(this byte[] publicKey)
{
var publicKeyFull = publicKey.Decompress().Skip(1);
var publicKeyFull = publicKey.Decompress().Skip(1).ToArray();
var key = ECDsa.Create(new ECParameters
{
Curve = ECCurve.NamedCurves.nistP256,

9
src/FrostFS.SDK.Protos/AssemblyInfo.cs
View file

@ -1,11 +1,8 @@
using System.Reflection;
using System.Runtime.InteropServices;
[assembly: AssemblyCompany("FrostFS.SDK.Protos")]
[assembly: AssemblyFileVersion("1.0.2.0")]
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
[assembly: AssemblyProduct("FrostFS.SDK.Protos")]
[assembly: AssemblyTitle("FrostFS.SDK.Protos")]
[assembly: AssemblyVersion("1.0.1")]
[assembly: AssemblyFileVersion("1.0.1")]
[assembly: ComVisible(false)]
[assembly: AssemblyVersion("1.0.2.0")]

85
src/FrostFS.SDK.Tests/Unit/ObjectTest.cs
View file

@ -77,68 +77,53 @@ public class ObjectTest : ObjectTestsBase
var singleObjects = Mocker.PutSingleRequests.ToArray();
Assert.NotNull(Mocker.ClientStreamWriter?.Messages);
var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray();
Assert.Single(singleObjects);
var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray();
Assert.Equal(11, streamObjects.Length);
Assert.Equal(4, objects.Length);
var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray();
// linked object
Assert.Equal(0, objects[3].Payload.Length);
Assert.Equal(3, bodies.Count(b => b.Init != null));
Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024));
// PART1
Assert.Equal(blockSize, objects[0].Payload.Length);
Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload));
Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length);
Assert.NotNull(objects[0].Header.Split.SplitId);
Assert.Null(objects[0].Header.Split.Previous);
Assert.True(objects[0].Header.Attributes.Count == 0);
Assert.Null(objects[0].Header.Split.Parent);
var linkObject = singleObjects[0].Body.Object;
var header1 = bodies[0].Init.Header;
var header2 = bodies[4].Init.Header;
var header3 = bodies[8].Init.Header;
// PART2
Assert.Equal(blockSize, objects[1].Payload.Length);
Assert.True(bytes.AsMemory(blockSize, blockSize).ToArray().SequenceEqual(objects[1].Payload));
var payload1 = bodies[1].Chunk
.Concat(bodies[2].Chunk)
.Concat(bodies[3].Chunk)
.ToArray();
var payload2 = bodies[5].Chunk
.Concat(bodies[6].Chunk)
.Concat(bodies[7].Chunk)
.ToArray();
var payload3 = bodies[9].Chunk
.Concat(bodies[10].Chunk)
.ToArray();
Assert.NotNull(header1.Split.SplitId);
Assert.Null(header1.Split.Previous);
Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(0, blockSize).ToArray()), SHA256.HashData(payload1));
Assert.True(header1.Attributes.Count == 0);
Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(blockSize, blockSize).ToArray()), SHA256.HashData(payload2));
Assert.True(header2.Attributes.Count == 0);
Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId);
Assert.True(objects[1].Header.Attributes.Count == 0);
Assert.Null(objects[1].Header.Split.Parent);
// last part
Assert.NotNull(header3.Split.Parent);
Assert.NotNull(header3.Split.ParentHeader);
Assert.NotNull(header3.Split.ParentSignature);
Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(fileLength - fileLength % blockSize, fileLength % blockSize).ToArray()), SHA256.HashData(payload3));
Assert.True(header3.Attributes.Count == 0);
Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length);
Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload));
//link object
Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent);
Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader);
Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId);
Assert.Equal(0, (int)linkObject.Header.PayloadLength);
Assert.True(header3.Attributes.Count == 0);
Assert.NotNull(objects[3].Header.Split.Parent);
Assert.NotNull(objects[3].Header.Split.ParentHeader);
Assert.NotNull(objects[3].Header.Split.ParentSignature);
Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId);
Assert.True(objects[2].Header.Attributes.Count == 0);
Assert.Single(linkObject.Header.Split.ParentHeader.Attributes);
Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key);
Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value);
// link object
Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent);
Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader);
Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId);
Assert.Equal(0, (int)objects[3].Header.PayloadLength);
Assert.True(objects[3].Header.Attributes.Count == 0);
var modelObjId = FrostFsObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray());
Assert.Single(objects[3].Header.Split.ParentHeader.Attributes);
Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key);
Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value);
var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray());
Assert.Equal(result.Value, modelObjId.ToString());
}