KurlesHS/frostfs-sdk-csharp
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-csharp
Code Pull requests Activity

[#20] Optimize memory usage

Browse source 
Provide custom buffer and use ArrayPool

Signed-off-by: Pavel Gross <p.gross@yando.com>
This commit is contained in:
Pavel Gross 2024-08-05 11:21:05 +03:00
parent 6083834582
commit 18126ea763
3 changed files with 76 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

88
src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs
View file

@ -306,46 +306,68 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
var restBytes = args.FullLength - args.CurrentStreamPosition;
chunkSize = (int)Math.Min(restBytes, (ulong)chunkSize);
bool isRentBuffer = false;
byte[]? chunkBuffer = null;
var chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
var sentBytes = 0;
// 0 means no limit from client, so server side cut is performed
var objectLimitSize = args.ClientCut ? args.MaxObjectSizeCache : 0;
var stream = await GetUploadStream(args, ctx);
while (objectLimitSize == 0 || sentBytes < objectLimitSize)
try
{
// send chanks limited to default or user's settings
var bufferSize = objectLimitSize > 0 ?
(int)Math.Min(objectLimitSize - sentBytes, chunkSize)
: chunkSize;
var bytesCount = await payload.ReadAsync(chunkBuffer, 0, bufferSize, ctx.CancellationToken);
if (bytesCount == 0)
break;
sentBytes += bytesCount;
var chunkRequest = new PutRequest
if (args.CustomBuffer != null)
{
Body = new PutRequest.Types.Body
chunkBuffer = args.CustomBuffer;
}
else
{
chunkBuffer = env.GetArrayPool(Constants.ObjectChunkSize).Rent(chunkSize);
isRentBuffer = true;
}
var sentBytes = 0;
// 0 means no limit from client, so server side cut is performed
var objectLimitSize = args.ClientCut ? args.MaxObjectSizeCache : 0;
var stream = await GetUploadStream(args, ctx);
while (objectLimitSize == 0 || sentBytes < objectLimitSize)
{
// send chanks limited to default or user's settings
var bufferSize = objectLimitSize > 0 ?
(int)Math.Min(objectLimitSize - sentBytes, chunkSize)
: chunkSize;
var bytesCount = await payload.ReadAsync(chunkBuffer, 0, bufferSize, ctx.CancellationToken);
if (bytesCount == 0)
break;
sentBytes += bytesCount;
var chunkRequest = new PutRequest
{
Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount)
}
};
Body = new PutRequest.Types.Body
{
Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount)
}
};
chunkRequest.Sign(Context.Key.ECDsaKey);
chunkRequest.Sign(Context.Key.ECDsaKey);
await stream.Write(chunkRequest);
}
await stream.Write(chunkRequest);
var response = await stream.Close();
Verifier.CheckResponse(response);
return new PutObjectResult(ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()), sentBytes);
}
finally
{
if (isRentBuffer && chunkBuffer != null)
{
ArrayPool<byte>.Shared.Return(chunkBuffer);
}
}
var response = await stream.Close();
Verifier.CheckResponse(response);
return new PutObjectResult(ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()), sentBytes);
}
private async Task<ObjectStreamer> GetUploadStream(PrmObjectPut args, Context ctx)