[#20] Client: Optimize memory usage
Avoid memory allocation, use cache and static Signed-off-by: Pavel Gross <p.gross@yando.com>
This commit is contained in:
Pavel Gross
parent
35fe791406
commit
0ddde467cd
46 changed files with 596 additions and 372 deletions
|
|
@ -7,6 +7,7 @@ using FrostFS.SDK.Cryptography;
|
|||
using FrostFS.SDK.ModelsV2;
|
||||
using FrostFS.SDK.ModelsV2.Enums;
|
||||
using FrostFS.SDK.ModelsV2.Netmap;
|
||||
using FrostFS.SDK.ProtosV2.Interfaces;
|
||||
using Google.Protobuf;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Security.Cryptography;
|
||||
|
|
@ -82,7 +83,7 @@ public class ObjectTest : ObjectTestsBase
|
|||
Assert.NotNull(result);
|
||||
|
||||
Assert.Equal(Mocker.ObjectHeader!.ContainerId.Value, result.Header.ContainerId.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), result.Header.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, result.Header.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader.PayloadLength, result.Header.PayloadLength);
|
||||
Assert.Single(result.Header.Attributes);
|
||||
Assert.Equal(Mocker.ObjectHeader.Attributes[0].Key, result.Header.Attributes[0].Key);
|
||||
|
|
@ -92,7 +93,7 @@ public class ObjectTest : ObjectTestsBase
|
|||
[Fact]
|
||||
public async void PutObjectTest()
|
||||
{
|
||||
Mocker.ResultObjectId = SHA256.HashData([]);
|
||||
Mocker.ResultObjectIds = new([SHA256.HashData([])]);
|
||||
|
||||
Random rnd = new();
|
||||
var bytes = new byte[1024];
|
||||
|
|
@ -113,7 +114,7 @@ public class ObjectTest : ObjectTestsBase
|
|||
var body2 = sentMessages.ElementAt(1).GetBody() as Object.PutRequest.Types.Body;
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(Mocker.ResultObjectId, result.ToHash());
|
||||
Assert.Equal(Mocker.ResultObjectIds.First(), result.ToHash());
|
||||
|
||||
Assert.True(Mocker.ClientStreamWriter.CompletedTask);
|
||||
|
||||
|
|
@ -137,57 +138,85 @@ public class ObjectTest : ObjectTestsBase
|
|||
{
|
||||
Header = Mocker.ObjectHeader,
|
||||
Payload = new MemoryStream(bytes),
|
||||
BufferMaxSize = 1024,
|
||||
ClientCut = true
|
||||
};
|
||||
|
||||
Random rnd = new();
|
||||
|
||||
List<byte[]> objIds = new([new byte[32], new byte[32], new byte[32]]);
|
||||
rnd.NextBytes(objIds.ElementAt(0));
|
||||
rnd.NextBytes(objIds.ElementAt(1));
|
||||
rnd.NextBytes(objIds.ElementAt(2));
|
||||
|
||||
Mocker.ResultObjectIds = objIds;
|
||||
|
||||
var result = await GetClient().PutObjectAsync(param);
|
||||
|
||||
var sentMessages = Mocker.PutSingleRequests.ToArray();
|
||||
var singleObjects = Mocker.PutSingleRequests.ToArray();
|
||||
|
||||
Assert.Equal(4, sentMessages.Length);
|
||||
var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray();
|
||||
|
||||
var object_0 = sentMessages[0].Body.Object;
|
||||
var object_1 = sentMessages[1].Body.Object;
|
||||
var object_2 = sentMessages[2].Body.Object;
|
||||
var object_3 = sentMessages[3].Body.Object;
|
||||
Assert.Single(singleObjects);
|
||||
|
||||
Assert.NotNull(object_0.Header.Split.SplitId);
|
||||
Assert.Null(object_0.Header.Split.Previous);
|
||||
Assert.Equal(blockSize, (int)object_0.Header.PayloadLength);
|
||||
Assert.Equal(bytes[..blockSize], object_0.Payload);
|
||||
Assert.True(object_0.Header.Attributes.Count == 0);
|
||||
Assert.Equal(11, streamObjects.Length);
|
||||
|
||||
Assert.Equal(object_0.Header.Split.SplitId, object_1.Header.Split.SplitId);
|
||||
Assert.Equal(object_0.ObjectId, object_1.Header.Split.Previous);
|
||||
Assert.Equal(blockSize, (int)object_1.Header.PayloadLength);
|
||||
Assert.Equal(bytes[blockSize..(blockSize * 2)], object_1.Payload);
|
||||
Assert.True(object_1.Header.Attributes.Count == 0);
|
||||
var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray();
|
||||
|
||||
Assert.Equal(3, bodies.Count(b => b.Init != null));
|
||||
Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024));
|
||||
|
||||
Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length);
|
||||
|
||||
var linkObject = singleObjects[0].Body.Object;
|
||||
var header1 = bodies[0].Init.Header;
|
||||
var header2 = bodies[4].Init.Header;
|
||||
var header3 = bodies[8].Init.Header;
|
||||
|
||||
var payload1 = bodies[1].Chunk
|
||||
.Concat(bodies[2].Chunk)
|
||||
.Concat(bodies[3].Chunk)
|
||||
.ToArray();
|
||||
|
||||
var payload2 = bodies[5].Chunk
|
||||
.Concat(bodies[6].Chunk)
|
||||
.Concat(bodies[7].Chunk)
|
||||
.ToArray();
|
||||
|
||||
var payload3 = bodies[9].Chunk
|
||||
.Concat(bodies[10].Chunk)
|
||||
.ToArray();
|
||||
|
||||
Assert.NotNull(header1.Split.SplitId);
|
||||
Assert.Null(header1.Split.Previous);
|
||||
Assert.Equal(bytes[..blockSize], payload1);
|
||||
Assert.True(header1.Attributes.Count == 0);
|
||||
|
||||
Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
|
||||
Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
|
||||
Assert.Equal(bytes[blockSize..(blockSize * 2)], payload2);
|
||||
Assert.True(header2.Attributes.Count == 0);
|
||||
|
||||
// last part
|
||||
Assert.NotNull(object_2.Header.Split.Parent);
|
||||
Assert.NotNull(object_2.Header.Split.ParentHeader);
|
||||
Assert.NotNull(object_2.Header.Split.ParentSignature);
|
||||
Assert.Equal(object_1.Header.Split.SplitId, object_2.Header.Split.SplitId);
|
||||
Assert.Equal(object_1.ObjectId, object_2.Header.Split.Previous);
|
||||
Assert.Equal(fileLength%blockSize, (int)object_2.Header.PayloadLength);
|
||||
Assert.Equal(bytes[((fileLength/blockSize) * blockSize)..fileLength], object_2.Payload);
|
||||
Assert.True(object_2.Header.Attributes.Count == 0);
|
||||
Assert.NotNull(header3.Split.Parent);
|
||||
Assert.NotNull(header3.Split.ParentHeader);
|
||||
Assert.NotNull(header3.Split.ParentSignature);
|
||||
Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
|
||||
Assert.Equal(bytes[((fileLength / blockSize) * blockSize)..fileLength], payload3);
|
||||
Assert.True(header3.Attributes.Count == 0);
|
||||
|
||||
// link object
|
||||
Assert.Equal(object_2.Header.Split.Parent, object_3.Header.Split.Parent);
|
||||
Assert.Equal(object_2.Header.Split.ParentHeader, object_3.Header.Split.ParentHeader);
|
||||
Assert.Equal(object_2.Header.Split.SplitId, object_3.Header.Split.SplitId);
|
||||
Assert.Equal(0, (int)object_3.Header.PayloadLength);
|
||||
Assert.Contains(object_0.ObjectId, object_3.Header.Split.Children);
|
||||
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
|
||||
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
|
||||
Assert.True(object_2.Header.Attributes.Count == 0);
|
||||
//link object
|
||||
Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent);
|
||||
Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader);
|
||||
Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId);
|
||||
Assert.Equal(0, (int)linkObject.Header.PayloadLength);
|
||||
Assert.True(header3.Attributes.Count == 0);
|
||||
|
||||
Assert.Single(object_3.Header.Split.ParentHeader.Attributes);
|
||||
Assert.Equal("k", object_3.Header.Split.ParentHeader.Attributes[0].Key);
|
||||
Assert.Equal("v", object_3.Header.Split.ParentHeader.Attributes[0].Value);
|
||||
Assert.Single(linkObject.Header.Split.ParentHeader.Attributes);
|
||||
Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key);
|
||||
Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value);
|
||||
|
||||
var modelObjId = ObjectId.FromHash(object_3.Header.Split.Parent.Value.ToByteArray());
|
||||
var modelObjId = ObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray());
|
||||
|
||||
Assert.Equal(result.Value, modelObjId.ToString());
|
||||
}
|
||||
|
|
@ -201,8 +230,8 @@ public class ObjectTest : ObjectTestsBase
|
|||
|
||||
var request = Mocker.DeleteRequests.FirstOrDefault();
|
||||
Assert.NotNull(request);
|
||||
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
|
||||
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
|
||||
Assert.Equal(ContainerId.ToMessage().Value, request.Body.Address.ContainerId.Value);
|
||||
Assert.Equal(Mocker.ObjectId.ToMessage().Value, request.Body.Address.ObjectId.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
@ -214,13 +243,13 @@ public class ObjectTest : ObjectTestsBase
|
|||
|
||||
var request = Mocker.HeadRequests.FirstOrDefault();
|
||||
Assert.NotNull(request);
|
||||
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
|
||||
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
|
||||
Assert.Equal(ContainerId.ToMessage(), request.Body.Address.ContainerId);
|
||||
Assert.Equal(Mocker.ObjectId.ToMessage(), request.Body.Address.ObjectId);
|
||||
|
||||
Assert.NotNull(response);
|
||||
Assert.Equal(ContainerId.Value, response.ContainerId.Value);
|
||||
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), response.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, response.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.Version!.ToString(), response.Version!.ToString());
|
||||
|
||||
Assert.Equal(Mocker.HeadResponse!.PayloadLength, response.PayloadLength);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue