[] Client cut internal

This commit is contained in:
Pavel Gross 2024-06-14 11:58:29 +03:00 committed by PavelGrossSpb
parent 545e647d7b
commit b69d22966f
15 changed files with 216 additions and 36 deletions
FrostFS.SDK.slnREADME.md
src
FrostFS.SDK.ClientV2
FrostFS.SDK.Cryptography
FrostFS.SDK.ModelsV2
FrostFS.SDK.ProtosV2/object
FrostFS.SDK.Tests

View file

@ -1,4 +1,7 @@
Microsoft Visual Studio Solution File, Format Version 12.00
#
VisualStudioVersion = 17.5.002.0
MinimumVisualStudioVersion =
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.ClientV2", "src\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj", "{50D8F61F-C302-4AC9-8D8A-AB0B8C0988C3}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.Cryptography", "src\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj", "{3D804F4A-B0B2-47A5-B006-BE447BE64B50}"

View file

@ -121,7 +121,7 @@ static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, Contain
largeObject.AppendBlock(buffer, bytesCount);
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, buffer)
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
.AddAttribute(fileNameAttribute)
.SetSplit(split);
@ -134,15 +134,19 @@ static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, Contain
if (sentObjectIds.Any())
{
largeObject.CalculateHash();
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds);
_ = await fsClient.PutSingleObjectAsync(linkObject);
largeObject.CalculateHash()
.AddAttribute(fileNameAttribute);
currentObject.SetParent(largeObject);
_ = await fsClient.PutSingleObjectAsync(currentObject);
var objectId = await fsClient.PutSingleObjectAsync(currentObject);
sentObjectIds.Add(objectId);
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds)
.AddAttribute(fileNameAttribute);
_ = await fsClient.PutSingleObjectAsync(linkObject);
return currentObject.GetParentId();
}

View file

@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using FrostFS.Container;
using FrostFS.Netmap;
@ -10,6 +12,7 @@ using FrostFS.SDK.ModelsV2;
using FrostFS.Session;
using Grpc.Core;
using Grpc.Net.Client;
using static FrostFS.Netmap.NetworkConfig.Types;
using Version = FrostFS.SDK.ModelsV2.Version;
namespace FrostFS.SDK.ClientV2;
@ -21,6 +24,8 @@ public partial class Client: IFrostFSClient
public readonly OwnerId OwnerId;
public readonly Version Version = new(2, 13);
private readonly Dictionary<string, ulong> NetworkSettings = [];
private ContainerService.ContainerServiceClient? _containerServiceClient;
private NetmapService.NetmapServiceClient? _netmapServiceClient;
private ObjectService.ObjectServiceClient? _objectServiceClient;
@ -42,6 +47,33 @@ public partial class Client: IFrostFSClient
InitObjectClient();
InitSessionClient();
CheckFrostFsVersionSupport();
InitNetworkInfoAsync();
}
private async void InitNetworkInfoAsync()
{
var info = await GetNetworkInfoAsync();
foreach (var param in info.Body.NetworkInfo.NetworkConfig.Parameters)
{
SetNetworksParam(param);
}
}
private void SetNetworksParam(Parameter param)
{
var key = Encoding.UTF8.GetString(param.Key.ToByteArray());
var encodedValue = param.Value.ToByteArray();
ulong val = 0;
for (var i = encodedValue.Length - 1; i >= 0; i--)
{
val = (val << 8) + encodedValue[i];
}
NetworkSettings.Add(key, val);
}
private async void CheckFrostFsVersionSupport()
@ -84,7 +116,11 @@ public partial class Client: IFrostFSClient
throw new ArgumentException(msg);
}
_channel = GrpcChannel.ForAddress(uri, new GrpcChannelOptions { Credentials = grpcCredentials });
_channel = GrpcChannel.ForAddress(uri, new GrpcChannelOptions
{
Credentials = grpcCredentials,
HttpHandler = new System.Net.Http.HttpClientHandler()
});
}
private void InitContainerClient()

View file

@ -38,7 +38,7 @@ public static class Extensions
public static LinkObject AddChildren(this LinkObject linkObject, IEnumerable<ObjectId> objectIds)
{
linkObject.Header.Split.Children.AddRange(objectIds);
linkObject.Header.Split!.Children.AddRange(objectIds);
return linkObject;
}
}

View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FrostFS.SDK.ModelsV2;
@ -18,13 +19,14 @@ public interface IFrostFSClient
Task<ObjectHeader> GetObjectHeadAsync(ContainerId containerId, ObjectId objectId);
Task<ModelsV2.Object> GetObjectAsync(ContainerId containerId, ObjectId objectId);
Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload);
Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload, CancellationToken cancellationToken = default);
Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload);
Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload, CancellationToken cancellationToken = default);
Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj);
Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj, CancellationToken cancellationToken = default);
Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId);

View file

@ -103,15 +103,31 @@ public static class ObjectHeaderMapper
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
};
return new ObjectHeader(
var model = new ObjectHeader(
new ContainerId(Base58.Encode(header.ContainerId.Value.ToByteArray())),
objTypeName,
header.Attributes.Select(attribute => attribute.ToModel()).ToArray()
)
{
PayloadLength = header.PayloadLength,
Version = header.Version.ToModel()
Version = header.Version.ToModel(),
OwnerId = header.OwnerId.ToModel()
};
if (header.Split != null)
{
model.Split = new Split(SplitId.CrateFromBinary(header.Split.SplitId.ToByteArray()))
{
Parent = header.Split.Parent?.ToModel(),
ParentHeader = header.Split.ParentHeader?.ToModel(),
Previous = header.Split.Previous?.ToModel()
};
if (header.Split.Children.Any())
model.Split.Children.AddRange(header.Split.Children.Select(x => x.ToModel()));
}
return model;
}
}
@ -148,4 +164,3 @@ public static class SignatureMapper
};
}
}

View file

@ -13,4 +13,9 @@ public static class OwnerIdMapper
Value = ByteString.CopyFrom(ownerId.ToHash())
};
}
public static OwnerId ToModel(this OwnerID ownerId)
{
return new OwnerId(ownerId.ToString());
}
}

View file

@ -13,6 +13,8 @@ using FrostFS.SDK.Cryptography;
using FrostFS.Session;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ClientV2.Extensions;
using System.Threading;
namespace FrostFS.SDK.ClientV2;
@ -32,6 +34,7 @@ public partial class Client
}
};
request.AddMetaHeader();
request.Sign(_key);
var response = await _objectServiceClient!.HeadAsync(request);
@ -70,18 +73,83 @@ public partial class Client
return obj.ToModel();
}
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload)
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload, CancellationToken cancellationToken = default)
{
return await PutObject(header, payload);
return await PutObject(header, payload, cancellationToken);
}
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload)
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload, CancellationToken cancellationToken = default)
{
using var stream = new MemoryStream(payload);
return await PutObject(header, stream);
return await PutObject(header, stream, cancellationToken);
}
private async Task<ObjectId> PutObject(ObjectHeader header, Stream payload)
private Task<ObjectId> PutObject(ObjectHeader header, Stream payload, CancellationToken cancellationToken)
{
if (header.ClientCut)
return PutClientCutObject(header, payload, cancellationToken);
else
return PutStreamObject(header, payload, cancellationToken);
}
private async Task<ObjectId> PutClientCutObject(ObjectHeader header, Stream payloadStream, CancellationToken cancellationToken)
{
ObjectId? objectId = null;
List<ObjectId> sentObjectIds = [];
ModelsV2.Object? currentObject;
var partSize = (int)NetworkSettings["MaxObjectSize"];
var buffer = new byte[partSize];
var largeObject = new LargeObject(header.ContainerId);
var split = new Split();
var fullLength = (ulong)payloadStream.Length;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var bytesCount = await payloadStream.ReadAsync(buffer, 0, partSize);
split.Previous = sentObjectIds.LastOrDefault();
largeObject.AppendBlock(buffer, bytesCount);
currentObject = new ModelsV2.Object(header.ContainerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
.AddAttributes(header.Attributes)
.SetSplit(split);
if (largeObject.PayloadLength == fullLength)
break;
objectId = await PutSingleObjectAsync(currentObject, cancellationToken);
sentObjectIds.Add(objectId!);
}
if (sentObjectIds.Any())
{
largeObject.CalculateHash();
currentObject.SetParent(largeObject);
objectId = await PutSingleObjectAsync(currentObject, cancellationToken);
sentObjectIds.Add(objectId);
var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds);
_ = await PutSingleObjectAsync(linkObject, cancellationToken);
return currentObject.GetParentId();
}
return await PutSingleObjectAsync(currentObject, cancellationToken);
}
private async Task<ObjectId> PutStreamObject(ObjectHeader header, Stream payload, CancellationToken cancellationToken)
{
var sessionToken = await CreateSessionAsync(uint.MaxValue);
var hdr = header.ToGrpcMessage();
@ -120,6 +188,8 @@ public partial class Client
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize);
if (bufferLength == 0)
@ -141,7 +211,7 @@ public partial class Client
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
}
public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object)
public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object, CancellationToken cancellationToken = default)
{
var sessionToken = await CreateSessionAsync(uint.MaxValue);
@ -163,7 +233,7 @@ public partial class Client
request.Sign(_key);
var response = await _objectServiceClient!.PutSingleAsync(request);
var response = await _objectServiceClient!.PutSingleAsync(request, null, null, cancellationToken);
Verifier.CheckResponse(response);
return ObjectId.FromHash(obj.ObjectId.Value.ToByteArray());
@ -207,6 +277,8 @@ public partial class Client
split.Parent = grpcHeader.Split.Parent.ToModel();
}
grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage();
}
var obj = new Object.Object
@ -286,10 +358,12 @@ public partial class Client
request.Body.Filters.Add(filter.ToGrpcMessage());
}
request.AddMetaHeader();
request.Sign(_key);
var objectsIds = SearchObjects(request);
await foreach (var oid in objectsIds)
{
yield return ObjectId.FromHash(oid.Value.ToByteArray());
@ -301,12 +375,14 @@ public partial class Client
using var stream = GetObjectInit(request);
var obj = await stream.ReadHeader();
var payload = new byte[obj.Header.PayloadLength];
var offset = 0;
var offset = 0L;
var chunk = await stream.ReadChunk();
while (chunk is not null)
while (chunk is not null && (ulong)offset < obj.Header.PayloadLength)
{
chunk.CopyTo(payload, offset);
var length = Math.Min((long)obj.Header.PayloadLength - offset, chunk.Length);
Array.Copy(chunk, 0, payload, offset, length);
offset += chunk.Length;
chunk = await stream.ReadChunk();
}
@ -321,6 +397,7 @@ public partial class Client
if (initRequest is null)
throw new ArgumentNullException(nameof(initRequest));
return new ObjectReader
{
Call = _objectServiceClient!.Get(initRequest)
@ -376,3 +453,5 @@ public partial class Client
}

View file

@ -7,8 +7,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
<PackageReference Include="Google.Protobuf" Version="3.27.0" />
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
<PackageReference Include="System.Memory" Version="4.5.5" />
</ItemGroup>

View file

@ -11,6 +11,7 @@ public static class Helper
internal static byte[] RIPEMD160(this byte[] value)
{
var hash = new byte[20];
var digest = new RipeMD160Digest();
digest.BlockUpdate(value, 0, value.Length);
digest.DoFinal(hash, 0);

View file

@ -5,5 +5,5 @@ namespace FrostFS.SDK.ModelsV2.Netmap;
public class NodeInfo
{
public NodeState State { get; set; }
public Version Version { get; set; }
public Version? Version { get; set; }
}

View file

@ -22,7 +22,7 @@ public class Object
public void SetParent(LargeObject largeObject)
{
Header.Split!.ParentHeader = largeObject.Header;
Header.Split.ParentHeader = largeObject.Header;
}
public ObjectId? GetParentId()
@ -41,10 +41,11 @@ public class LargeObject(ContainerId container) : Object(container, [])
this.payloadHash.TransformBlock(bytes, 0, count, bytes, 0);
}
public void CalculateHash()
public LargeObject CalculateHash()
{
this.payloadHash.TransformFinalBlock([], 0, 0);
Header.PayloadCheckSum = this.payloadHash.Hash;
return this;
}
public ulong PayloadLength

View file

@ -6,7 +6,7 @@ namespace FrostFS.SDK.ModelsV2;
public class ObjectHeader
{
public OwnerId OwnerId { get; set; }
public OwnerId? OwnerId { get; set; }
public List<ObjectAttribute> Attributes { get; set; }
@ -18,7 +18,7 @@ public class ObjectHeader
public ObjectType ObjectType { get; set; }
public Version Version { get; set; }
public Version? Version { get; set; }
public Split? Split { get; set; }

View file

@ -1,3 +1,4 @@
using System.Diagnostics;
using FrostFS.Session;
using Google.Protobuf;
@ -201,26 +202,31 @@ namespace FrostFS.Object
public partial class DeleteResponse : IResponse
{
[DebuggerStepThrough]
IMetaHeader IVerificableMessage.GetMetaHeader()
{
return MetaHeader;
}
[DebuggerStepThrough]
IVerificationHeader IVerificableMessage.GetVerificationHeader()
{
return VerifyHeader;
}
[DebuggerStepThrough]
void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader)
{
MetaHeader = (ResponseMetaHeader)metaHeader;
}
[DebuggerStepThrough]
void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader)
{
VerifyHeader = (ResponseVerificationHeader)verificationHeader;
}
[DebuggerStepThrough]
public IMessage GetBody()
{
return Body;

View file

@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="Moq.AutoMock" Version="3.5.0" />
<PackageReference Include="xunit" Version="2.5.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj" />
</ItemGroup>
</Project>