Merge branch 'master' of https://git.frostfs.info/TrueCloudLab/frostfs-sdk-csharp
Some checks failed
DCO / DCO (pull_request) Failing after 2m41s

This commit is contained in:
p.gross 2024-06-14 15:55:07 +03:00
commit e9603dff28
15 changed files with 216 additions and 36 deletions

View file

@ -1,4 +1,7 @@
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
#
VisualStudioVersion = 17.5.002.0
MinimumVisualStudioVersion =
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.ClientV2", "src\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj", "{50D8F61F-C302-4AC9-8D8A-AB0B8C0988C3}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.ClientV2", "src\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj", "{50D8F61F-C302-4AC9-8D8A-AB0B8C0988C3}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.Cryptography", "src\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj", "{3D804F4A-B0B2-47A5-B006-BE447BE64B50}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.Cryptography", "src\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj", "{3D804F4A-B0B2-47A5-B006-BE447BE64B50}"

View file

@ -121,7 +121,7 @@ static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, Contain
largeObject.AppendBlock(buffer, bytesCount); largeObject.AppendBlock(buffer, bytesCount);
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, buffer) currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
.AddAttribute(fileNameAttribute) .AddAttribute(fileNameAttribute)
.SetSplit(split); .SetSplit(split);
@ -134,15 +134,19 @@ static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, Contain
if (sentObjectIds.Any()) if (sentObjectIds.Any())
{ {
largeObject.CalculateHash(); largeObject.CalculateHash()
.AddAttribute(fileNameAttribute);
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds);
_ = await fsClient.PutSingleObjectAsync(linkObject);
currentObject.SetParent(largeObject); currentObject.SetParent(largeObject);
_ = await fsClient.PutSingleObjectAsync(currentObject);
var objectId = await fsClient.PutSingleObjectAsync(currentObject);
sentObjectIds.Add(objectId);
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds)
.AddAttribute(fileNameAttribute);
_ = await fsClient.PutSingleObjectAsync(linkObject);
return currentObject.GetParentId(); return currentObject.GetParentId();
} }

View file

@ -1,5 +1,7 @@
using System; using System;
using System.Collections.Generic;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using FrostFS.Container; using FrostFS.Container;
using FrostFS.Netmap; using FrostFS.Netmap;
@ -10,6 +12,7 @@ using FrostFS.SDK.ModelsV2;
using FrostFS.Session; using FrostFS.Session;
using Grpc.Core; using Grpc.Core;
using Grpc.Net.Client; using Grpc.Net.Client;
using static FrostFS.Netmap.NetworkConfig.Types;
using Version = FrostFS.SDK.ModelsV2.Version; using Version = FrostFS.SDK.ModelsV2.Version;
namespace FrostFS.SDK.ClientV2; namespace FrostFS.SDK.ClientV2;
@ -21,6 +24,8 @@ public partial class Client: IFrostFSClient
public readonly OwnerId OwnerId; public readonly OwnerId OwnerId;
public readonly Version Version = new(2, 13); public readonly Version Version = new(2, 13);
private readonly Dictionary<string, ulong> NetworkSettings = [];
private ContainerService.ContainerServiceClient? _containerServiceClient; private ContainerService.ContainerServiceClient? _containerServiceClient;
private NetmapService.NetmapServiceClient? _netmapServiceClient; private NetmapService.NetmapServiceClient? _netmapServiceClient;
private ObjectService.ObjectServiceClient? _objectServiceClient; private ObjectService.ObjectServiceClient? _objectServiceClient;
@ -42,6 +47,33 @@ public partial class Client: IFrostFSClient
InitObjectClient(); InitObjectClient();
InitSessionClient(); InitSessionClient();
CheckFrostFsVersionSupport(); CheckFrostFsVersionSupport();
InitNetworkInfoAsync();
}
private async void InitNetworkInfoAsync()
{
var info = await GetNetworkInfoAsync();
foreach (var param in info.Body.NetworkInfo.NetworkConfig.Parameters)
{
SetNetworksParam(param);
}
}
private void SetNetworksParam(Parameter param)
{
var key = Encoding.UTF8.GetString(param.Key.ToByteArray());
var encodedValue = param.Value.ToByteArray();
ulong val = 0;
for (var i = encodedValue.Length - 1; i >= 0; i--)
{
val = (val << 8) + encodedValue[i];
}
NetworkSettings.Add(key, val);
} }
private async void CheckFrostFsVersionSupport() private async void CheckFrostFsVersionSupport()
@ -84,7 +116,11 @@ public partial class Client: IFrostFSClient
throw new ArgumentException(msg); throw new ArgumentException(msg);
} }
_channel = GrpcChannel.ForAddress(uri, new GrpcChannelOptions { Credentials = grpcCredentials }); _channel = GrpcChannel.ForAddress(uri, new GrpcChannelOptions
{
Credentials = grpcCredentials,
HttpHandler = new System.Net.Http.HttpClientHandler()
});
} }
private void InitContainerClient() private void InitContainerClient()

View file

@ -38,7 +38,7 @@ public static class Extensions
public static LinkObject AddChildren(this LinkObject linkObject, IEnumerable<ObjectId> objectIds) public static LinkObject AddChildren(this LinkObject linkObject, IEnumerable<ObjectId> objectIds)
{ {
linkObject.Header.Split.Children.AddRange(objectIds); linkObject.Header.Split!.Children.AddRange(objectIds);
return linkObject; return linkObject;
} }
} }

View file

@ -1,5 +1,6 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2;
@ -18,13 +19,14 @@ public interface IFrostFSClient
Task<ObjectHeader> GetObjectHeadAsync(ContainerId containerId, ObjectId objectId); Task<ObjectHeader> GetObjectHeadAsync(ContainerId containerId, ObjectId objectId);
Task<ModelsV2.Object> GetObjectAsync(ContainerId containerId, ObjectId objectId); Task<ModelsV2.Object> GetObjectAsync(ContainerId containerId, ObjectId objectId);
Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload); Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload, CancellationToken cancellationToken = default);
Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload); Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload, CancellationToken cancellationToken = default);
Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj); Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj, CancellationToken cancellationToken = default);
Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId); Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId);

View file

@ -103,15 +103,31 @@ public static class ObjectHeaderMapper
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.") _ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
}; };
return new ObjectHeader( var model = new ObjectHeader(
new ContainerId(Base58.Encode(header.ContainerId.Value.ToByteArray())), new ContainerId(Base58.Encode(header.ContainerId.Value.ToByteArray())),
objTypeName, objTypeName,
header.Attributes.Select(attribute => attribute.ToModel()).ToArray() header.Attributes.Select(attribute => attribute.ToModel()).ToArray()
) )
{ {
PayloadLength = header.PayloadLength, PayloadLength = header.PayloadLength,
Version = header.Version.ToModel() Version = header.Version.ToModel(),
OwnerId = header.OwnerId.ToModel()
}; };
if (header.Split != null)
{
model.Split = new Split(SplitId.CrateFromBinary(header.Split.SplitId.ToByteArray()))
{
Parent = header.Split.Parent?.ToModel(),
ParentHeader = header.Split.ParentHeader?.ToModel(),
Previous = header.Split.Previous?.ToModel()
};
if (header.Split.Children.Any())
model.Split.Children.AddRange(header.Split.Children.Select(x => x.ToModel()));
}
return model;
} }
} }
@ -148,4 +164,3 @@ public static class SignatureMapper
}; };
} }
} }

View file

@ -13,4 +13,9 @@ public static class OwnerIdMapper
Value = ByteString.CopyFrom(ownerId.ToHash()) Value = ByteString.CopyFrom(ownerId.ToHash())
}; };
} }
public static OwnerId ToModel(this OwnerID ownerId)
{
return new OwnerId(ownerId.ToString());
}
} }

View file

@ -13,6 +13,8 @@ using FrostFS.SDK.Cryptography;
using FrostFS.Session; using FrostFS.Session;
using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ClientV2.Extensions;
using System.Threading;
namespace FrostFS.SDK.ClientV2; namespace FrostFS.SDK.ClientV2;
@ -32,6 +34,7 @@ public partial class Client
} }
}; };
request.AddMetaHeader(); request.AddMetaHeader();
request.Sign(_key); request.Sign(_key);
var response = await _objectServiceClient!.HeadAsync(request); var response = await _objectServiceClient!.HeadAsync(request);
@ -70,18 +73,83 @@ public partial class Client
return obj.ToModel(); return obj.ToModel();
} }
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload) public async Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload, CancellationToken cancellationToken = default)
{ {
return await PutObject(header, payload); return await PutObject(header, payload, cancellationToken);
} }
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload) public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload, CancellationToken cancellationToken = default)
{ {
using var stream = new MemoryStream(payload); using var stream = new MemoryStream(payload);
return await PutObject(header, stream); return await PutObject(header, stream, cancellationToken);
} }
private async Task<ObjectId> PutObject(ObjectHeader header, Stream payload) private Task<ObjectId> PutObject(ObjectHeader header, Stream payload, CancellationToken cancellationToken)
{
if (header.ClientCut)
return PutClientCutObject(header, payload, cancellationToken);
else
return PutStreamObject(header, payload, cancellationToken);
}
private async Task<ObjectId> PutClientCutObject(ObjectHeader header, Stream payloadStream, CancellationToken cancellationToken)
{
ObjectId? objectId = null;
List<ObjectId> sentObjectIds = [];
ModelsV2.Object? currentObject;
var partSize = (int)NetworkSettings["MaxObjectSize"];
var buffer = new byte[partSize];
var largeObject = new LargeObject(header.ContainerId);
var split = new Split();
var fullLength = (ulong)payloadStream.Length;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var bytesCount = await payloadStream.ReadAsync(buffer, 0, partSize);
split.Previous = sentObjectIds.LastOrDefault();
largeObject.AppendBlock(buffer, bytesCount);
currentObject = new ModelsV2.Object(header.ContainerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
.AddAttributes(header.Attributes)
.SetSplit(split);
if (largeObject.PayloadLength == fullLength)
break;
objectId = await PutSingleObjectAsync(currentObject, cancellationToken);
sentObjectIds.Add(objectId!);
}
if (sentObjectIds.Any())
{
largeObject.CalculateHash();
currentObject.SetParent(largeObject);
objectId = await PutSingleObjectAsync(currentObject, cancellationToken);
sentObjectIds.Add(objectId);
var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds);
_ = await PutSingleObjectAsync(linkObject, cancellationToken);
return currentObject.GetParentId();
}
return await PutSingleObjectAsync(currentObject, cancellationToken);
}
private async Task<ObjectId> PutStreamObject(ObjectHeader header, Stream payload, CancellationToken cancellationToken)
{ {
var sessionToken = await CreateSessionAsync(uint.MaxValue); var sessionToken = await CreateSessionAsync(uint.MaxValue);
var hdr = header.ToGrpcMessage(); var hdr = header.ToGrpcMessage();
@ -120,6 +188,8 @@ public partial class Client
while (true) while (true)
{ {
cancellationToken.ThrowIfCancellationRequested();
var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize); var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize);
if (bufferLength == 0) if (bufferLength == 0)
@ -141,7 +211,7 @@ public partial class Client
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()); return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
} }
public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object) public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object, CancellationToken cancellationToken = default)
{ {
var sessionToken = await CreateSessionAsync(uint.MaxValue); var sessionToken = await CreateSessionAsync(uint.MaxValue);
@ -163,7 +233,7 @@ public partial class Client
request.Sign(_key); request.Sign(_key);
var response = await _objectServiceClient!.PutSingleAsync(request); var response = await _objectServiceClient!.PutSingleAsync(request, null, null, cancellationToken);
Verifier.CheckResponse(response); Verifier.CheckResponse(response);
return ObjectId.FromHash(obj.ObjectId.Value.ToByteArray()); return ObjectId.FromHash(obj.ObjectId.Value.ToByteArray());
@ -207,6 +277,8 @@ public partial class Client
split.Parent = grpcHeader.Split.Parent.ToModel(); split.Parent = grpcHeader.Split.Parent.ToModel();
} }
grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage();
} }
var obj = new Object.Object var obj = new Object.Object
@ -286,10 +358,12 @@ public partial class Client
request.Body.Filters.Add(filter.ToGrpcMessage()); request.Body.Filters.Add(filter.ToGrpcMessage());
} }
request.AddMetaHeader(); request.AddMetaHeader();
request.Sign(_key); request.Sign(_key);
var objectsIds = SearchObjects(request); var objectsIds = SearchObjects(request);
await foreach (var oid in objectsIds) await foreach (var oid in objectsIds)
{ {
yield return ObjectId.FromHash(oid.Value.ToByteArray()); yield return ObjectId.FromHash(oid.Value.ToByteArray());
@ -301,12 +375,14 @@ public partial class Client
using var stream = GetObjectInit(request); using var stream = GetObjectInit(request);
var obj = await stream.ReadHeader(); var obj = await stream.ReadHeader();
var payload = new byte[obj.Header.PayloadLength]; var payload = new byte[obj.Header.PayloadLength];
var offset = 0; var offset = 0L;
var chunk = await stream.ReadChunk(); var chunk = await stream.ReadChunk();
while (chunk is not null) while (chunk is not null && (ulong)offset < obj.Header.PayloadLength)
{ {
chunk.CopyTo(payload, offset); var length = Math.Min((long)obj.Header.PayloadLength - offset, chunk.Length);
Array.Copy(chunk, 0, payload, offset, length);
offset += chunk.Length; offset += chunk.Length;
chunk = await stream.ReadChunk(); chunk = await stream.ReadChunk();
} }
@ -321,6 +397,7 @@ public partial class Client
if (initRequest is null) if (initRequest is null)
throw new ArgumentNullException(nameof(initRequest)); throw new ArgumentNullException(nameof(initRequest));
return new ObjectReader return new ObjectReader
{ {
Call = _objectServiceClient!.Get(initRequest) Call = _objectServiceClient!.Get(initRequest)
@ -376,3 +453,5 @@ public partial class Client
} }

View file

@ -7,8 +7,8 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
<PackageReference Include="Google.Protobuf" Version="3.27.0" /> <PackageReference Include="Google.Protobuf" Version="3.27.0" />
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
<PackageReference Include="System.Memory" Version="4.5.5" /> <PackageReference Include="System.Memory" Version="4.5.5" />
</ItemGroup> </ItemGroup>

View file

@ -11,6 +11,7 @@ public static class Helper
internal static byte[] RIPEMD160(this byte[] value) internal static byte[] RIPEMD160(this byte[] value)
{ {
var hash = new byte[20]; var hash = new byte[20];
var digest = new RipeMD160Digest(); var digest = new RipeMD160Digest();
digest.BlockUpdate(value, 0, value.Length); digest.BlockUpdate(value, 0, value.Length);
digest.DoFinal(hash, 0); digest.DoFinal(hash, 0);

View file

@ -5,5 +5,5 @@ namespace FrostFS.SDK.ModelsV2.Netmap;
public class NodeInfo public class NodeInfo
{ {
public NodeState State { get; set; } public NodeState State { get; set; }
public Version Version { get; set; } public Version? Version { get; set; }
} }

View file

@ -22,7 +22,7 @@ public class Object
public void SetParent(LargeObject largeObject) public void SetParent(LargeObject largeObject)
{ {
Header.Split!.ParentHeader = largeObject.Header; Header.Split.ParentHeader = largeObject.Header;
} }
public ObjectId? GetParentId() public ObjectId? GetParentId()
@ -41,10 +41,11 @@ public class LargeObject(ContainerId container) : Object(container, [])
this.payloadHash.TransformBlock(bytes, 0, count, bytes, 0); this.payloadHash.TransformBlock(bytes, 0, count, bytes, 0);
} }
public void CalculateHash() public LargeObject CalculateHash()
{ {
this.payloadHash.TransformFinalBlock([], 0, 0); this.payloadHash.TransformFinalBlock([], 0, 0);
Header.PayloadCheckSum = this.payloadHash.Hash; Header.PayloadCheckSum = this.payloadHash.Hash;
return this;
} }
public ulong PayloadLength public ulong PayloadLength

View file

@ -6,7 +6,7 @@ namespace FrostFS.SDK.ModelsV2;
public class ObjectHeader public class ObjectHeader
{ {
public OwnerId OwnerId { get; set; } public OwnerId? OwnerId { get; set; }
public List<ObjectAttribute> Attributes { get; set; } public List<ObjectAttribute> Attributes { get; set; }
@ -18,7 +18,7 @@ public class ObjectHeader
public ObjectType ObjectType { get; set; } public ObjectType ObjectType { get; set; }
public Version Version { get; set; } public Version? Version { get; set; }
public Split? Split { get; set; } public Split? Split { get; set; }

View file

@ -1,3 +1,4 @@
using System.Diagnostics;
using FrostFS.Session; using FrostFS.Session;
using Google.Protobuf; using Google.Protobuf;
@ -201,26 +202,31 @@ namespace FrostFS.Object
public partial class DeleteResponse : IResponse public partial class DeleteResponse : IResponse
{ {
[DebuggerStepThrough]
IMetaHeader IVerificableMessage.GetMetaHeader() IMetaHeader IVerificableMessage.GetMetaHeader()
{ {
return MetaHeader; return MetaHeader;
} }
[DebuggerStepThrough]
IVerificationHeader IVerificableMessage.GetVerificationHeader() IVerificationHeader IVerificableMessage.GetVerificationHeader()
{ {
return VerifyHeader; return VerifyHeader;
} }
[DebuggerStepThrough]
void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader) void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader)
{ {
MetaHeader = (ResponseMetaHeader)metaHeader; MetaHeader = (ResponseMetaHeader)metaHeader;
} }
[DebuggerStepThrough]
void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader) void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader)
{ {
VerifyHeader = (ResponseVerificationHeader)verificationHeader; VerifyHeader = (ResponseVerificationHeader)verificationHeader;
} }
[DebuggerStepThrough]
public IMessage GetBody() public IMessage GetBody()
{ {
return Body; return Body;

View file

@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="Moq.AutoMock" Version="3.5.0" />
<PackageReference Include="xunit" Version="2.5.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj" />
</ItemGroup>
</Project>