[#7] Client cut internal
This commit is contained in:
parent
545e647d7b
commit
b69d22966f
15 changed files with 216 additions and 36 deletions
FrostFS.SDK.slnREADME.md
src
FrostFS.SDK.ClientV2
FrostFS.SDK.Cryptography
FrostFS.SDK.ModelsV2
FrostFS.SDK.ProtosV2/object
FrostFS.SDK.Tests
|
@ -1,4 +1,7 @@
|
|||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
#
|
||||
VisualStudioVersion = 17.5.002.0
|
||||
MinimumVisualStudioVersion =
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.ClientV2", "src\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj", "{50D8F61F-C302-4AC9-8D8A-AB0B8C0988C3}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrostFS.SDK.Cryptography", "src\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj", "{3D804F4A-B0B2-47A5-B006-BE447BE64B50}"
|
||||
|
|
20
README.md
20
README.md
|
@ -121,7 +121,7 @@ static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, Contain
|
|||
|
||||
largeObject.AppendBlock(buffer, bytesCount);
|
||||
|
||||
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, buffer)
|
||||
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
|
||||
.AddAttribute(fileNameAttribute)
|
||||
.SetSplit(split);
|
||||
|
||||
|
@ -134,15 +134,19 @@ static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, Contain
|
|||
|
||||
if (sentObjectIds.Any())
|
||||
{
|
||||
largeObject.CalculateHash();
|
||||
|
||||
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
|
||||
.AddChildren(sentObjectIds);
|
||||
|
||||
_ = await fsClient.PutSingleObjectAsync(linkObject);
|
||||
largeObject.CalculateHash()
|
||||
.AddAttribute(fileNameAttribute);
|
||||
|
||||
currentObject.SetParent(largeObject);
|
||||
_ = await fsClient.PutSingleObjectAsync(currentObject);
|
||||
|
||||
var objectId = await fsClient.PutSingleObjectAsync(currentObject);
|
||||
sentObjectIds.Add(objectId);
|
||||
|
||||
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
|
||||
.AddChildren(sentObjectIds)
|
||||
.AddAttribute(fileNameAttribute);
|
||||
|
||||
_ = await fsClient.PutSingleObjectAsync(linkObject);
|
||||
|
||||
return currentObject.GetParentId();
|
||||
}
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using FrostFS.Container;
|
||||
using FrostFS.Netmap;
|
||||
|
@ -10,6 +12,7 @@ using FrostFS.SDK.ModelsV2;
|
|||
using FrostFS.Session;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using static FrostFS.Netmap.NetworkConfig.Types;
|
||||
using Version = FrostFS.SDK.ModelsV2.Version;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2;
|
||||
|
@ -21,6 +24,8 @@ public partial class Client: IFrostFSClient
|
|||
public readonly OwnerId OwnerId;
|
||||
public readonly Version Version = new(2, 13);
|
||||
|
||||
private readonly Dictionary<string, ulong> NetworkSettings = [];
|
||||
|
||||
private ContainerService.ContainerServiceClient? _containerServiceClient;
|
||||
private NetmapService.NetmapServiceClient? _netmapServiceClient;
|
||||
private ObjectService.ObjectServiceClient? _objectServiceClient;
|
||||
|
@ -42,6 +47,33 @@ public partial class Client: IFrostFSClient
|
|||
InitObjectClient();
|
||||
InitSessionClient();
|
||||
CheckFrostFsVersionSupport();
|
||||
|
||||
InitNetworkInfoAsync();
|
||||
}
|
||||
|
||||
private async void InitNetworkInfoAsync()
|
||||
{
|
||||
var info = await GetNetworkInfoAsync();
|
||||
|
||||
foreach (var param in info.Body.NetworkInfo.NetworkConfig.Parameters)
|
||||
{
|
||||
SetNetworksParam(param);
|
||||
}
|
||||
}
|
||||
|
||||
private void SetNetworksParam(Parameter param)
|
||||
{
|
||||
var key = Encoding.UTF8.GetString(param.Key.ToByteArray());
|
||||
|
||||
var encodedValue = param.Value.ToByteArray();
|
||||
|
||||
ulong val = 0;
|
||||
for (var i = encodedValue.Length - 1; i >= 0; i--)
|
||||
{
|
||||
val = (val << 8) + encodedValue[i];
|
||||
}
|
||||
|
||||
NetworkSettings.Add(key, val);
|
||||
}
|
||||
|
||||
private async void CheckFrostFsVersionSupport()
|
||||
|
@ -84,7 +116,11 @@ public partial class Client: IFrostFSClient
|
|||
throw new ArgumentException(msg);
|
||||
}
|
||||
|
||||
_channel = GrpcChannel.ForAddress(uri, new GrpcChannelOptions { Credentials = grpcCredentials });
|
||||
_channel = GrpcChannel.ForAddress(uri, new GrpcChannelOptions
|
||||
{
|
||||
Credentials = grpcCredentials,
|
||||
HttpHandler = new System.Net.Http.HttpClientHandler()
|
||||
});
|
||||
}
|
||||
|
||||
private void InitContainerClient()
|
||||
|
|
|
@ -38,7 +38,7 @@ public static class Extensions
|
|||
|
||||
public static LinkObject AddChildren(this LinkObject linkObject, IEnumerable<ObjectId> objectIds)
|
||||
{
|
||||
linkObject.Header.Split.Children.AddRange(objectIds);
|
||||
linkObject.Header.Split!.Children.AddRange(objectIds);
|
||||
return linkObject;
|
||||
}
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using FrostFS.SDK.ModelsV2;
|
||||
|
@ -18,13 +19,14 @@ public interface IFrostFSClient
|
|||
|
||||
Task<ObjectHeader> GetObjectHeadAsync(ContainerId containerId, ObjectId objectId);
|
||||
|
||||
|
||||
Task<ModelsV2.Object> GetObjectAsync(ContainerId containerId, ObjectId objectId);
|
||||
|
||||
Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload);
|
||||
Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload);
|
||||
Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj);
|
||||
Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj, CancellationToken cancellationToken = default);
|
||||
|
||||
Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId);
|
||||
|
||||
|
|
|
@ -103,15 +103,31 @@ public static class ObjectHeaderMapper
|
|||
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
|
||||
};
|
||||
|
||||
return new ObjectHeader(
|
||||
var model = new ObjectHeader(
|
||||
new ContainerId(Base58.Encode(header.ContainerId.Value.ToByteArray())),
|
||||
objTypeName,
|
||||
header.Attributes.Select(attribute => attribute.ToModel()).ToArray()
|
||||
)
|
||||
{
|
||||
PayloadLength = header.PayloadLength,
|
||||
Version = header.Version.ToModel()
|
||||
Version = header.Version.ToModel(),
|
||||
OwnerId = header.OwnerId.ToModel()
|
||||
};
|
||||
|
||||
if (header.Split != null)
|
||||
{
|
||||
model.Split = new Split(SplitId.CrateFromBinary(header.Split.SplitId.ToByteArray()))
|
||||
{
|
||||
Parent = header.Split.Parent?.ToModel(),
|
||||
ParentHeader = header.Split.ParentHeader?.ToModel(),
|
||||
Previous = header.Split.Previous?.ToModel()
|
||||
};
|
||||
|
||||
if (header.Split.Children.Any())
|
||||
model.Split.Children.AddRange(header.Split.Children.Select(x => x.ToModel()));
|
||||
}
|
||||
|
||||
return model;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -148,4 +164,3 @@ public static class SignatureMapper
|
|||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -13,4 +13,9 @@ public static class OwnerIdMapper
|
|||
Value = ByteString.CopyFrom(ownerId.ToHash())
|
||||
};
|
||||
}
|
||||
|
||||
public static OwnerId ToModel(this OwnerID ownerId)
|
||||
{
|
||||
return new OwnerId(ownerId.ToString());
|
||||
}
|
||||
}
|
|
@ -13,6 +13,8 @@ using FrostFS.SDK.Cryptography;
|
|||
using FrostFS.Session;
|
||||
|
||||
using FrostFS.SDK.ModelsV2;
|
||||
using FrostFS.SDK.ClientV2.Extensions;
|
||||
using System.Threading;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2;
|
||||
|
||||
|
@ -32,6 +34,7 @@ public partial class Client
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
request.AddMetaHeader();
|
||||
request.Sign(_key);
|
||||
var response = await _objectServiceClient!.HeadAsync(request);
|
||||
|
@ -70,18 +73,83 @@ public partial class Client
|
|||
return obj.ToModel();
|
||||
}
|
||||
|
||||
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload)
|
||||
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await PutObject(header, payload);
|
||||
return await PutObject(header, payload, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload)
|
||||
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var stream = new MemoryStream(payload);
|
||||
return await PutObject(header, stream);
|
||||
return await PutObject(header, stream, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<ObjectId> PutObject(ObjectHeader header, Stream payload)
|
||||
private Task<ObjectId> PutObject(ObjectHeader header, Stream payload, CancellationToken cancellationToken)
|
||||
{
|
||||
if (header.ClientCut)
|
||||
return PutClientCutObject(header, payload, cancellationToken);
|
||||
else
|
||||
return PutStreamObject(header, payload, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<ObjectId> PutClientCutObject(ObjectHeader header, Stream payloadStream, CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectId? objectId = null;
|
||||
List<ObjectId> sentObjectIds = [];
|
||||
ModelsV2.Object? currentObject;
|
||||
|
||||
var partSize = (int)NetworkSettings["MaxObjectSize"];
|
||||
var buffer = new byte[partSize];
|
||||
|
||||
var largeObject = new LargeObject(header.ContainerId);
|
||||
|
||||
var split = new Split();
|
||||
|
||||
var fullLength = (ulong)payloadStream.Length;
|
||||
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var bytesCount = await payloadStream.ReadAsync(buffer, 0, partSize);
|
||||
|
||||
split.Previous = sentObjectIds.LastOrDefault();
|
||||
|
||||
largeObject.AppendBlock(buffer, bytesCount);
|
||||
|
||||
currentObject = new ModelsV2.Object(header.ContainerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
|
||||
.AddAttributes(header.Attributes)
|
||||
.SetSplit(split);
|
||||
|
||||
if (largeObject.PayloadLength == fullLength)
|
||||
break;
|
||||
|
||||
objectId = await PutSingleObjectAsync(currentObject, cancellationToken);
|
||||
|
||||
sentObjectIds.Add(objectId!);
|
||||
}
|
||||
|
||||
if (sentObjectIds.Any())
|
||||
{
|
||||
largeObject.CalculateHash();
|
||||
|
||||
currentObject.SetParent(largeObject);
|
||||
|
||||
objectId = await PutSingleObjectAsync(currentObject, cancellationToken);
|
||||
sentObjectIds.Add(objectId);
|
||||
|
||||
var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject)
|
||||
.AddChildren(sentObjectIds);
|
||||
|
||||
_ = await PutSingleObjectAsync(linkObject, cancellationToken);
|
||||
|
||||
return currentObject.GetParentId();
|
||||
}
|
||||
|
||||
return await PutSingleObjectAsync(currentObject, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<ObjectId> PutStreamObject(ObjectHeader header, Stream payload, CancellationToken cancellationToken)
|
||||
{
|
||||
var sessionToken = await CreateSessionAsync(uint.MaxValue);
|
||||
var hdr = header.ToGrpcMessage();
|
||||
|
@ -120,6 +188,8 @@ public partial class Client
|
|||
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize);
|
||||
|
||||
if (bufferLength == 0)
|
||||
|
@ -141,7 +211,7 @@ public partial class Client
|
|||
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
|
||||
}
|
||||
|
||||
public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object)
|
||||
public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var sessionToken = await CreateSessionAsync(uint.MaxValue);
|
||||
|
||||
|
@ -163,7 +233,7 @@ public partial class Client
|
|||
|
||||
request.Sign(_key);
|
||||
|
||||
var response = await _objectServiceClient!.PutSingleAsync(request);
|
||||
var response = await _objectServiceClient!.PutSingleAsync(request, null, null, cancellationToken);
|
||||
Verifier.CheckResponse(response);
|
||||
|
||||
return ObjectId.FromHash(obj.ObjectId.Value.ToByteArray());
|
||||
|
@ -207,6 +277,8 @@ public partial class Client
|
|||
|
||||
split.Parent = grpcHeader.Split.Parent.ToModel();
|
||||
}
|
||||
|
||||
grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage();
|
||||
}
|
||||
|
||||
var obj = new Object.Object
|
||||
|
@ -286,10 +358,12 @@ public partial class Client
|
|||
request.Body.Filters.Add(filter.ToGrpcMessage());
|
||||
}
|
||||
|
||||
|
||||
request.AddMetaHeader();
|
||||
request.Sign(_key);
|
||||
var objectsIds = SearchObjects(request);
|
||||
|
||||
|
||||
await foreach (var oid in objectsIds)
|
||||
{
|
||||
yield return ObjectId.FromHash(oid.Value.ToByteArray());
|
||||
|
@ -301,12 +375,14 @@ public partial class Client
|
|||
using var stream = GetObjectInit(request);
|
||||
var obj = await stream.ReadHeader();
|
||||
var payload = new byte[obj.Header.PayloadLength];
|
||||
var offset = 0;
|
||||
var offset = 0L;
|
||||
var chunk = await stream.ReadChunk();
|
||||
|
||||
while (chunk is not null)
|
||||
while (chunk is not null && (ulong)offset < obj.Header.PayloadLength)
|
||||
{
|
||||
chunk.CopyTo(payload, offset);
|
||||
var length = Math.Min((long)obj.Header.PayloadLength - offset, chunk.Length);
|
||||
|
||||
Array.Copy(chunk, 0, payload, offset, length);
|
||||
offset += chunk.Length;
|
||||
chunk = await stream.ReadChunk();
|
||||
}
|
||||
|
@ -321,6 +397,7 @@ public partial class Client
|
|||
if (initRequest is null)
|
||||
throw new ArgumentNullException(nameof(initRequest));
|
||||
|
||||
|
||||
return new ObjectReader
|
||||
{
|
||||
Call = _objectServiceClient!.Get(initRequest)
|
||||
|
@ -376,3 +453,5 @@ public partial class Client
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
|
||||
<PackageReference Include="Google.Protobuf" Version="3.27.0" />
|
||||
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
|
||||
<PackageReference Include="System.Memory" Version="4.5.5" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
@ -11,6 +11,7 @@ public static class Helper
|
|||
internal static byte[] RIPEMD160(this byte[] value)
|
||||
{
|
||||
var hash = new byte[20];
|
||||
|
||||
var digest = new RipeMD160Digest();
|
||||
digest.BlockUpdate(value, 0, value.Length);
|
||||
digest.DoFinal(hash, 0);
|
||||
|
|
|
@ -5,5 +5,5 @@ namespace FrostFS.SDK.ModelsV2.Netmap;
|
|||
public class NodeInfo
|
||||
{
|
||||
public NodeState State { get; set; }
|
||||
public Version Version { get; set; }
|
||||
public Version? Version { get; set; }
|
||||
}
|
|
@ -22,7 +22,7 @@ public class Object
|
|||
|
||||
public void SetParent(LargeObject largeObject)
|
||||
{
|
||||
Header.Split!.ParentHeader = largeObject.Header;
|
||||
Header.Split.ParentHeader = largeObject.Header;
|
||||
}
|
||||
|
||||
public ObjectId? GetParentId()
|
||||
|
@ -41,10 +41,11 @@ public class LargeObject(ContainerId container) : Object(container, [])
|
|||
this.payloadHash.TransformBlock(bytes, 0, count, bytes, 0);
|
||||
}
|
||||
|
||||
public void CalculateHash()
|
||||
public LargeObject CalculateHash()
|
||||
{
|
||||
this.payloadHash.TransformFinalBlock([], 0, 0);
|
||||
Header.PayloadCheckSum = this.payloadHash.Hash;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ulong PayloadLength
|
||||
|
|
|
@ -6,7 +6,7 @@ namespace FrostFS.SDK.ModelsV2;
|
|||
|
||||
public class ObjectHeader
|
||||
{
|
||||
public OwnerId OwnerId { get; set; }
|
||||
public OwnerId? OwnerId { get; set; }
|
||||
|
||||
public List<ObjectAttribute> Attributes { get; set; }
|
||||
|
||||
|
@ -18,7 +18,7 @@ public class ObjectHeader
|
|||
|
||||
public ObjectType ObjectType { get; set; }
|
||||
|
||||
public Version Version { get; set; }
|
||||
public Version? Version { get; set; }
|
||||
|
||||
public Split? Split { get; set; }
|
||||
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
using System.Diagnostics;
|
||||
using FrostFS.Session;
|
||||
using Google.Protobuf;
|
||||
|
||||
|
@ -201,26 +202,31 @@ namespace FrostFS.Object
|
|||
|
||||
public partial class DeleteResponse : IResponse
|
||||
{
|
||||
[DebuggerStepThrough]
|
||||
IMetaHeader IVerificableMessage.GetMetaHeader()
|
||||
{
|
||||
return MetaHeader;
|
||||
}
|
||||
|
||||
[DebuggerStepThrough]
|
||||
IVerificationHeader IVerificableMessage.GetVerificationHeader()
|
||||
{
|
||||
return VerifyHeader;
|
||||
}
|
||||
|
||||
[DebuggerStepThrough]
|
||||
void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader)
|
||||
{
|
||||
MetaHeader = (ResponseMetaHeader)metaHeader;
|
||||
}
|
||||
|
||||
[DebuggerStepThrough]
|
||||
void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader)
|
||||
{
|
||||
VerifyHeader = (ResponseVerificationHeader)verificationHeader;
|
||||
}
|
||||
|
||||
[DebuggerStepThrough]
|
||||
public IMessage GetBody()
|
||||
{
|
||||
return Body;
|
||||
|
|
28
src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj
Normal file
28
src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj
Normal file
|
@ -0,0 +1,28 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
|
||||
<PackageReference Include="Moq.AutoMock" Version="3.5.0" />
|
||||
<PackageReference Include="xunit" Version="2.5.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\FrostFS.SDK.ClientV2\FrostFS.SDK.ClientV2.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
Loading…
Reference in a new issue