From 013d8a8436bfaa3948bc4bb345a32f6a7bdb9b4d Mon Sep 17 00:00:00 2001 From: Ivan Pchelintsev Date: Thu, 16 May 2024 10:01:08 +0300 Subject: [PATCH] [#1] Add Object Head/Put/Delete operations Signed-off-by: Ivan Pchelintsev --- .../Interfaces/IFrostFSClient.cs | 6 +- .../Mappers/GRPC/Object.cs | 18 ++++ .../RequestConstructor.cs | 25 ++++++ .../FrostFS.SDK.ClientV2/Services/Object.cs | 86 +++++++++++++++++++ .../FrostFS.SDK.ClientV2/Services/Session.cs | 7 +- sdk/src/FrostFS.SDK.Cryptography/Helper.cs | 2 - sdk/src/FrostFS.SDK.Cryptography/Tz/GF127.cs | 1 - sdk/src/FrostFS.SDK.ModelsV2/Constants.cs | 7 ++ sdk/src/FrostFS.SDK.ModelsV2/ContainerId.cs | 2 +- .../FrostFS.SDK.ModelsV2/Enums/ObjectType.cs | 8 ++ sdk/src/FrostFS.SDK.ModelsV2/Object.cs | 9 ++ sdk/src/FrostFS.SDK.ModelsV2/ObjectId.cs | 2 +- sdk/src/FrostFS.SDK.Service/Service.cs | 31 ++++++- 13 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 sdk/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs create mode 100644 sdk/src/FrostFS.SDK.ModelsV2/Constants.cs create mode 100644 sdk/src/FrostFS.SDK.ModelsV2/Enums/ObjectType.cs create mode 100644 sdk/src/FrostFS.SDK.ModelsV2/Object.cs diff --git a/sdk/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs b/sdk/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs index bbd3d1d..3cac9ba 100644 --- a/sdk/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs +++ b/sdk/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs @@ -1,3 +1,4 @@ +using FrostFS.Object; using FrostFS.Refs; namespace FrostFS.SDK.ClientV2; @@ -7,5 +8,8 @@ public interface IFrostFSClient Task ListContainersAsync(); Task CreateContainerAsync(Container.Container container); Task GetContainerAsync(ContainerID containerId); - Task DeleteContainerAsync(ContainerID cid); + Task DeleteContainerAsync(ContainerID containerId); + Task GetObjectHeadAsync(ContainerID containerId, ObjectID objectId); + Task PutObjectAsync(ContainerID containerId, Stream data); + Task DeleteObjectAsync(ContainerID containerId, ObjectID objectId); } \ No newline at end of file diff --git a/sdk/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs b/sdk/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs new file mode 100644 index 0000000..d476106 --- /dev/null +++ b/sdk/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs @@ -0,0 +1,18 @@ +using FrostFS.SDK.ModelsV2; +using FrostFS.SDK.ModelsV2.Enums; + +namespace FrostFS.SDK.ClientV2.Mappers.GRPC; + +public static class ObjectHeadMapper +{ + public static ObjectHead ToModel(this Object.Header head) + { + // var obtype = Enum.Parse(head.ObjectType.ToString()); + return new ObjectHead + { + ContainerId = ContainerId.FromHash(head.ContainerId.Value.ToByteArray()), + Size = (long)head.PayloadLength, + Version = head.Version.ToModel() + }; + } +} \ No newline at end of file diff --git a/sdk/src/FrostFS.SDK.ClientV2/RequestConstructor.cs b/sdk/src/FrostFS.SDK.ClientV2/RequestConstructor.cs index 4b02714..fb632ee 100644 --- a/sdk/src/FrostFS.SDK.ClientV2/RequestConstructor.cs +++ b/sdk/src/FrostFS.SDK.ClientV2/RequestConstructor.cs @@ -1,3 +1,5 @@ +using System.Security.Cryptography; +using FrostFS.Refs; using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.ModelsV2; using FrostFS.Session; @@ -12,4 +14,27 @@ public static class RequestConstructor metaHeader ??= MetaHeader.Default().ToGrpcMessage(); request.MetaHeader = metaHeader; } + + public static void AddObjectSessionToken( + this IRequest request, + SessionToken sessionToken, + ContainerID cid, + ObjectID oid, + ObjectSessionContext.Types.Verb verb, + ECDsa key) + { + if (request.MetaHeader.SessionToken is not null) return; + request.MetaHeader.SessionToken = sessionToken; + var ctx = new ObjectSessionContext + { + Target = new ObjectSessionContext.Types.Target + { + Container = cid, + Objects = { oid } + }, + Verb = verb + }; + request.MetaHeader.SessionToken.Body.Object = ctx; + request.MetaHeader.SessionToken.Signature = key.SignMessagePart(request.MetaHeader.SessionToken.Body); + } } diff --git a/sdk/src/FrostFS.SDK.ClientV2/Services/Object.cs b/sdk/src/FrostFS.SDK.ClientV2/Services/Object.cs index da98868..499423f 100644 --- a/sdk/src/FrostFS.SDK.ClientV2/Services/Object.cs +++ b/sdk/src/FrostFS.SDK.ClientV2/Services/Object.cs @@ -1,5 +1,10 @@ using FrostFS.Object; using FrostFS.Refs; +using FrostFS.SDK.ClientV2.Mappers.GRPC; +using FrostFS.SDK.Cryptography; +using FrostFS.Session; +using Google.Protobuf; +using Grpc.Core; namespace FrostFS.SDK.ClientV2; @@ -23,6 +28,61 @@ public partial class Client return await _objectServiceClient.HeadAsync(request); } + public async Task PutObjectAsync(ContainerID cid, Stream data) + { + var sessionToken = await CreateSessionAsync(uint.MaxValue); + var header = new Header + { + ContainerId = cid, + OwnerId = _owner.ToGrpcMessage(), + Version = Version.ToGrpcMessage() + }; + var oid = new ObjectID + { + Value = header.Sha256() + }; + var request = new PutRequest + { + Body = new PutRequest.Types.Body + { + Init = new PutRequest.Types.Body.Types.Init + { + Header = header, + }, + } + }; + request.AddMetaHeader(); + request.AddObjectSessionToken(sessionToken, cid, oid, ObjectSessionContext.Types.Verb.Put, _key); + request.Sign(_key); + + using var stream = await InitObject(request); + var buffer = new byte[ModelsV2.Constants.ObjectChunkSize]; + var bufferLength = data.Read(buffer, 0, ModelsV2.Constants.ObjectChunkSize); + while (bufferLength > 0) + { + request.Body = new PutRequest.Types.Body + { + Chunk = ByteString.CopyFrom(buffer[..bufferLength]), + }; + request.VerifyHeader = null; + request.Sign(_key); + await stream.Write(request); + bufferLength = data.Read(buffer, 0, ModelsV2.Constants.ObjectChunkSize); + } + return await stream.Close(); + } + + private async Task InitObject(PutRequest initRequest) + { + if (initRequest is null) + { + throw new ArgumentNullException(nameof(initRequest)); + } + var call = _objectServiceClient.Put(); + await call.RequestStream.WriteAsync(initRequest); + return new ObjectStreamer { Call = call }; + } + public async Task DeleteObjectAsync(ContainerID cid, ObjectID oid) { var request = new DeleteRequest @@ -40,4 +100,30 @@ public partial class Client request.Sign(_key); return await _objectServiceClient.DeleteAsync(request); } +} + +internal class ObjectStreamer : IDisposable +{ + public AsyncClientStreamingCall Call { get; init; } + + public void Dispose() + { + Call.Dispose(); + } + + public async Task Write(PutRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + await Call.RequestStream.WriteAsync(request); + } + + public async Task Close() + { + await Call.RequestStream.CompleteAsync(); + return await Call.ResponseAsync; + } } \ No newline at end of file diff --git a/sdk/src/FrostFS.SDK.ClientV2/Services/Session.cs b/sdk/src/FrostFS.SDK.ClientV2/Services/Session.cs index 96104b4..df8e3e3 100644 --- a/sdk/src/FrostFS.SDK.ClientV2/Services/Session.cs +++ b/sdk/src/FrostFS.SDK.ClientV2/Services/Session.cs @@ -1,14 +1,11 @@ -using FrostFS.Refs; using FrostFS.SDK.ClientV2.Mappers.GRPC; -using FrostFS.SDK.ModelsV2; using FrostFS.Session; -using Google.Protobuf; namespace FrostFS.SDK.ClientV2; public partial class Client { - public async Task CreateSessionAsync(ulong expiration) + private async Task CreateSessionAsync(ulong expiration) { var request = new CreateRequest { @@ -23,7 +20,7 @@ public partial class Client return await CreateSession(request); } - public async Task CreateSession(CreateRequest request) + private async Task CreateSession(CreateRequest request) { var resp = await _sessionServiceClient.CreateAsync(request); return new SessionToken diff --git a/sdk/src/FrostFS.SDK.Cryptography/Helper.cs b/sdk/src/FrostFS.SDK.Cryptography/Helper.cs index 1c15458..90f7150 100644 --- a/sdk/src/FrostFS.SDK.Cryptography/Helper.cs +++ b/sdk/src/FrostFS.SDK.Cryptography/Helper.cs @@ -8,8 +8,6 @@ namespace FrostFS.SDK.Cryptography; public static class Helper { - public const int Sha256HashLength = 32; - internal static byte[] RIPEMD160(this byte[] value) { var hash = new byte[20]; diff --git a/sdk/src/FrostFS.SDK.Cryptography/Tz/GF127.cs b/sdk/src/FrostFS.SDK.Cryptography/Tz/GF127.cs index bc67b90..260d627 100644 --- a/sdk/src/FrostFS.SDK.Cryptography/Tz/GF127.cs +++ b/sdk/src/FrostFS.SDK.Cryptography/Tz/GF127.cs @@ -6,7 +6,6 @@ namespace FrostFS.SDK.Cryptography.Tz public class GF127 : IEquatable { public const int ByteSize = 16; - public const ulong MaxUlong = ulong.MaxValue; public const ulong MSB64 = (ulong)1 << 63; // 2^63 public static readonly GF127 Zero = new(0, 0); public static readonly GF127 One = new(1, 0); diff --git a/sdk/src/FrostFS.SDK.ModelsV2/Constants.cs b/sdk/src/FrostFS.SDK.ModelsV2/Constants.cs new file mode 100644 index 0000000..eeb2b67 --- /dev/null +++ b/sdk/src/FrostFS.SDK.ModelsV2/Constants.cs @@ -0,0 +1,7 @@ +namespace FrostFS.SDK.ModelsV2; + +public class Constants +{ + public const int ObjectChunkSize = 3 * (1 << 20); + public const int Sha256HashLength = 32; +} \ No newline at end of file diff --git a/sdk/src/FrostFS.SDK.ModelsV2/ContainerId.cs b/sdk/src/FrostFS.SDK.ModelsV2/ContainerId.cs index 65695b9..ef4542e 100644 --- a/sdk/src/FrostFS.SDK.ModelsV2/ContainerId.cs +++ b/sdk/src/FrostFS.SDK.ModelsV2/ContainerId.cs @@ -13,7 +13,7 @@ public class ContainerId public static ContainerId FromHash(byte[] hash) { - if (hash.Length != Helper.Sha256HashLength) + if (hash.Length != Constants.Sha256HashLength) { throw new FormatException("ContainerID must be a sha256 hash."); } diff --git a/sdk/src/FrostFS.SDK.ModelsV2/Enums/ObjectType.cs b/sdk/src/FrostFS.SDK.ModelsV2/Enums/ObjectType.cs new file mode 100644 index 0000000..5d206a0 --- /dev/null +++ b/sdk/src/FrostFS.SDK.ModelsV2/Enums/ObjectType.cs @@ -0,0 +1,8 @@ +namespace FrostFS.SDK.ModelsV2.Enums; + +public enum ObjectType +{ + Regular = 1, + Tombstone = 2, + Lock = 3 +} \ No newline at end of file diff --git a/sdk/src/FrostFS.SDK.ModelsV2/Object.cs b/sdk/src/FrostFS.SDK.ModelsV2/Object.cs new file mode 100644 index 0000000..46664e6 --- /dev/null +++ b/sdk/src/FrostFS.SDK.ModelsV2/Object.cs @@ -0,0 +1,9 @@ + +namespace FrostFS.SDK.ModelsV2; + +public class ObjectHead +{ + public ContainerId ContainerId { get; set; } + public long Size { get; set; } + public Version Version { get; set; } +} \ No newline at end of file diff --git a/sdk/src/FrostFS.SDK.ModelsV2/ObjectId.cs b/sdk/src/FrostFS.SDK.ModelsV2/ObjectId.cs index 680e20f..375e439 100644 --- a/sdk/src/FrostFS.SDK.ModelsV2/ObjectId.cs +++ b/sdk/src/FrostFS.SDK.ModelsV2/ObjectId.cs @@ -13,7 +13,7 @@ public class ObjectId public static ObjectId FromHash(byte[] hash) { - if (hash.Length != Helper.Sha256HashLength) + if (hash.Length != Constants.Sha256HashLength) { throw new FormatException("ObjectID must be a sha256 hash."); } diff --git a/sdk/src/FrostFS.SDK.Service/Service.cs b/sdk/src/FrostFS.SDK.Service/Service.cs index b977885..bc11d34 100644 --- a/sdk/src/FrostFS.SDK.Service/Service.cs +++ b/sdk/src/FrostFS.SDK.Service/Service.cs @@ -1,10 +1,10 @@ using FrostFS.SDK.ClientV2; using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.ModelsV2; -using Google.Protobuf; namespace FrostFS.SDK.Service; +// TODO: Проверять ответы Grpc public class FrostFsService { private readonly IFrostFSClient _client; @@ -42,4 +42,33 @@ public class FrostFsService { var deleteContainerResponse = await _client.DeleteContainerAsync(containerId.ToGrpcMessage()); } + + public async Task GetObjectHeadAsync(ContainerId containerId, ObjectId objectId) + { + var getObjectHeadResponse = await _client.GetObjectHeadAsync( + containerId.ToGrpcMessage(), + objectId.ToGrpcMessage() + ); + return getObjectHeadResponse.Body.Header.Header.ToModel(); + } + + public async Task PutObjectAsync(ContainerId containerId, Stream data) + { + var putObjectResponse = await _client.PutObjectAsync(containerId.ToGrpcMessage(), data); + return ObjectId.FromHash(putObjectResponse.Body.ObjectId.Value.ToByteArray()); + } + + public async Task PutObjectAsync(ContainerId containerId, byte[] data) + { + var putObjectResponse = await _client.PutObjectAsync(containerId.ToGrpcMessage(), new MemoryStream(data)); + return ObjectId.FromHash(putObjectResponse.Body.ObjectId.Value.ToByteArray()); + } + + public async Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId) + { + var deleteObjectResponse = await _client.DeleteObjectAsync( + containerId.ToGrpcMessage(), + objectId.ToGrpcMessage() + ); + } } \ No newline at end of file