From 545e647d7b01b04b86133a7ec519ce3229aeeeee Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Mon, 10 Jun 2024 11:31:36 +0300 Subject: [PATCH] [#4] infrastructure and sample Client Cut Signed-off-by: Pavel Gross --- FrostFS.SDK.sln | 7 +- README.md | 77 ++++- src/FrostFS.SDK.ClientV2/Client.cs | 10 +- src/FrostFS.SDK.ClientV2/Extensions/Object.cs | 44 +++ .../FrostFS.SDK.ClientV2.csproj | 1 + .../Interfaces/IFrostFSClient.cs | 13 +- .../Mappers/GRPC/ContainerId.cs | 3 +- .../Mappers/GRPC/Object.cs | 92 ++++- .../Mappers/GRPC/ObjectId.cs | 5 + src/FrostFS.SDK.ClientV2/RequestSigner.cs | 2 +- .../Services/Container.cs | 8 +- src/FrostFS.SDK.ClientV2/Services/Object.cs | 317 ++++++++++++------ src/FrostFS.SDK.Cryptography/Base58.cs | 2 - src/FrostFS.SDK.Cryptography/Helper.cs | 1 - src/FrostFS.SDK.ModelsV2/ContainerId.cs | 28 +- src/FrostFS.SDK.ModelsV2/Object/Object.cs | 65 ++++ .../Object/ObjectAttribute.cs | 13 + .../{Object.cs => Object/ObjectFilter.cs} | 39 --- .../Object/ObjectHeader.cs | 37 ++ src/FrostFS.SDK.ModelsV2/Splitter.cs | 88 +++++ .../object/Extension.Message.cs | 56 ++++ .../session/Extension.XHeader.cs | 2 +- 22 files changed, 717 insertions(+), 193 deletions(-) create mode 100644 src/FrostFS.SDK.ClientV2/Extensions/Object.cs create mode 100644 src/FrostFS.SDK.ModelsV2/Object/Object.cs create mode 100644 src/FrostFS.SDK.ModelsV2/Object/ObjectAttribute.cs rename src/FrostFS.SDK.ModelsV2/{Object.cs => Object/ObjectFilter.cs} (56%) create mode 100644 src/FrostFS.SDK.ModelsV2/Object/ObjectHeader.cs create mode 100644 src/FrostFS.SDK.ModelsV2/Splitter.cs diff --git a/FrostFS.SDK.sln b/FrostFS.SDK.sln index d7d789e..67e5eb3 100644 --- a/FrostFS.SDK.sln +++ b/FrostFS.SDK.sln @@ -29,9 +29,8 @@ Global {5012EF96-9C9E-4E77-BC78-B4111EE54107}.Debug|Any CPU.Build.0 = Debug|Any CPU {5012EF96-9C9E-4E77-BC78-B4111EE54107}.Release|Any CPU.ActiveCfg = Release|Any CPU {5012EF96-9C9E-4E77-BC78-B4111EE54107}.Release|Any CPU.Build.0 = Release|Any CPU - {B738F3E1-654D-41A3-B068-58ED122BB688}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B738F3E1-654D-41A3-B068-58ED122BB688}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B738F3E1-654D-41A3-B068-58ED122BB688}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B738F3E1-654D-41A3-B068-58ED122BB688}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 832b42f..542a544 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2.Enums; using FrostFS.SDK.ModelsV2.Netmap; -var fsClient = new Client(, ); +var fsClient = Client.GetInstance(, ); // List containers var containersIds = await fsClient.ListContainersAsync(); @@ -55,7 +55,7 @@ using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2.Enums; using FrostFS.SDK.ModelsV2.Netmap; -var fsClient = new Client(, ); +var fsClient = Client.GetInstance(, ); // Search regular objects var objectsIds = await fsClient.SearchObjectAsync( @@ -77,4 +77,77 @@ var objHeader = await fsClient.GetObjectHeadAsync(cId, oId); // Get object var obj = await fsClient.GetObjectAsync(cId, oId); +``` + +### Custom client cut +```csharp + +using FrostFS.SDK.ClientV2; +using FrostFS.SDK.ModelsV2; +using FrostFS.SDK.ModelsV2.Enums; +using FrostFS.SDK.ModelsV2.Netmap; +using FrostFS.SDK.ClientV2.Extensions; +using FrostFS.SDK.ClientV2.Interfaces; + +var fsClient = Client.GetInstance(, ); + +ContainerId containerId = ; +string fileName = ; + +await PutObjectClientCut(fsClient, containerId, fileName); + +static async Task PutObjectClientCut(IFrostFSClient fsClient, ContainerId containerId, string fileName) +{ + List sentObjectIds = []; + FrostFS.SDK.ModelsV2.Object? currentObject; + + var partSize = 1024 * 1024; + var buffer = new byte[partSize]; + + var largeObject = new LargeObject(containerId); + + var split = new Split(); + + var fileInfo = new FileInfo(fileName); + var fullLength = (ulong)fileInfo.Length; + var fileNameAttribute = new ObjectAttribute("fileName", fileInfo.Name); + + using var stream = File.OpenRead(fileName); + while (true) + { + var bytesCount = await stream.ReadAsync(buffer.AsMemory(0, partSize)); + + split.Previous = sentObjectIds.LastOrDefault(); + + largeObject.AppendBlock(buffer, bytesCount); + + currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, buffer) + .AddAttribute(fileNameAttribute) + .SetSplit(split); + + if (largeObject.PayloadLength == fullLength) + break; + + var objectId = await fsClient.PutSingleObjectAsync(currentObject); + sentObjectIds.Add(objectId); + } + + if (sentObjectIds.Any()) + { + largeObject.CalculateHash(); + + var linkObject = new LinkObject(containerId, split.SplitId, largeObject) + .AddChildren(sentObjectIds); + + _ = await fsClient.PutSingleObjectAsync(linkObject); + + currentObject.SetParent(largeObject); + _ = await fsClient.PutSingleObjectAsync(currentObject); + + return currentObject.GetParentId(); + } + + return await fsClient.PutSingleObjectAsync(currentObject); +} + ``` \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Client.cs b/src/FrostFS.SDK.ClientV2/Client.cs index 7cb47d6..7e9e29c 100644 --- a/src/FrostFS.SDK.ClientV2/Client.cs +++ b/src/FrostFS.SDK.ClientV2/Client.cs @@ -1,5 +1,6 @@ using System; using System.Security.Cryptography; +using System.Threading.Tasks; using FrostFS.Container; using FrostFS.Netmap; using FrostFS.Object; @@ -25,7 +26,12 @@ public partial class Client: IFrostFSClient private ObjectService.ObjectServiceClient? _objectServiceClient; private SessionService.SessionServiceClient? _sessionServiceClient; - public Client(string key, string host) + public static IFrostFSClient GetInstance(string key, string host) + { + return new Client(key, host); + } + + private Client(string key, string host) { // TODO: Развязать клиент и реализацию GRPC _key = key.LoadWif(); @@ -37,7 +43,7 @@ public partial class Client: IFrostFSClient InitSessionClient(); CheckFrostFsVersionSupport(); } - + private async void CheckFrostFsVersionSupport() { var localNodeInfo = await GetLocalNodeInfoAsync(); diff --git a/src/FrostFS.SDK.ClientV2/Extensions/Object.cs b/src/FrostFS.SDK.ClientV2/Extensions/Object.cs new file mode 100644 index 0000000..93d8d5f --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Extensions/Object.cs @@ -0,0 +1,44 @@ + +using System.Collections.Generic; +using FrostFS.SDK.ModelsV2; + +namespace FrostFS.SDK.ClientV2.Extensions; + +public static class Extensions +{ + public static ModelsV2.Object SetPayloadLength(this ModelsV2.Object obj, ulong length) + { + obj.Header.PayloadLength = length; + return obj; + } + + public static ModelsV2.Object AddAttribute(this ModelsV2.Object obj, string key, string value) + { + obj.AddAttribute(new ObjectAttribute(key, value)); + return obj; + } + + public static ModelsV2.Object AddAttribute(this ModelsV2.Object obj, ObjectAttribute attribute) + { + obj.Header.Attributes.Add(attribute); + return obj; + } + + public static ModelsV2.Object AddAttributes(this ModelsV2.Object obj, IEnumerable attributes) + { + obj.Header.Attributes.AddRange(attributes); + return obj; + } + + public static ModelsV2.Object SetSplit(this ModelsV2.Object obj, Split split) + { + obj.Header.Split = split; + return obj; + } + + public static LinkObject AddChildren(this LinkObject linkObject, IEnumerable objectIds) + { + linkObject.Header.Split.Children.AddRange(objectIds); + return linkObject; + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj b/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj index 0208178..d446e0e 100644 --- a/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj +++ b/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj @@ -8,6 +8,7 @@ + diff --git a/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs b/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs index 8412c6c..5b82d1b 100644 --- a/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs +++ b/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs @@ -9,13 +9,24 @@ namespace FrostFS.SDK.ClientV2.Interfaces; public interface IFrostFSClient { Task GetContainerAsync(ContainerId containerId); + IAsyncEnumerable ListContainersAsync(); + Task CreateContainerAsync(ModelsV2.Container container); + Task DeleteContainerAsync(ContainerId containerId); + Task GetObjectHeadAsync(ContainerId containerId, ObjectId objectId); + Task GetObjectAsync(ContainerId containerId, ObjectId objectId); + Task PutObjectAsync(ObjectHeader header, Stream payload); + Task PutObjectAsync(ObjectHeader header, byte[] payload); + + Task PutSingleObjectAsync(ModelsV2.Object obj); + Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId); - IAsyncEnumerable SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters); + + IAsyncEnumerable SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters); } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ContainerId.cs b/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ContainerId.cs index 1bfd614..920c49a 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ContainerId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ContainerId.cs @@ -1,4 +1,5 @@ using FrostFS.Refs; +using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; using Google.Protobuf; @@ -10,7 +11,7 @@ public static class ContainerIdMapper { return new ContainerID { - Value = ByteString.CopyFrom(containerId.ToHash()) + Value = ByteString.CopyFrom(Base58.Decode(containerId.Value)) }; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs b/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs index b6e68f6..ef21202 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/GRPC/Object.cs @@ -1,8 +1,9 @@ using System; using System.Linq; - using FrostFS.Object; +using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; +using Google.Protobuf; using MatchType = FrostFS.Object.MatchType; using ObjectType = FrostFS.Object.ObjectType; @@ -29,11 +30,20 @@ public static class ObjectFilterMapper { public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this ObjectFilter filter) { - var objMatchTypeName = Enum.GetName(typeof(MatchType), filter.MatchType) - ?? throw new ArgumentException($"Unknown MatchType. Value: '{filter.MatchType}'."); + var objMatchTypeName = filter.MatchType switch + { + ModelsV2.Enums.ObjectMatchType.Unspecified => MatchType.Unspecified, + ModelsV2.Enums.ObjectMatchType.Equals => MatchType.StringEqual, + ModelsV2.Enums.ObjectMatchType.NotEquals => MatchType.StringNotEqual, + ModelsV2.Enums.ObjectMatchType.KeyAbsent => MatchType.NotPresent, + ModelsV2.Enums.ObjectMatchType.StartsWith => MatchType.CommonPrefix, + + _ => throw new ArgumentException($"Unknown MatchType. Value: '{filter.MatchType}'.") + }; + return new SearchRequest.Types.Body.Types.Filter { - MatchType = (MatchType)Enum.Parse(typeof(MatchType), objMatchTypeName), + MatchType = objMatchTypeName, Key = filter.Key, Value = filter.Value }; @@ -44,13 +54,19 @@ public static class ObjectHeaderMapper { public static Header ToGrpcMessage(this ObjectHeader header) { - var objTypeName = Enum.GetName(typeof(ObjectType), header.ObjectType) - ?? throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'."); - var head = new Header + var objTypeName = header.ObjectType switch { - Attributes = { }, + ModelsV2.Enums.ObjectType.Regular => ObjectType.Regular, + ModelsV2.Enums.ObjectType.Lock => ObjectType.Lock, + ModelsV2.Enums.ObjectType.Tombstone => ObjectType.Tombstone, + _ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.") + }; + + var head = new Header + { ContainerId = header.ContainerId.ToGrpcMessage(), - ObjectType = (ObjectType)Enum.Parse(typeof(ObjectType), objTypeName) + ObjectType = objTypeName, + PayloadLength = header.PayloadLength }; foreach (var attribute in header.Attributes) @@ -58,20 +74,42 @@ public static class ObjectHeaderMapper head.Attributes.Add(attribute.ToGrpcMessage()); } + var split = header.Split; + if (split != null) + { + head.Split = new Header.Types.Split + { + Parent = split.Parent?.ToGrpcMessage(), + ParentSignature = split.ParentSignature?.ToGrpcMessage(), + ParentHeader = split.ParentHeader?.ToGrpcMessage(), + Previous = split.Previous?.ToGrpcMessage(), + SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null + }; + + if (split.Children != null && split.Children.Any()) + head.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage())); + } + return head; } public static ObjectHeader ToModel(this Header header) { - var objTypeName = Enum.GetName(typeof(ModelsV2.Enums.ObjectType), header.ObjectType) - ?? throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'."); + var objTypeName = header.ObjectType switch + { + ObjectType.Regular => ModelsV2.Enums.ObjectType.Regular, + ObjectType.Lock => ModelsV2.Enums.ObjectType.Lock, + ObjectType.Tombstone => ModelsV2.Enums.ObjectType.Tombstone, + _ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.") + }; + return new ObjectHeader( - ContainerId.FromHash(header.ContainerId.Value.ToByteArray()), - (ModelsV2.Enums.ObjectType)Enum.Parse(typeof(ModelsV2.Enums.ObjectType), objTypeName), + new ContainerId(Base58.Encode(header.ContainerId.Value.ToByteArray())), + objTypeName, header.Attributes.Select(attribute => attribute.ToModel()).ToArray() ) { - Size = (long)header.PayloadLength, + PayloadLength = header.PayloadLength, Version = header.Version.ToModel() }; } @@ -81,11 +119,33 @@ public static class ObjectMapper { public static ModelsV2.Object ToModel(this Object.Object obj) { - return new ModelsV2.Object + return new ModelsV2.Object() { Header = obj.Header.ToModel(), ObjectId = ObjectId.FromHash(obj.ObjectId.Value.ToByteArray()), Payload = obj.Payload.ToByteArray() }; + } +} + +public static class SignatureMapper +{ + public static Refs.Signature ToGrpcMessage(this Signature signature) + { + var scheme = signature.Scheme switch + { + SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256, + SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect, + SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512, + _ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", paramName: nameof(signature.Scheme)) + }; + + return new Refs.Signature + { + Key = ByteString.CopyFrom(signature.Key), + Scheme = scheme, + Sign = ByteString.CopyFrom(signature.Sign) + }; } -} \ No newline at end of file +} + diff --git a/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ObjectId.cs b/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ObjectId.cs index 4c28035..bfd568d 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ObjectId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/GRPC/ObjectId.cs @@ -13,4 +13,9 @@ public static class ObjectIdMapper Value = ByteString.CopyFrom(objectId.ToHash()) }; } + + public static ObjectId ToModel(this ObjectID objectId) + { + return ObjectId.FromHash(objectId.Value.ToByteArray()); + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/RequestSigner.cs b/src/FrostFS.SDK.ClientV2/RequestSigner.cs index 3747c92..23f8eb6 100644 --- a/src/FrostFS.SDK.ClientV2/RequestSigner.cs +++ b/src/FrostFS.SDK.ClientV2/RequestSigner.cs @@ -91,7 +91,7 @@ public static class RequestSigner { IRequest => new RequestVerificationHeader(), IResponse => new ResponseVerificationHeader(), - _ => throw new InvalidOperationException("Unsopported message type") + _ => throw new InvalidOperationException("Unsupported message type") }; var verifyOrigin = message.GetVerificationHeader(); diff --git a/src/FrostFS.SDK.ClientV2/Services/Container.cs b/src/FrostFS.SDK.ClientV2/Services/Container.cs index 886768c..7566492 100644 --- a/src/FrostFS.SDK.ClientV2/Services/Container.cs +++ b/src/FrostFS.SDK.ClientV2/Services/Container.cs @@ -1,5 +1,6 @@ using FrostFS.Container; using FrostFS.SDK.ClientV2.Mappers.GRPC; +using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; using System.Collections.Generic; using System.Threading.Tasks; @@ -17,6 +18,7 @@ public partial class Client ContainerId = cid.ToGrpcMessage() }, }; + request.AddMetaHeader(); request.Sign(_key); var response = await _containerServiceClient.GetAsync(request); @@ -33,13 +35,14 @@ public partial class Client OwnerId = OwnerId.ToGrpcMessage() } }; + request.AddMetaHeader(); request.Sign(_key); var response = await _containerServiceClient.ListAsync(request); Verifier.CheckResponse(response); foreach (var cid in response.Body.ContainerIds) { - yield return ContainerId.FromHash(cid.Value.ToByteArray()); + yield return new ContainerId(Base58.Encode(cid.Value.ToByteArray())); } } @@ -48,6 +51,7 @@ public partial class Client var cntnr = container.ToGrpcMessage(); cntnr.OwnerId = OwnerId.ToGrpcMessage(); cntnr.Version = Version.ToGrpcMessage(); + var request = new PutRequest { Body = new PutRequest.Types.Body @@ -60,7 +64,7 @@ public partial class Client request.Sign(_key); var response = await _containerServiceClient.PutAsync(request); Verifier.CheckResponse(response); - return ContainerId.FromHash(response.Body.ContainerId.Value.ToByteArray()); + return new ContainerId(Base58.Encode(response.Body.ContainerId.Value.ToByteArray())); } public async Task DeleteContainerAsync(ContainerId cid) diff --git a/src/FrostFS.SDK.ClientV2/Services/Object.cs b/src/FrostFS.SDK.ClientV2/Services/Object.cs index 180e351..bc98dc8 100644 --- a/src/FrostFS.SDK.ClientV2/Services/Object.cs +++ b/src/FrostFS.SDK.ClientV2/Services/Object.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using Google.Protobuf; @@ -9,9 +10,10 @@ using FrostFS.Object; using FrostFS.Refs; using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.Cryptography; -using FrostFS.SDK.ModelsV2; using FrostFS.Session; +using FrostFS.SDK.ModelsV2; + namespace FrostFS.SDK.ClientV2; public partial class Client @@ -20,7 +22,7 @@ public partial class Client { var request = new HeadRequest { - Body = new HeadRequest.Types.Body + Body = new HeadRequest.Types.Body { Address = new Address { @@ -29,12 +31,12 @@ public partial class Client } } }; - + request.AddMetaHeader(); request.Sign(_key); - var response = await _objectServiceClient.HeadAsync(request); + var response = await _objectServiceClient!.HeadAsync(request); Verifier.CheckResponse(response); - + return response.Body.Header.Header.ToModel(); } @@ -45,7 +47,6 @@ public partial class Client { Body = new GetRequest.Types.Body { - Raw = false, Address = new Address { ContainerId = cid.ToGrpcMessage(), @@ -76,85 +77,8 @@ public partial class Client public async Task PutObjectAsync(ObjectHeader header, byte[] payload) { - return await PutObject(header, new MemoryStream(payload)); - } - - public async Task DeleteObjectAsync(ContainerId cid, ObjectId oid) - { - var request = new DeleteRequest - { - Body = new DeleteRequest.Types.Body - { - Address = new Address - { - ContainerId = cid.ToGrpcMessage(), - ObjectId = oid.ToGrpcMessage() - } - } - }; - - request.AddMetaHeader(); - request.Sign(_key); - var response = await _objectServiceClient.DeleteAsync(request); - Verifier.CheckResponse(response); - } - - public async IAsyncEnumerable SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters) - { - var request = new SearchRequest - { - Body = new SearchRequest.Types.Body - { - ContainerId = cid.ToGrpcMessage(), - Filters = { }, - Version = 1 - } - }; - - foreach (var filter in filters) - { - request.Body.Filters.Add(filter.ToGrpcMessage()); - } - - request.AddMetaHeader(); - request.Sign(_key); - var objectsIds = SearchObjects(request); - - await foreach (var oid in objectsIds) - { - yield return ObjectId.FromHash(oid.Value.ToByteArray()); - } - } - - private async Task GetObject(GetRequest request) - { - using var stream = GetObjectInit(request); - var obj = await stream.ReadHeader(); - var payload = new byte[obj.Header.PayloadLength]; - var offset = 0; - var chunk = await stream.ReadChunk(); - - while (chunk is not null) - { - chunk.CopyTo(payload, offset); - offset += chunk.Length; - chunk = await stream.ReadChunk(); - } - - obj.Payload = ByteString.CopyFrom(payload); - - return obj; - } - - private ObjectReader GetObjectInit(GetRequest initRequest) - { - if (initRequest is null) - throw new ArgumentNullException(nameof(initRequest)); - - return new ObjectReader - { - Call = _objectServiceClient.Get(initRequest) - }; + using var stream = new MemoryStream(payload); + return await PutObject(header, stream); } private async Task PutObject(ObjectHeader header, Stream payload) @@ -163,12 +87,12 @@ public partial class Client var hdr = header.ToGrpcMessage(); hdr.OwnerId = OwnerId.ToGrpcMessage(); hdr.Version = Version.ToGrpcMessage(); - + var oid = new ObjectID { Value = hdr.Sha256() }; - + var request = new PutRequest { Body = new PutRequest.Types.Body @@ -188,23 +112,27 @@ public partial class Client ObjectSessionContext.Types.Verb.Put, _key ); - + request.Sign(_key); using var stream = await PutObjectInit(request); var buffer = new byte[Constants.ObjectChunkSize]; - var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize); - - while (bufferLength > 0) + + while (true) { + var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize); + + if (bufferLength == 0) + break; + request.Body = new PutRequest.Types.Body { Chunk = ByteString.CopyFrom(buffer[..bufferLength]), }; + request.VerifyHeader = null; request.Sign(_key); await stream.Write(request); - bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize); } var response = await stream.Close(); @@ -213,6 +141,192 @@ public partial class Client return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()); } + public async Task PutSingleObjectAsync(ModelsV2.Object @object) + { + var sessionToken = await CreateSessionAsync(uint.MaxValue); + + var obj = CreateObject(@object); + + var request = new PutSingleRequest + { + Body = new () { Object = obj } + }; + + request.AddMetaHeader(); + request.AddObjectSessionToken( + sessionToken, + obj.Header.ContainerId, + obj.ObjectId, + ObjectSessionContext.Types.Verb.Put, + _key + ); + + request.Sign(_key); + + var response = await _objectServiceClient!.PutSingleAsync(request); + Verifier.CheckResponse(response); + + return ObjectId.FromHash(obj.ObjectId.Value.ToByteArray()); + } + + public Object.Object CreateObject(ModelsV2.Object @object) + { + var grpcHeader = @object.Header.ToGrpcMessage(); + + grpcHeader.OwnerId = OwnerId.ToGrpcMessage(); + grpcHeader.Version = Version.ToGrpcMessage(); + + if (@object.Payload != null) + { + grpcHeader.PayloadLength = (ulong)@object.Payload.Length; + grpcHeader.PayloadHash = Sha256Checksum(@object.Payload); + } + + var split = @object.Header.Split; + if (split != null) + { + grpcHeader.Split = new Header.Types.Split + { + SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null + }; + + if (split.Children != null && split.Children.Any()) + grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage())); + + if (split.ParentHeader is not null) + { + var grpcParentHeader = CreateHeader(split.ParentHeader, []); + + grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() }; + grpcHeader.Split.ParentHeader = grpcParentHeader; + grpcHeader.Split.ParentSignature = new Refs.Signature + { + Key = ByteString.CopyFrom(_key.PublicKey()), + Sign = ByteString.CopyFrom(_key.SignData(grpcHeader.Split.Parent.ToByteArray())), + }; + + split.Parent = grpcHeader.Split.Parent.ToModel(); + } + } + + var obj = new Object.Object + { + Header = grpcHeader, + ObjectId = new ObjectID { Value = grpcHeader.Sha256() }, + Payload = ByteString.CopyFrom(@object.Payload) + }; + + obj.Signature = new Refs.Signature + { + Key = ByteString.CopyFrom(_key.PublicKey()), + Sign = ByteString.CopyFrom(_key.SignData(obj.ObjectId.ToByteArray())), + }; + + return obj; + } + + public Header CreateHeader(ObjectHeader header, byte[]? payload) + { + var grpcHeader = header.ToGrpcMessage(); + + grpcHeader.OwnerId = OwnerId.ToGrpcMessage(); + grpcHeader.Version = Version.ToGrpcMessage(); + + if (header.PayloadCheckSum != null) + { + grpcHeader.PayloadHash = new Checksum + { + Type = ChecksumType.Sha256, + Sum = ByteString.CopyFrom(header.PayloadCheckSum) + }; + } + else + { + if (payload != null) + grpcHeader.PayloadHash = Sha256Checksum(payload); + } + + return grpcHeader; + } + + public async Task DeleteObjectAsync(ContainerId cid, ObjectId oid) + { + var request = new DeleteRequest + { + Body = new DeleteRequest.Types.Body + { + Address = new Address + { + ContainerId = cid.ToGrpcMessage(), + ObjectId = oid.ToGrpcMessage() + } + } + }; + + request.AddMetaHeader(); + request.Sign(_key); + var response = await _objectServiceClient!.DeleteAsync(request); + Verifier.CheckResponse(response); + } + + public async IAsyncEnumerable SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters) + { + var request = new SearchRequest + { + Body = new SearchRequest.Types.Body + { + ContainerId = cid.ToGrpcMessage(), + Filters = { }, + Version = 1 + } + }; + + foreach (var filter in filters) + { + request.Body.Filters.Add(filter.ToGrpcMessage()); + } + + request.AddMetaHeader(); + request.Sign(_key); + var objectsIds = SearchObjects(request); + + await foreach (var oid in objectsIds) + { + yield return ObjectId.FromHash(oid.Value.ToByteArray()); + } + } + + private async Task GetObject(GetRequest request) + { + using var stream = GetObjectInit(request); + var obj = await stream.ReadHeader(); + var payload = new byte[obj.Header.PayloadLength]; + var offset = 0; + var chunk = await stream.ReadChunk(); + + while (chunk is not null) + { + chunk.CopyTo(payload, offset); + offset += chunk.Length; + chunk = await stream.ReadChunk(); + } + + obj.Payload = ByteString.CopyFrom(payload); + + return obj; + } + + private ObjectReader GetObjectInit(GetRequest initRequest) + { + if (initRequest is null) + throw new ArgumentNullException(nameof(initRequest)); + + return new ObjectReader + { + Call = _objectServiceClient!.Get(initRequest) + }; + } + private async Task PutObjectInit(PutRequest initRequest) { if (initRequest is null) @@ -220,9 +334,9 @@ public partial class Client throw new ArgumentNullException(nameof(initRequest)); } - var call = _objectServiceClient.Put(); + var call = _objectServiceClient!.Put(); await call.RequestStream.WriteAsync(initRequest); - + return new ObjectStreamer(call); } @@ -236,7 +350,7 @@ public partial class Client { yield return oid; } - + ids = await stream.Read(); } } @@ -248,6 +362,17 @@ public partial class Client throw new ArgumentNullException(nameof(initRequest)); } - return new SearchReader(_objectServiceClient.Search(initRequest)); + return new SearchReader(_objectServiceClient!.Search(initRequest)); + } + + public Checksum Sha256Checksum(byte[] data) + { + return new Checksum + { + Type = ChecksumType.Sha256, + Sum = ByteString.CopyFrom(data.Sha256()) + }; } } + + diff --git a/src/FrostFS.SDK.Cryptography/Base58.cs b/src/FrostFS.SDK.Cryptography/Base58.cs index 36711c0..d9d91cf 100644 --- a/src/FrostFS.SDK.Cryptography/Base58.cs +++ b/src/FrostFS.SDK.Cryptography/Base58.cs @@ -3,8 +3,6 @@ using System.Linq; using System.Numerics; using System.Text; -using Org.BouncyCastle.Security.Certificates; - namespace FrostFS.SDK.Cryptography; public static class Base58 diff --git a/src/FrostFS.SDK.Cryptography/Helper.cs b/src/FrostFS.SDK.Cryptography/Helper.cs index 93058d9..266a193 100644 --- a/src/FrostFS.SDK.Cryptography/Helper.cs +++ b/src/FrostFS.SDK.Cryptography/Helper.cs @@ -44,7 +44,6 @@ public static class Helper { return ByteString.CopyFrom(data.ToByteArray().Sha256()); } - public static ulong Murmur64(this byte[] value, uint seed) { diff --git a/src/FrostFS.SDK.ModelsV2/ContainerId.cs b/src/FrostFS.SDK.ModelsV2/ContainerId.cs index 8bfbc54..349eb1c 100644 --- a/src/FrostFS.SDK.ModelsV2/ContainerId.cs +++ b/src/FrostFS.SDK.ModelsV2/ContainerId.cs @@ -1,30 +1,8 @@ -using System; +namespace FrostFS.SDK.ModelsV2; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.ModelsV2; - -public class ContainerId +public class ContainerId(string id) { - public string Value { get; set; } - - public ContainerId(string id) - { - Value = id; - } - - public static ContainerId FromHash(byte[] hash) - { - if (hash.Length != Constants.Sha256HashLength) - throw new FormatException("ContainerID must be a sha256 hash."); - - return new ContainerId(Base58.Encode(hash)); - } - - public byte[] ToHash() - { - return Base58.Decode(Value); - } + public string Value { get; set; } = id; public override string ToString() { diff --git a/src/FrostFS.SDK.ModelsV2/Object/Object.cs b/src/FrostFS.SDK.ModelsV2/Object/Object.cs new file mode 100644 index 0000000..5810699 --- /dev/null +++ b/src/FrostFS.SDK.ModelsV2/Object/Object.cs @@ -0,0 +1,65 @@ +using System.Security.Cryptography; +using FrostFS.SDK.ModelsV2.Enums; + +namespace FrostFS.SDK.ModelsV2; + +public class Object +{ + public Object() + { + } + + public Object(ContainerId container, byte[] payload, ObjectType objectType = ObjectType.Regular) + { + Payload = payload; + Header = new ObjectHeader(containerId: container, type: objectType, attributes: []); + } + + public ObjectHeader Header { get; set; } + public ObjectId ObjectId { get; set; } + public byte[] Payload { get; set; } + public Signature Signature { get; set; } + + public void SetParent(LargeObject largeObject) + { + Header.Split!.ParentHeader = largeObject.Header; + } + + public ObjectId? GetParentId() + { + return Header.Split?.Parent; + } +} + +public class LargeObject(ContainerId container) : Object(container, []) +{ + private SHA256 payloadHash = SHA256.Create(); + + public void AppendBlock(byte[] bytes, int count) + { + Header.PayloadLength += (ulong)count; + this.payloadHash.TransformBlock(bytes, 0, count, bytes, 0); + } + + public void CalculateHash() + { + this.payloadHash.TransformFinalBlock([], 0, 0); + Header.PayloadCheckSum = this.payloadHash.Hash; + } + + public ulong PayloadLength + { + get { return Header.PayloadLength; } + } +} + +public class LinkObject : Object +{ + public LinkObject(ContainerId containerId, SplitId splitId, LargeObject largeObject) : base (containerId, []) + { + Header.Split = new Split(splitId) + { + ParentHeader = largeObject.Header + }; + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.ModelsV2/Object/ObjectAttribute.cs b/src/FrostFS.SDK.ModelsV2/Object/ObjectAttribute.cs new file mode 100644 index 0000000..b114f2b --- /dev/null +++ b/src/FrostFS.SDK.ModelsV2/Object/ObjectAttribute.cs @@ -0,0 +1,13 @@ +namespace FrostFS.SDK.ModelsV2; + +public class ObjectAttribute +{ + public string Key { get; set; } + public string Value { get; set; } + + public ObjectAttribute(string key, string value) + { + Key = key; + Value = value; + } +} diff --git a/src/FrostFS.SDK.ModelsV2/Object.cs b/src/FrostFS.SDK.ModelsV2/Object/ObjectFilter.cs similarity index 56% rename from src/FrostFS.SDK.ModelsV2/Object.cs rename to src/FrostFS.SDK.ModelsV2/Object/ObjectFilter.cs index 7db6289..5fdb54f 100644 --- a/src/FrostFS.SDK.ModelsV2/Object.cs +++ b/src/FrostFS.SDK.ModelsV2/Object/ObjectFilter.cs @@ -2,18 +2,6 @@ using FrostFS.SDK.ModelsV2.Enums; namespace FrostFS.SDK.ModelsV2; -public class ObjectAttribute -{ - public string Key { get; set; } - public string Value { get; set; } - - public ObjectAttribute(string key, string value) - { - Key = key; - Value = value; - } -} - public class ObjectFilter { private const string HeaderPrefix = "$Object:"; @@ -48,30 +36,3 @@ public class ObjectFilter return new ObjectFilter(matchType, HeaderPrefix + "version", version.ToString()); } } - -public class ObjectHeader -{ - public ObjectAttribute[] Attributes { get; set; } - public ContainerId ContainerId { get; set; } - public long Size { get; set; } - public ObjectType ObjectType { get; set; } - public Version Version { get; set; } - - public ObjectHeader( - ContainerId containerId, - ObjectType type = ObjectType.Regular, - params ObjectAttribute[] attributes - ) - { - Attributes = attributes; - ContainerId = containerId; - ObjectType = type; - } -} - -public class Object -{ - public ObjectHeader Header { get; set; } - public ObjectId ObjectId { get; set; } - public byte[] Payload { get; set; } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.ModelsV2/Object/ObjectHeader.cs b/src/FrostFS.SDK.ModelsV2/Object/ObjectHeader.cs new file mode 100644 index 0000000..8f51160 --- /dev/null +++ b/src/FrostFS.SDK.ModelsV2/Object/ObjectHeader.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; + +using FrostFS.SDK.ModelsV2.Enums; + +namespace FrostFS.SDK.ModelsV2; + +public class ObjectHeader +{ + public OwnerId OwnerId { get; set; } + + public List Attributes { get; set; } + + public ContainerId ContainerId { get; set; } + + public ulong PayloadLength { get; set; } + + public byte[]? PayloadCheckSum { get; set; } + + public ObjectType ObjectType { get; set; } + + public Version Version { get; set; } + + public Split? Split { get; set; } + + public bool ClientCut { get; set; } + + public ObjectHeader( + ContainerId containerId, + ObjectType type = ObjectType.Regular, + params ObjectAttribute[] attributes + ) + { + Attributes = [.. attributes]; + ContainerId = containerId; + ObjectType = type; + } +} diff --git a/src/FrostFS.SDK.ModelsV2/Splitter.cs b/src/FrostFS.SDK.ModelsV2/Splitter.cs new file mode 100644 index 0000000..f48f903 --- /dev/null +++ b/src/FrostFS.SDK.ModelsV2/Splitter.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; + +namespace FrostFS.SDK.ModelsV2; + +public class Split +{ + public Split() : this(new SplitId()) + { + } + + public Split(SplitId splitId) + { + SplitId = splitId; + } + + public SplitId SplitId { get; private set; } + + public ObjectId? Parent { get; set; } + + public ObjectId? Previous { get; set; } + + public Signature? ParentSignature { get; set; } + + public ObjectHeader? ParentHeader { get; set; } + + public List Children { get; } = []; +} + + public enum SignatureScheme { + EcdsaSha512, + EcdsaRfc6979Sha256, + EcdsaRfc6979Sha256WalletConnect + } + +public class Signature +{ + public byte[] Key { get; set; } + public byte[] Sign { get; set; } + public SignatureScheme Scheme { get; set; } +} + +public class SplitId +{ + private Guid id; + + public SplitId() + { + this.id = Guid.NewGuid(); + } + public SplitId(Guid guid) + { + this.id = guid; + } + + private SplitId(byte[] binary) + { + this.id = new Guid(binary); + } + + private SplitId(string str) + { + this.id = new Guid(str); + } + + public static SplitId CrateFromBinary(byte[] binaryData) + { + return new SplitId(binaryData); + } + + public static SplitId CrateFromString(string stringData) + { + return new SplitId(stringData); + } + + public override string ToString() + { + return this.id.ToString(); + } + + public byte[]? ToBinary() + { + if (this.id == Guid.Empty) + return null; + + return this.id.ToByteArray(); + } +} diff --git a/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs b/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs index 243ab00..6334c71 100644 --- a/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs +++ b/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs @@ -115,6 +115,62 @@ namespace FrostFS.Object } } + public partial class PutSingleRequest : IRequest + { + IMetaHeader IVerificableMessage.GetMetaHeader() + { + return MetaHeader; + } + + IVerificationHeader IVerificableMessage.GetVerificationHeader() + { + return VerifyHeader; + } + + void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader) + { + MetaHeader = (RequestMetaHeader)metaHeader; + } + + void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader) + { + VerifyHeader = (RequestVerificationHeader)verificationHeader; + } + + public IMessage GetBody() + { + return Body; + } + } + + public partial class PutSingleResponse : IResponse + { + IMetaHeader IVerificableMessage.GetMetaHeader() + { + return MetaHeader; + } + + IVerificationHeader IVerificableMessage.GetVerificationHeader() + { + return VerifyHeader; + } + + void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader) + { + MetaHeader = (ResponseMetaHeader)metaHeader; + } + + void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader) + { + VerifyHeader = (ResponseVerificationHeader)verificationHeader; + } + + public IMessage GetBody() + { + return Body; + } + } + public partial class DeleteRequest : IRequest { IMetaHeader IVerificableMessage.GetMetaHeader() diff --git a/src/FrostFS.SDK.ProtosV2/session/Extension.XHeader.cs b/src/FrostFS.SDK.ProtosV2/session/Extension.XHeader.cs index f8f2695..c0c5b25 100644 --- a/src/FrostFS.SDK.ProtosV2/session/Extension.XHeader.cs +++ b/src/FrostFS.SDK.ProtosV2/session/Extension.XHeader.cs @@ -2,7 +2,7 @@ public partial class XHeader { - public const string ReservedXHeaderPrefix = "__NEOFS__"; + public const string ReservedXHeaderPrefix = "__SYSTEM__"; public const string XHeaderNetmapEpoch = ReservedXHeaderPrefix + "NETMAP_EPOCH"; public const string XHeaderNetmapLookupDepth = ReservedXHeaderPrefix + "NETMAP_LOOKUP_DEPTH"; }