From 0ddde467cdb734a6f77cda2a18258752347e891f Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Thu, 1 Aug 2024 16:17:36 +0300 Subject: [PATCH 1/2] [#20] Client: Optimize memory usage Avoid memory allocation, use cache and static Signed-off-by: Pavel Gross --- src/FrostFS.SDK.ClientV2/Cache.cs | 25 ++ src/FrostFS.SDK.ClientV2/Client.cs | 8 +- src/FrostFS.SDK.ClientV2/CllientKey.cs | 13 + .../Exceptions/ResponseException.cs | 13 +- .../FrostFS.SDK.ClientV2.csproj | 3 + src/FrostFS.SDK.ClientV2/Mappers/Container.cs | 4 +- .../Mappers/ContainerId.cs | 21 +- .../Mappers/MetaHeader.cs | 12 +- .../Mappers/Netmap/Netmap.cs | 4 +- .../Mappers/Netmap/PlacementPolicy.cs | 5 +- .../Mappers/Netmap/Replica.cs | 2 +- .../Mappers/Object/ObjectAttributeMapper.cs | 2 +- .../Mappers/Object/ObjectFilterMapper.cs | 2 +- .../Mappers/Object/ObjectHeaderMapper.cs | 17 +- .../Mappers/Object/ObjectId.cs | 2 +- src/FrostFS.SDK.ClientV2/Mappers/OwnerId.cs | 33 +- .../Mappers/SignatureMapper.cs | 5 +- src/FrostFS.SDK.ClientV2/Mappers/Status.cs | 8 +- src/FrostFS.SDK.ClientV2/Mappers/Version.cs | 68 +++- .../Parameters/PrmObjectPut.cs | 6 + .../Services/ContainerServiceProvider.cs | 24 +- .../Services/NetmapServiceProvider.cs | 6 +- .../Services/ObjectServiceProvider.cs | 301 ++++++++++-------- .../Services/SessionServiceProvider.cs | 4 +- .../Tools/ClientEnvironment.cs | 7 +- src/FrostFS.SDK.ClientV2/Tools/ObjectTools.cs | 92 +++--- .../Tools/RequestConstructor.cs | 2 +- .../Tools/RequestSigner.cs | 2 +- src/FrostFS.SDK.ClientV2/Tools/Verifier.cs | 5 +- src/FrostFS.SDK.Cryptography/Base58.cs | 8 +- src/FrostFS.SDK.Cryptography/Extentions.cs | 45 ++- src/FrostFS.SDK.Cryptography/Key.cs | 5 +- src/FrostFS.SDK.Cryptography/Murmur3_128.cs | 2 +- src/FrostFS.SDK.Cryptography/Range.cs | 14 +- .../FrostFS.SDK.ModelsV2.csproj | 1 + src/FrostFS.SDK.ModelsV2/Misc/CheckSum.cs | 9 +- .../Object/FrostFsObject.cs | 10 +- src/FrostFS.SDK.ModelsV2/Object/Splitter.cs | 3 + .../Response/{Status.cs => ResponseStatus.cs} | 2 +- src/FrostFS.SDK.Tests/ContainerTest.cs | 2 +- .../Mocks/AsyncStreamReaderMock.cs | 8 +- .../ContainerServiceBase.cs | 4 +- .../ContainerServiceMocks/GetContainerMock.cs | 6 +- src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs | 16 +- src/FrostFS.SDK.Tests/ObjectTest.cs | 119 ++++--- src/FrostFS.SDK.Tests/SmokeTests.cs | 18 +- 46 files changed, 596 insertions(+), 372 deletions(-) create mode 100644 src/FrostFS.SDK.ClientV2/Cache.cs create mode 100644 src/FrostFS.SDK.ClientV2/CllientKey.cs rename src/FrostFS.SDK.ModelsV2/Response/{Status.cs => ResponseStatus.cs} (83%) diff --git a/src/FrostFS.SDK.ClientV2/Cache.cs b/src/FrostFS.SDK.ClientV2/Cache.cs new file mode 100644 index 0000000..a41c8b9 --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Cache.cs @@ -0,0 +1,25 @@ + + +using Microsoft.Extensions.Caching.Memory; + +namespace FrostFS.SDK.ClientV2 +{ + internal static class Cache + { + private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions + { + // TODO: get from options? + SizeLimit = 256 + }); + + private static readonly IMemoryCache _containersCache = new MemoryCache(new MemoryCacheOptions + { + // TODO: get from options? + SizeLimit = 1024 + }); + + internal static IMemoryCache Owners => _ownersCache; + + internal static IMemoryCache Containers => _containersCache; + } +} diff --git a/src/FrostFS.SDK.ClientV2/Client.cs b/src/FrostFS.SDK.ClientV2/Client.cs index 294726a..faf1802 100644 --- a/src/FrostFS.SDK.ClientV2/Client.cs +++ b/src/FrostFS.SDK.ClientV2/Client.cs @@ -125,7 +125,7 @@ public class Client : IFrostFSClient public IAsyncEnumerable ListContainersAsync(PrmContainerGetAll? args = null) { - args = args ?? new PrmContainerGetAll(); + args ??= new PrmContainerGetAll(); var service = GetContainerService(args); return service.ListContainersAsync(args); } @@ -223,7 +223,10 @@ public class Client : IFrostFSClient #region ToolsImplementation public ObjectId CalculateObjectId(ObjectHeader header) { - return new ObjectTools(ClientCtx).CalculateObjectId(header); + if (header == null) + throw new ArgumentNullException(nameof(header)); + + return ObjectTools.CalculateObjectId(header, ClientCtx); } #endregion @@ -236,7 +239,6 @@ public class Client : IFrostFSClient if (!localNodeInfo.Version.IsSupported(ClientCtx.Version)) { var msg = $"FrostFS {localNodeInfo.Version} is not supported."; - Console.WriteLine(msg); throw new ApplicationException(msg); } } diff --git a/src/FrostFS.SDK.ClientV2/CllientKey.cs b/src/FrostFS.SDK.ClientV2/CllientKey.cs new file mode 100644 index 0000000..e415a98 --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/CllientKey.cs @@ -0,0 +1,13 @@ +using FrostFS.SDK.Cryptography; +using Google.Protobuf; +using System.Security.Cryptography; + +namespace FrostFS.SDK.ClientV2 +{ + public class ClientKey(ECDsa key) + { + internal ECDsa ECDsaKey { get; } = key; + + internal ByteString PublicKeyProto { get; } = ByteString.CopyFrom(key.PublicKey()); + } +} diff --git a/src/FrostFS.SDK.ClientV2/Exceptions/ResponseException.cs b/src/FrostFS.SDK.ClientV2/Exceptions/ResponseException.cs index 7772e1e..d72f173 100644 --- a/src/FrostFS.SDK.ClientV2/Exceptions/ResponseException.cs +++ b/src/FrostFS.SDK.ClientV2/Exceptions/ResponseException.cs @@ -1,12 +1,9 @@ -using System; using FrostFS.SDK.ModelsV2; +using System; -public class ResponseException : Exception +namespace FrostFS.SDK.ClientV2; + +public class ResponseException(ResponseStatus status) : Exception() { - public Status Status { get; set; } - - public ResponseException(Status status) : base() - { - Status = status; - } + public ResponseStatus Status { get; set; } = status; } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj b/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj index 818bce2..f811e53 100644 --- a/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj +++ b/src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj @@ -12,8 +12,11 @@ + + + diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Container.cs b/src/FrostFS.SDK.ClientV2/Mappers/Container.cs index 3d1207f..83bfb32 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Container.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Container.cs @@ -10,12 +10,12 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class ContainerMapper { - public static Container.Container ToGrpcMessage(this ModelsV2.Container container) + public static Container.Container ToMessage(this ModelsV2.Container container) { return new Container.Container { BasicAcl = (uint)container.BasicAcl, - PlacementPolicy = container.PlacementPolicy.ToGrpcMessage(), + PlacementPolicy = container.PlacementPolicy.ToMessage(), Nonce = ByteString.CopyFrom(container.Nonce.ToBytes()) }; } diff --git a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs index 920c49a..ac9634d 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs @@ -2,16 +2,29 @@ using FrostFS.Refs; using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; using Google.Protobuf; +using Microsoft.Extensions.Caching.Memory; +using System; namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class ContainerIdMapper { - public static ContainerID ToGrpcMessage(this ContainerId containerId) + private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromHours(1)) + .SetSize(1); + + public static ContainerID ToMessage(this ContainerId model) { - return new ContainerID + if (!Cache.Owners.TryGetValue(model, out ContainerID? message)) { - Value = ByteString.CopyFrom(Base58.Decode(containerId.Value)) - }; + message = new ContainerID + { + Value = ByteString.CopyFrom(Base58.Decode(model.Value)) + }; + + Cache.Owners.Set(model, message, _oneHourExpiration); + } + + return message!; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/MetaHeader.cs b/src/FrostFS.SDK.ClientV2/Mappers/MetaHeader.cs index d3bf04f..360ddd4 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/MetaHeader.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/MetaHeader.cs @@ -1,23 +1,17 @@ using FrostFS.SDK.ModelsV2; using FrostFS.Session; -using Version = FrostFS.Refs.Version; namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class MetaHeaderMapper { - public static RequestMetaHeader ToGrpcMessage(this MetaHeader metaHeader) + public static RequestMetaHeader ToMessage(this MetaHeader metaHeader) { return new RequestMetaHeader { - Version = new Version - { - Major = (uint)metaHeader.Version.Major, - Minor = (uint)metaHeader.Version.Minor, - - }, + Version = metaHeader.Version.ToMessage(), Epoch = (uint)metaHeader.Epoch, Ttl = (uint)metaHeader.Ttl }; - } + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Netmap.cs b/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Netmap.cs index 359041c..7d44809 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Netmap.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Netmap.cs @@ -10,6 +10,8 @@ public static class NetmapMapper { return new NetmapSnapshot( netmap.Body.Netmap.Epoch, - netmap.Body.Netmap.Nodes.Select(n => n.ToModel(netmap.MetaHeader.Version)).ToArray()); + netmap.Body.Netmap.Nodes + .Select(n => n.ToModel(netmap.MetaHeader.Version)) + .ToArray()); } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Netmap/PlacementPolicy.cs b/src/FrostFS.SDK.ClientV2/Mappers/Netmap/PlacementPolicy.cs index a4cbb0d..47b51fa 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Netmap/PlacementPolicy.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Netmap/PlacementPolicy.cs @@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap; public static class PlacementPolicyMapper { - public static PlacementPolicy ToGrpcMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy) + public static PlacementPolicy ToMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy) { var pp = new PlacementPolicy { @@ -14,9 +14,10 @@ public static class PlacementPolicyMapper Replicas = { }, Unique = placementPolicy.Unique }; + foreach (var replica in placementPolicy.Replicas) { - pp.Replicas.Add(replica.ToGrpcMessage()); + pp.Replicas.Add(replica.ToMessage()); } return pp; diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Replica.cs b/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Replica.cs index 7435085..215a127 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Replica.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Netmap/Replica.cs @@ -4,7 +4,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap; public static class ReplicaMapper { - public static Replica ToGrpcMessage(this ModelsV2.Netmap.Replica replica) + public static Replica ToMessage(this ModelsV2.Netmap.Replica replica) { return new Replica { diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectAttributeMapper.cs b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectAttributeMapper.cs index 75c4bb8..7528152 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectAttributeMapper.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectAttributeMapper.cs @@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class ObjectAttributeMapper { - public static Header.Types.Attribute ToGrpcMessage(this ObjectAttribute attribute) + public static Header.Types.Attribute ToMessage(this ObjectAttribute attribute) { return new Header.Types.Attribute { diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectFilterMapper.cs b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectFilterMapper.cs index 2d9fed7..8353f5f 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectFilterMapper.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectFilterMapper.cs @@ -7,7 +7,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class ObjectFilterMapper { - public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this IObjectFilter filter) + public static SearchRequest.Types.Body.Types.Filter ToMessage(this IObjectFilter filter) { var objMatchTypeName = filter.MatchType switch { diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectHeaderMapper.cs b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectHeaderMapper.cs index 12ff85c..282e42e 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectHeaderMapper.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectHeaderMapper.cs @@ -10,7 +10,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class ObjectHeaderMapper { - public static Header ToGrpcMessage(this ObjectHeader header) + public static Header ToMessage(this ObjectHeader header) { var objTypeName = header.ObjectType switch { @@ -22,14 +22,16 @@ public static class ObjectHeaderMapper var head = new Header { - ContainerId = header.ContainerId.ToGrpcMessage(), + OwnerId = header!.OwnerId != null ? header.OwnerId.ToMessage() : null, + Version = header!.Version != null ? header.Version.ToMessage() : null, + ContainerId = header.ContainerId.ToMessage(), ObjectType = objTypeName, PayloadLength = header.PayloadLength }; foreach (var attribute in header.Attributes) { - head.Attributes.Add(attribute.ToGrpcMessage()); + head.Attributes.Add(attribute.ToMessage()); } var split = header.Split; @@ -37,17 +39,10 @@ public static class ObjectHeaderMapper { head.Split = new Header.Types.Split { - Parent = split.Parent?.ToGrpcMessage(), - ParentSignature = split.ParentSignature?.ToGrpcMessage(), - ParentHeader = split.ParentHeader?.ToGrpcMessage(), - Previous = split.Previous?.ToGrpcMessage(), SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null }; - - if (split.Children != null && split.Children.Count != 0) - head.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage())); } - + return head; } diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectId.cs b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectId.cs index bfd568d..c9afe0c 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectId.cs @@ -6,7 +6,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class ObjectIdMapper { - public static ObjectID ToGrpcMessage(this ObjectId objectId) + public static ObjectID ToMessage(this ObjectId objectId) { return new ObjectID { diff --git a/src/FrostFS.SDK.ClientV2/Mappers/OwnerId.cs b/src/FrostFS.SDK.ClientV2/Mappers/OwnerId.cs index b4f332e..de8db49 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/OwnerId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/OwnerId.cs @@ -1,21 +1,42 @@ using FrostFS.Refs; +using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; using Google.Protobuf; +using Microsoft.Extensions.Caching.Memory; +using System; namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class OwnerIdMapper { - public static OwnerID ToGrpcMessage(this OwnerId ownerId) + private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromHours(1)) + .SetSize(1); + + public static OwnerID ToMessage(this OwnerId model) { - return new OwnerID + if (!Cache.Owners.TryGetValue(model, out OwnerID? message)) { - Value = ByteString.CopyFrom(ownerId.ToHash()) - }; + message = new OwnerID + { + Value = ByteString.CopyFrom(model.ToHash()) + }; + + Cache.Owners.Set(model, message, _oneHourExpiration); + } + + return message!; } - public static OwnerId ToModel(this OwnerID ownerId) + public static OwnerId ToModel(this OwnerID message) { - return new OwnerId(ownerId.ToString()); + if (!Cache.Owners.TryGetValue(message, out OwnerId? model)) + { + model = new OwnerId(Base58.Encode(message.Value.ToByteArray())); + + Cache.Owners.Set(message, model, _oneHourExpiration); + } + + return model!; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/SignatureMapper.cs b/src/FrostFS.SDK.ClientV2/Mappers/SignatureMapper.cs index 2da358a..4343d4d 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/SignatureMapper.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/SignatureMapper.cs @@ -6,14 +6,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class SignatureMapper { - public static Refs.Signature ToGrpcMessage(this Signature signature) + public static Refs.Signature ToMessage(this Signature signature) { var scheme = signature.Scheme switch { SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256, SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect, SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512, - _ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", paramName: nameof(signature.Scheme)) + _ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", + paramName: nameof(signature.Scheme)) }; return new Refs.Signature diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Status.cs b/src/FrostFS.SDK.ClientV2/Mappers/Status.cs index 9e3d2eb..d489239 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Status.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Status.cs @@ -5,13 +5,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class StatusMapper { - public static ModelsV2.Status ToModel(this Status.Status status) + public static ModelsV2.ResponseStatus ToModel(this Status.Status status) { - if (status is null) return new ModelsV2.Status(StatusCode.Success); + if (status is null) + return new ModelsV2.ResponseStatus(StatusCode.Success); + var codeName = Enum.GetName(typeof(StatusCode), status.Code); return codeName is null ? throw new ArgumentException($"Unknown StatusCode. Value: '{status.Code}'.") - : new ModelsV2.Status((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message); + : new ModelsV2.ResponseStatus((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message); } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Mappers/Version.cs b/src/FrostFS.SDK.ClientV2/Mappers/Version.cs index 549ff1a..df2066c 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/Version.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/Version.cs @@ -1,20 +1,74 @@ +using System.Collections; +using System.Threading; using Version = FrostFS.Refs.Version; namespace FrostFS.SDK.ClientV2.Mappers.GRPC; public static class VersionMapper { - public static Version ToGrpcMessage(this ModelsV2.Version version) + private static readonly Hashtable _cacheMessages = []; + private static readonly Hashtable _cacheModels = []; + private static SpinLock _spinlock = new(); + + public static Version ToMessage(this ModelsV2.Version model) { - return new Version + var key = model.Major << 16 + model.Minor; + + if (!_cacheMessages.ContainsKey(key)) { - Major = (uint)version.Major, - Minor = (uint)version.Minor - }; + bool lockTaken = false; + try + { + _spinlock.Enter(ref lockTaken); + var message = new Version + { + Major = (uint)model.Major, + Minor = (uint)model.Minor + }; + + _cacheMessages.Add(key, message); + return message; + } + catch (System.ArgumentException) + { + // ignore attempt to add duplicate error. + } + finally + { + if (lockTaken) + _spinlock.Exit(false); + } + } + + return (Version)_cacheMessages[key]; } - public static ModelsV2.Version ToModel(this Version version) + public static ModelsV2.Version ToModel(this Version message) { - return new ModelsV2.Version((int)version.Major, (int)version.Minor); + var key = (int)message.Major << 16 + (int)message.Minor; + + if (!_cacheModels.ContainsKey(key)) + { + bool lockTaken = false; + try + { + _spinlock.Enter(ref lockTaken); + var model = new ModelsV2.Version((int)message.Major, (int)message.Minor); + + _cacheModels.Add(key, model); + return model; + } + catch (System.ArgumentException) + { + // ignore attempt to add duplicate error. + } + finally + { + if (lockTaken) + _spinlock.Exit(false); + } + } + + return (ModelsV2.Version)_cacheModels[key]; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPut.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPut.cs index 219af3e..cd859d7 100644 --- a/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPut.cs +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPut.cs @@ -41,4 +41,10 @@ public sealed class PrmObjectPut : IContext, ISessionToken /// public SessionToken? SessionToken { get; set; } + + internal int MaxObjectSizeCache { get; set; } + + internal ulong CurrentStreamPosition { get; set; } = 0; + + internal ulong FullLength { get; set; } = 0; } diff --git a/src/FrostFS.SDK.ClientV2/Services/ContainerServiceProvider.cs b/src/FrostFS.SDK.ClientV2/Services/ContainerServiceProvider.cs index 2a15828..7f412c1 100644 --- a/src/FrostFS.SDK.ClientV2/Services/ContainerServiceProvider.cs +++ b/src/FrostFS.SDK.ClientV2/Services/ContainerServiceProvider.cs @@ -17,7 +17,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient { internal async Task GetContainerAsync(PrmContainerGet args) { - GetRequest request = GetContainerRequest(args.ContainerId.ToGrpcMessage(), args.XHeaders); + GetRequest request = GetContainerRequest(args.ContainerId.ToMessage(), args.XHeaders); var response = await service.GetAsync(request, null, args.Context!.Deadline, args.Context.CancellationToken); @@ -34,12 +34,12 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient { Body = new ListRequest.Types.Body { - OwnerId = Context.Owner.ToGrpcMessage() + OwnerId = Context.Owner.ToMessage() } }; request.AddMetaHeader(args.XHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await service.ListAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -54,21 +54,21 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient internal async Task CreateContainerAsync(PrmContainerCreate args) { var ctx = args.Context!; - var grpcContainer = args.Container.ToGrpcMessage(); - grpcContainer.OwnerId = Context.Owner.ToGrpcMessage(); - grpcContainer.Version = Context.Version.ToGrpcMessage(); + var grpcContainer = args.Container.ToMessage(); + grpcContainer.OwnerId = Context.Owner.ToMessage(); + grpcContainer.Version = Context.Version.ToMessage(); var request = new PutRequest { Body = new PutRequest.Types.Body { Container = grpcContainer, - Signature = Context.Key.SignRFC6979(grpcContainer) + Signature = Context.Key.ECDsaKey.SignRFC6979(grpcContainer) } }; request.AddMetaHeader(args.XHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await service.PutAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -86,14 +86,14 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient { Body = new DeleteRequest.Types.Body { - ContainerId = args.ContainerId.ToGrpcMessage(), - Signature = Context.Key.SignRFC6979(args.ContainerId.ToGrpcMessage().Value) + ContainerId = args.ContainerId.ToMessage(), + Signature = Context.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value) } }; request.AddMetaHeader(args.XHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await service.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -113,7 +113,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient }; request.AddMetaHeader(xHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); return request; } diff --git a/src/FrostFS.SDK.ClientV2/Services/NetmapServiceProvider.cs b/src/FrostFS.SDK.ClientV2/Services/NetmapServiceProvider.cs index f4d239a..e088ba4 100644 --- a/src/FrostFS.SDK.ClientV2/Services/NetmapServiceProvider.cs +++ b/src/FrostFS.SDK.ClientV2/Services/NetmapServiceProvider.cs @@ -49,7 +49,7 @@ internal class NetmapServiceProvider : ContextAccessor }; request.AddMetaHeader(args.XHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -63,7 +63,7 @@ internal class NetmapServiceProvider : ContextAccessor var request = new NetworkInfoRequest(); request.AddMetaHeader(null); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -79,7 +79,7 @@ internal class NetmapServiceProvider : ContextAccessor var request = new NetmapSnapshotRequest(); request.AddMetaHeader(args.XHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.Deadline, ctx.CancellationToken); diff --git a/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs index 1c8bf2f..6696aab 100644 --- a/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs @@ -13,13 +13,13 @@ using FrostFS.Session; using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ClientV2.Extensions; using FrostFS.SDK.ClientV2.Parameters; +using System.Buffers; namespace FrostFS.SDK.ClientV2; -internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment ctx) : ContextAccessor(ctx), ISessionProvider +internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment env) : ContextAccessor(env), ISessionProvider { - readonly ObjectTools tools = new(ctx); - readonly SessionProvider sessions = new (ctx); + readonly SessionProvider sessions = new (env); public async ValueTask GetOrCreateSession(ISessionToken args, Context ctx) { @@ -35,8 +35,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C { Address = new Address { - ContainerId = args.ContainerId.ToGrpcMessage(), - ObjectId = args.ObjectId.ToGrpcMessage() + ContainerId = args.ContainerId.ToMessage(), + ObjectId = args.ObjectId.ToMessage() } } }; @@ -46,11 +46,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C sessionToken.CreateObjectTokenContext( request.Body.Address, ObjectSessionContext.Types.Verb.Head, - Context.Key); + Context.Key.ECDsaKey); request.AddMetaHeader(args.XHeaders, sessionToken); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await client!.HeadAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -69,8 +69,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C { Address = new Address { - ContainerId = args.ContainerId.ToGrpcMessage(), - ObjectId = args.ObjectId.ToGrpcMessage() + ContainerId = args.ContainerId.ToMessage(), + ObjectId = args.ObjectId.ToMessage() } } }; @@ -80,11 +80,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C sessionToken.CreateObjectTokenContext( request.Body.Address, ObjectSessionContext.Types.Verb.Get, - Context.Key); + Context.Key.ECDsaKey); request.AddMetaHeader(args.XHeaders, sessionToken); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); return await GetObject(request, ctx); } @@ -98,8 +98,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C { Address = new Address { - ContainerId = args.ContainerId.ToGrpcMessage(), - ObjectId = args.ObjectId.ToGrpcMessage() + ContainerId = args.ContainerId.ToMessage(), + ObjectId = args.ObjectId.ToMessage() } } }; @@ -109,10 +109,10 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C sessionToken.CreateObjectTokenContext( request.Body.Address, ObjectSessionContext.Types.Verb.Delete, - Context.Key); + Context.Key.ECDsaKey); request.AddMetaHeader(args.XHeaders, sessionToken); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await client.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -126,24 +126,24 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C { Body = new SearchRequest.Types.Body { - ContainerId = args.ContainerId.ToGrpcMessage(), + ContainerId = args.ContainerId.ToMessage(), Filters = { }, Version = 1 // TODO: clarify this param } }; - request.Body.Filters.AddRange(args.Filters.Select(f => f.ToGrpcMessage())); + request.Body.Filters.AddRange(args.Filters.Select(f => f.ToMessage())); var sessionToken = await GetOrCreateSession(args, ctx); sessionToken.CreateObjectTokenContext( new Address { ContainerId = request.Body.ContainerId }, ObjectSessionContext.Types.Verb.Search, - Context.Key); + Context.Key.ECDsaKey); request.AddMetaHeader(args.XHeaders, sessionToken); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var objectsIds = SearchObjects(request, ctx); @@ -153,24 +153,31 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C } } - internal Task PutObjectAsync(PrmObjectPut args) + internal async Task PutObjectAsync(PrmObjectPut args) { if (args.Header == null) throw new ArgumentException("Value cannot be null", nameof(args.Header)); if (args.Payload == null) throw new ArgumentException("Value cannot be null", nameof(args.Payload)); - + if (args.ClientCut) - return PutClientCutObject(args); - else - return PutStreamObject(args); + return await PutClientCutObject(args); + else + { + if (args.Header.PayloadLength > 0) + args.FullLength = args.Header.PayloadLength; + else if (args.Payload.CanSeek) + args.FullLength = (ulong)args.Payload.Length; + + return (await PutStreamObject(args)).ObjectId; + } } internal async Task PutSingleObjectAsync(PrmSingleObjectPut args) { var ctx = args.Context!; - var grpcObject = tools.CreateObject(args.FrostFsObject); + var grpcObject = ObjectTools.CreateObject(args.FrostFsObject, env); var request = new PutSingleRequest { @@ -185,11 +192,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C sessionToken.CreateObjectTokenContext( new Address { ContainerId = grpcObject.Header.ContainerId, ObjectId = grpcObject.ObjectId}, ObjectSessionContext.Types.Verb.Put, - Context.Key); + Context.Key.ECDsaKey); request.AddMetaHeader(args.XHeaders, sessionToken); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); var response = await client.PutSingleAsync(request, null, ctx.Deadline, ctx.CancellationToken); @@ -201,156 +208,136 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C private async Task PutClientCutObject(PrmObjectPut args) { var ctx = args.Context!; + var tokenRaw = await GetOrCreateSession(args, ctx); var token = new ModelsV2.SessionToken(tokenRaw.Serialize()); + args.SessionToken = token; + var payloadStream = args.Payload!; var header = args.Header!; - ObjectId? objectId; - List sentObjectIds = []; - - FrostFsObject? currentObject; - - var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx)); - - var objectSize = (int)networkSettings.MaxObjectSize; - var fullLength = header.PayloadLength; - - if (payloadStream.CanSeek) - { - objectSize = (int)Math.Min(objectSize, payloadStream.Length); - - if (fullLength == 0) - fullLength = (ulong)payloadStream.Length; - } - if (fullLength == 0) - throw new ArgumentException("Payload stream must be able to seek or PayloadLength must be specified"); - - var buffer = new byte[objectSize]; + if (payloadStream.CanSeek && fullLength == 0) + fullLength = (ulong)payloadStream.Length; - var largeObject = new LargeObject(header.ContainerId); + args.FullLength = fullLength; - var split = new Split(); - - while (true) + if (args.MaxObjectSizeCache == 0) { - var bytesCount = await payloadStream.ReadAsync(buffer, 0, objectSize); - - split.Previous = sentObjectIds.LastOrDefault(); - - largeObject.Header.PayloadLength += (ulong)bytesCount; - - currentObject = new FrostFsObject(header.ContainerId) - .SetPayload(bytesCount < objectSize ? buffer[..bytesCount] : buffer) - .SetSplit(split); - - if (largeObject.PayloadLength == fullLength) - break; - - objectId = await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token }); - - sentObjectIds.Add(objectId!); + var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx)); + args.MaxObjectSizeCache = (int)networkSettings.MaxObjectSize; } - if (sentObjectIds.Count != 0) + var restBytes = fullLength - args.CurrentStreamPosition; + var objectSize = restBytes > 0 ? (int)Math.Min((ulong)args.MaxObjectSizeCache, restBytes) : args.MaxObjectSizeCache; + + //define collection capacity + var restPart = (restBytes % (ulong)objectSize) > 0 ? 1 : 0; + var objectsCount = fullLength > 0 ? (int)(restBytes / (ulong)objectSize) + restPart : 0; + + List sentObjectIds = new(objectsCount); + + Split? split = null; + + // keep attributes for the large object + var attributes = args.Header!.Attributes; + + // send all parts except the last one as separate Objects + while (restBytes > (ulong)args.MaxObjectSizeCache) { - largeObject.AddAttributes(args.Header!.Attributes); - - currentObject.SetParent(largeObject); + if (split == null) + { + split = new Split(); + args.Header!.Attributes = []; + } - var putSingleObjectParams = new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token }; + split!.Previous = sentObjectIds.LastOrDefault(); + args.Header!.Split = split; - objectId = await PutSingleObjectAsync(putSingleObjectParams); + var result = await PutStreamObject(args); - sentObjectIds.Add(objectId); + sentObjectIds.Add(result.ObjectId); - var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject) + restBytes -= (ulong)result.ObjectSize; + } + + // send the last part and create linkObject + if (sentObjectIds.Count > 0) + { + var largeObjectHeader = new ObjectHeader(header.ContainerId) { PayloadLength = fullLength }; + + largeObjectHeader.Attributes.AddRange(attributes); + + args.Header.Split!.ParentHeader = largeObjectHeader; + + var result = await PutStreamObject(args); + + sentObjectIds.Add(result.ObjectId); + + var linkObject = new LinkObject(header.ContainerId, split!.SplitId, largeObjectHeader) .AddChildren(sentObjectIds); - linkObject.Header.Attributes.Clear(); + _ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject) { Context = args.Context}); - _ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject, ctx){ SessionToken = token }); - - return tools.CalculateObjectId(largeObject.Header); + return split.Parent!; } - currentObject - .SetSplit(null) - .AddAttributes(args.Header!.Attributes); - - return await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx)); + // We are here if the payload is placed to one Object. It means no cut action, just simple PUT. + var singlePartResult = await PutStreamObject(args); + + return singlePartResult.ObjectId; } - private async Task PutStreamObject(PrmObjectPut args) + struct PutObjectResult(ObjectId objectId, int objectSize) + { + public ObjectId ObjectId = objectId; + public int ObjectSize = objectSize; + } + + private async Task PutStreamObject(PrmObjectPut args) { var ctx = args.Context!; var payload = args.Payload!; - var header = args.Header!; - var hdr = header.ToGrpcMessage(); - hdr.OwnerId = Context.Owner.ToGrpcMessage(); - hdr.Version = Context.Version.ToGrpcMessage(); + var chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize; - var oid = new ObjectID { Value = hdr.Sha256() }; + var restBytes = args.FullLength - args.CurrentStreamPosition; - var initRequest = new PutRequest + chunkSize = (int)Math.Min(restBytes, (ulong)chunkSize); + + var chunkBuffer = ArrayPool.Shared.Rent(chunkSize); + var sentBytes = 0; + + // 0 means no limit from client, so server side cut is performed + var objectLimitSize = args.ClientCut ? args.MaxObjectSizeCache : 0; + + var stream = await GetUploadStream(args, ctx); + + while (objectLimitSize == 0 || sentBytes < objectLimitSize) { - Body = new PutRequest.Types.Body - { - Init = new PutRequest.Types.Body.Types.Init - { - Header = hdr - } - } - }; + // send chanks limited to default or user's settings + var bufferSize = objectLimitSize > 0 ? + (int)Math.Min(objectLimitSize - sentBytes, chunkSize) + : chunkSize; - var sessionToken = await GetOrCreateSession(args, ctx); - - sessionToken.CreateObjectTokenContext( - new Address { ContainerId = hdr.ContainerId, ObjectId = oid }, - ObjectSessionContext.Types.Verb.Put, - Context.Key - ); - - initRequest.AddMetaHeader(args.XHeaders, sessionToken); - - initRequest.Sign(Context.Key); - - using var stream = await PutObjectInit(initRequest, ctx); - - var bufferSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize; - - if (payload.CanSeek) - { - bufferSize = (int)Math.Min(payload.Length, bufferSize); - } - else if (header.PayloadLength > 0) - { - bufferSize = (int)Math.Min((long)header.PayloadLength, bufferSize); - } - - var buffer = new byte[bufferSize]; - - while (true) - { - var bytesCount = await payload.ReadAsync(buffer, 0, bufferSize, ctx.CancellationToken); + var bytesCount = await payload.ReadAsync(chunkBuffer, 0, bufferSize, ctx.CancellationToken); if (bytesCount == 0) break; - var chunkRequest = new PutRequest(initRequest) + sentBytes += bytesCount; + + var chunkRequest = new PutRequest { Body = new PutRequest.Types.Body { - Chunk = ByteString.CopyFrom(buffer.AsSpan()[..bytesCount]), - }, - VerifyHeader = null + Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount) + } }; - chunkRequest.Sign(Context.Key); + chunkRequest.Sign(Context.Key.ECDsaKey); await stream.Write(chunkRequest); } @@ -358,7 +345,49 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C var response = await stream.Close(); Verifier.CheckResponse(response); - return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()); + return new PutObjectResult(ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()), sentBytes); + } + + private async Task GetUploadStream(PrmObjectPut args, Context ctx) + { + var header = args.Header!; + + header.OwnerId = Context.Owner; + header.Version = Context.Version; + + var grpcHeader = header.ToMessage(); + + if (header.Split != null) + { + ObjectTools.SetSplitValues(grpcHeader, header.Split, env); + } + + var oid = new ObjectID { Value = grpcHeader.Sha256() }; + + var initRequest = new PutRequest + { + Body = new PutRequest.Types.Body + { + Init = new PutRequest.Types.Body.Types.Init + { + Header = grpcHeader + } + } + }; + + var sessionToken = await GetOrCreateSession(args, ctx); + + sessionToken.CreateObjectTokenContext( + new Address { ContainerId = grpcHeader.ContainerId, ObjectId = oid }, + ObjectSessionContext.Types.Verb.Put, + Context.Key.ECDsaKey + ); + + initRequest.AddMetaHeader(args.XHeaders, sessionToken); + + initRequest.Sign(Context.Key.ECDsaKey); + + return await PutObjectInit(initRequest, ctx); } private async Task GetObject(GetRequest request, Context ctx) diff --git a/src/FrostFS.SDK.ClientV2/Services/SessionServiceProvider.cs b/src/FrostFS.SDK.ClientV2/Services/SessionServiceProvider.cs index 61ea93f..0bb5499 100644 --- a/src/FrostFS.SDK.ClientV2/Services/SessionServiceProvider.cs +++ b/src/FrostFS.SDK.ClientV2/Services/SessionServiceProvider.cs @@ -21,13 +21,13 @@ internal class SessionServiceProvider : ContextAccessor { Body = new CreateRequest.Types.Body { - OwnerId = Context.Owner.ToGrpcMessage(), + OwnerId = Context.Owner.ToMessage(), Expiration = args.Expiration } }; request.AddMetaHeader(args.XHeaders); - request.Sign(Context.Key); + request.Sign(Context.Key.ECDsaKey); return await CreateSession(request, args.Context!); } diff --git a/src/FrostFS.SDK.ClientV2/Tools/ClientEnvironment.cs b/src/FrostFS.SDK.ClientV2/Tools/ClientEnvironment.cs index dae8ff2..c1f4d84 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/ClientEnvironment.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/ClientEnvironment.cs @@ -1,7 +1,9 @@ using FrostFS.SDK.ModelsV2; +using Google.Protobuf; using Grpc.Net.Client; using System; using System.Security.Cryptography; +using FrostFS.SDK.Cryptography; namespace FrostFS.SDK.ClientV2; @@ -9,11 +11,12 @@ public class ClientEnvironment(Client client, ECDsa key, OwnerId owner, GrpcChan { internal OwnerId Owner { get; } = owner; internal GrpcChannel Channel { get; private set; } = channel; - internal ECDsa Key { get; } = key; internal ModelsV2.Version Version { get; } = version; internal NetworkSettings? NetworkSettings { get; set; } - internal Client Client { get; set; } = client; + internal Client Client { get; } = client; + + internal ClientKey Key { get; } = new ClientKey(key); public void Dispose() { diff --git a/src/FrostFS.SDK.ClientV2/Tools/ObjectTools.cs b/src/FrostFS.SDK.ClientV2/Tools/ObjectTools.cs index f8d2c33..aad073b 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/ObjectTools.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/ObjectTools.cs @@ -7,54 +7,36 @@ using FrostFS.Refs; using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; +using static FrostFS.Object.Header.Types; namespace FrostFS.SDK.ClientV2; -internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx) +internal class ObjectTools { - internal ObjectId CalculateObjectId(ObjectHeader header) + internal static ObjectId CalculateObjectId(ObjectHeader header, ClientEnvironment env) { - var grpcHeader = CreateHeader(header, []); + var grpcHeader = CreateHeader(header, [], env); + + if (header.Split != null) + SetSplitValues(grpcHeader, header.Split, env); return new ObjectID { Value = grpcHeader.Sha256() }.ToModel(); } - internal Object.Object CreateObject(FrostFsObject @object) + internal static Object.Object CreateObject(FrostFsObject @object, ClientEnvironment env) { - var grpcHeader = @object.Header.ToGrpcMessage(); - - grpcHeader.OwnerId = Context.Owner.ToGrpcMessage(); - grpcHeader.Version = Context.Version.ToGrpcMessage(); + @object.Header.OwnerId = env.Owner; + @object.Header.Version = env.Version; + + var grpcHeader = @object.Header.ToMessage(); + grpcHeader.PayloadLength = (ulong)@object.Payload.Length; grpcHeader.PayloadHash = Sha256Checksum(@object.Payload); - + var split = @object.Header.Split; if (split != null) { - grpcHeader.Split = new Header.Types.Split - { - SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null - }; - - if (split.Children != null && split.Children.Count != 0) - grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage())); - - if (split.ParentHeader is not null) - { - var grpcParentHeader = CreateHeader(split.ParentHeader, []); - - grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() }; - grpcHeader.Split.ParentHeader = grpcParentHeader; - grpcHeader.Split.ParentSignature = new Refs.Signature - { - Key = ByteString.CopyFrom(Context.Key.PublicKey()), - Sign = ByteString.CopyFrom(Context.Key.SignData(grpcHeader.Split.Parent.ToByteArray())), - }; - - split.Parent = grpcHeader.Split.Parent.ToModel(); - } - - grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage(); + SetSplitValues(grpcHeader, split, env); } var obj = new Object.Object @@ -66,21 +48,49 @@ internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx) obj.Signature = new Refs.Signature { - Key = ByteString.CopyFrom(Context.Key.PublicKey()), - Sign = ByteString.CopyFrom(Context.Key.SignData(obj.ObjectId.ToByteArray())), + Key = env.Key.PublicKeyProto, + Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(obj.ObjectId.ToByteArray())), }; return obj; } - internal Header CreateHeader(ObjectHeader header, byte[]? payload) + internal static void SetSplitValues(Header grpcHeader, ModelsV2.Split split, ClientEnvironment env) { - var grpcHeader = header.ToGrpcMessage(); - - grpcHeader.OwnerId = Context.Owner.ToGrpcMessage(); - grpcHeader.Version = Context.Version.ToGrpcMessage(); + grpcHeader.Split = new Header.Types.Split + { + SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null + }; - if (payload != null) + if (split.Children != null && split.Children.Count != 0) + grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage())); + + if (split.ParentHeader is not null) + { + var grpcParentHeader = CreateHeader(split.ParentHeader, [], env); + + grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() }; + grpcHeader.Split.ParentHeader = grpcParentHeader; + grpcHeader.Split.ParentSignature = new Refs.Signature + { + Key = env.Key.PublicKeyProto, + Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(grpcHeader.Split.Parent.ToByteArray())), + }; + + split.Parent = grpcHeader.Split.Parent.ToModel(); + } + + grpcHeader.Split.Previous = split.Previous?.ToMessage(); + } + + internal static Header CreateHeader(ObjectHeader header, byte[]? payload, ClientEnvironment env) + { + var grpcHeader = header.ToMessage(); + + grpcHeader.OwnerId = env.Owner.ToMessage(); + grpcHeader.Version = env.Version.ToMessage(); + + if (payload != null) // && payload.Length > 0 grpcHeader.PayloadHash = Sha256Checksum(payload); return grpcHeader; diff --git a/src/FrostFS.SDK.ClientV2/Tools/RequestConstructor.cs b/src/FrostFS.SDK.ClientV2/Tools/RequestConstructor.cs index 4e1de51..acced67 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/RequestConstructor.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/RequestConstructor.cs @@ -17,7 +17,7 @@ public static class RequestConstructor if (request.MetaHeader is not null) return; - request.MetaHeader = MetaHeader.Default().ToGrpcMessage(); + request.MetaHeader = MetaHeader.Default().ToMessage(); if (sessionToken != null) request.MetaHeader.SessionToken = sessionToken; diff --git a/src/FrostFS.SDK.ClientV2/Tools/RequestSigner.cs b/src/FrostFS.SDK.ClientV2/Tools/RequestSigner.cs index d9fa633..f5057eb 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/RequestSigner.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/RequestSigner.cs @@ -69,7 +69,7 @@ public static class RequestSigner var hash = new byte[65]; hash[0] = 0x04; - key.SignHash(SHA512.Create().ComputeHash(data)).CopyTo(hash, 1); + key.SignHash(data.Sha512()).CopyTo(hash, 1); return hash; } diff --git a/src/FrostFS.SDK.ClientV2/Tools/Verifier.cs b/src/FrostFS.SDK.ClientV2/Tools/Verifier.cs index 2b36a64..bd8ddec 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/Verifier.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/Verifier.cs @@ -60,7 +60,7 @@ public static class Verifier public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig) { - return key.VerifyHash(SHA512.Create().ComputeHash(data), sig[1..]); + return key.VerifyHash(data.Sha512(), sig[1..]); } public static bool VerifyMessagePart(this Signature sig, IMessage data) @@ -101,7 +101,8 @@ public static class Verifier throw new FormatException($"invalid response, type={resp.GetType()}"); var status = resp.MetaHeader.Status.ToModel(); - if (!status.IsSuccess) + + if (status != null && !status.IsSuccess) throw new ResponseException(status); } diff --git a/src/FrostFS.SDK.Cryptography/Base58.cs b/src/FrostFS.SDK.Cryptography/Base58.cs index f33794a..95b07f0 100644 --- a/src/FrostFS.SDK.Cryptography/Base58.cs +++ b/src/FrostFS.SDK.Cryptography/Base58.cs @@ -51,13 +51,13 @@ public static class Base58 int digit = Alphabet.IndexOf(input[i]); if (digit < 0) throw new FormatException($"Invalid Base58 character '{input[i]}' at position {i}"); - + bi = bi * Alphabet.Length + digit; } int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count(); var leadingZeros = new byte[leadingZeroCount]; - + if (bi.IsZero) return leadingZeros; @@ -65,9 +65,9 @@ public static class Base58 var firstNonZeroIndex = 0; - while(bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0) + while (bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0) firstNonZeroIndex++; - + var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray(); return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros); diff --git a/src/FrostFS.SDK.Cryptography/Extentions.cs b/src/FrostFS.SDK.Cryptography/Extentions.cs index 534fb7d..32c8259 100644 --- a/src/FrostFS.SDK.Cryptography/Extentions.cs +++ b/src/FrostFS.SDK.Cryptography/Extentions.cs @@ -1,11 +1,18 @@ using Google.Protobuf; using Org.BouncyCastle.Crypto.Digests; using System.Security.Cryptography; +using System.Threading; namespace FrostFS.SDK.Cryptography; public static class Extentions { + private static readonly SHA256 _sha256 = SHA256.Create(); + private static SpinLock _spinlockSha256 = new(); + + private static readonly SHA512 _sha512 = SHA512.Create(); + private static SpinLock _spinlockSha512 = new(); + internal static byte[] RIPEMD160(this byte[] value) { var hash = new byte[20]; @@ -15,15 +22,41 @@ public static class Extentions digest.DoFinal(hash, 0); return hash; } - - public static byte[] Sha256(this byte[] value) - { - var sha256 = SHA256.Create(); - return sha256.ComputeHash(value); - } public static ByteString Sha256(this IMessage data) { return ByteString.CopyFrom(data.ToByteArray().Sha256()); } + + public static byte[] Sha256(this byte[] value) + { + bool lockTaken = false; + try + { + _spinlockSha256.Enter(ref lockTaken); + + return _sha256.ComputeHash(value); + } + finally + { + if (lockTaken) + _spinlockSha256.Exit(false); + } + } + + public static byte[] Sha512(this byte[] value) + { + bool lockTaken = false; + try + { + _spinlockSha512.Enter(ref lockTaken); + + return _sha512.ComputeHash(value); + } + finally + { + if (lockTaken) + _spinlockSha512.Exit(false); + } + } } diff --git a/src/FrostFS.SDK.Cryptography/Key.cs b/src/FrostFS.SDK.Cryptography/Key.cs index de96791..bbffab8 100644 --- a/src/FrostFS.SDK.Cryptography/Key.cs +++ b/src/FrostFS.SDK.Cryptography/Key.cs @@ -133,8 +133,9 @@ public static class KeyExtension public static ECDsa LoadPrivateKey(this byte[] privateKey) { var secp256R1 = SecNamedCurves.GetByName("secp256r1"); - var publicKey = - secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)).GetEncoded(false)[1..]; + var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)) + .GetEncoded(false)[1..]; + var key = ECDsa.Create(new ECParameters { Curve = ECCurve.NamedCurves.nistP256, diff --git a/src/FrostFS.SDK.Cryptography/Murmur3_128.cs b/src/FrostFS.SDK.Cryptography/Murmur3_128.cs index 1a846ab..5cb35a1 100644 --- a/src/FrostFS.SDK.Cryptography/Murmur3_128.cs +++ b/src/FrostFS.SDK.Cryptography/Murmur3_128.cs @@ -112,7 +112,7 @@ internal class Murmur3_128 : HashAlgorithm h2 = seed; } - private ulong Fimix64(ulong k) + private static ulong Fimix64(ulong k) { k ^= k >> 33; k *= 0xff51afd7ed558ccd; diff --git a/src/FrostFS.SDK.Cryptography/Range.cs b/src/FrostFS.SDK.Cryptography/Range.cs index 7313a64..92aa82f 100644 --- a/src/FrostFS.SDK.Cryptography/Range.cs +++ b/src/FrostFS.SDK.Cryptography/Range.cs @@ -77,14 +77,7 @@ namespace System { get { - if (_value < 0) - { - return ~_value; - } - else - { - return _value; - } + return _value < 0 ? ~_value : _value; } } @@ -105,12 +98,9 @@ namespace System var offset = _value; if (IsFromEnd) { - // offset = length - (~value) - // offset = length + (~(~value) + 1) - // offset = length + value + 1 - offset += length + 1; } + return offset; } diff --git a/src/FrostFS.SDK.ModelsV2/FrostFS.SDK.ModelsV2.csproj b/src/FrostFS.SDK.ModelsV2/FrostFS.SDK.ModelsV2.csproj index be180b0..1b4ad74 100644 --- a/src/FrostFS.SDK.ModelsV2/FrostFS.SDK.ModelsV2.csproj +++ b/src/FrostFS.SDK.ModelsV2/FrostFS.SDK.ModelsV2.csproj @@ -16,6 +16,7 @@ + diff --git a/src/FrostFS.SDK.ModelsV2/Misc/CheckSum.cs b/src/FrostFS.SDK.ModelsV2/Misc/CheckSum.cs index 6b5526a..0b7f3bd 100644 --- a/src/FrostFS.SDK.ModelsV2/Misc/CheckSum.cs +++ b/src/FrostFS.SDK.ModelsV2/Misc/CheckSum.cs @@ -1,3 +1,4 @@ +using FrostFS.SDK.Cryptography; using System; using System.Security.Cryptography; @@ -8,15 +9,9 @@ public class CheckSum // type is always Sha256 public byte[]? Hash { get; set; } - public static byte[] GetHash(byte[] content) - { - var sha256 = SHA256.Create(); - return sha256.ComputeHash(content); - } - public static CheckSum CreateCheckSum(byte[] content) { - return new CheckSum { Hash = GetHash(content) }; + return new CheckSum { Hash = content.Sha256() }; } public override string ToString() diff --git a/src/FrostFS.SDK.ModelsV2/Object/FrostFsObject.cs b/src/FrostFS.SDK.ModelsV2/Object/FrostFsObject.cs index 6ca2c22..971d41d 100644 --- a/src/FrostFS.SDK.ModelsV2/Object/FrostFsObject.cs +++ b/src/FrostFS.SDK.ModelsV2/Object/FrostFsObject.cs @@ -56,19 +56,17 @@ public class FrostFsObject /// Applied only for the last Object in chain in case of manual multipart uploading /// /// Parent for multipart object - public void SetParent(LargeObject largeObject) + public void SetParent(ObjectHeader largeObjectHeader) { if (Header?.Split == null) throw new Exception("The object is not initialized properly"); - Header.Split.ParentHeader = largeObject.Header; + Header.Split.ParentHeader = largeObjectHeader; } } public class LargeObject(ContainerId container) : FrostFsObject(container) { - private readonly SHA256 payloadHash = SHA256.Create(); - public ulong PayloadLength { get { return Header!.PayloadLength; } @@ -77,11 +75,11 @@ public class LargeObject(ContainerId container) : FrostFsObject(container) public class LinkObject : FrostFsObject { - public LinkObject(ContainerId containerId, SplitId splitId, LargeObject largeObject) : base (containerId) + public LinkObject(ContainerId containerId, SplitId splitId, ObjectHeader largeObjectHeader) : base (containerId) { Header!.Split = new Split(splitId) { - ParentHeader = largeObject.Header + ParentHeader = largeObjectHeader }; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ModelsV2/Object/Splitter.cs b/src/FrostFS.SDK.ModelsV2/Object/Splitter.cs index d4622df..3fb55df 100644 --- a/src/FrostFS.SDK.ModelsV2/Object/Splitter.cs +++ b/src/FrostFS.SDK.ModelsV2/Object/Splitter.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; + namespace FrostFS.SDK.ModelsV2; public class Split(SplitId splitId) @@ -19,4 +20,6 @@ public class Split(SplitId splitId) public ObjectHeader? ParentHeader { get; set; } public List Children { get; } = []; + + public Refs.Signature ParentSignatureGrpc { get; set; } } diff --git a/src/FrostFS.SDK.ModelsV2/Response/Status.cs b/src/FrostFS.SDK.ModelsV2/Response/ResponseStatus.cs similarity index 83% rename from src/FrostFS.SDK.ModelsV2/Response/Status.cs rename to src/FrostFS.SDK.ModelsV2/Response/ResponseStatus.cs index d52d189..f58e1bb 100644 --- a/src/FrostFS.SDK.ModelsV2/Response/Status.cs +++ b/src/FrostFS.SDK.ModelsV2/Response/ResponseStatus.cs @@ -2,7 +2,7 @@ using FrostFS.SDK.ModelsV2.Enums; namespace FrostFS.SDK.ModelsV2; -public class Status(StatusCode code, string? message = null) +public class ResponseStatus(StatusCode code, string? message = null) { public StatusCode Code { get; set; } = code; public string Message { get; set; } = message ?? string.Empty; diff --git a/src/FrostFS.SDK.Tests/ContainerTest.cs b/src/FrostFS.SDK.Tests/ContainerTest.cs index 8fbdc2a..74f80e3 100644 --- a/src/FrostFS.SDK.Tests/ContainerTest.cs +++ b/src/FrostFS.SDK.Tests/ContainerTest.cs @@ -110,6 +110,6 @@ public class ContainerTest : ContainerTestsBase var request = Mocker.Requests.First(); - Assert.Equal(cid.ToGrpcMessage(), request.Request.Body.ContainerId); + Assert.Equal(cid.ToMessage(), request.Request.Body.ContainerId); } } diff --git a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs index 45b17bc..3603a92 100644 --- a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs @@ -19,14 +19,14 @@ public class AsyncStreamReaderMock(string key, ObjectHeader objectHeader) : Serv { var header = new Header { - ContainerId = objectHeader.ContainerId.ToGrpcMessage(), + ContainerId = objectHeader.ContainerId.ToMessage(), PayloadLength = objectHeader.PayloadLength, - Version = objectHeader.Version!.ToGrpcMessage(), - OwnerId = objectHeader.OwnerId!.ToGrpcMessage() + Version = objectHeader.Version!.ToMessage(), + OwnerId = objectHeader.OwnerId!.ToMessage() }; foreach (var attr in objectHeader.Attributes) - header.Attributes.Add(attr.ToGrpcMessage()); + header.Attributes.Add(attr.ToMessage()); var response = new GetResponse { diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs index 530294a..9de903c 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs @@ -27,7 +27,7 @@ public abstract class ServiceBase(string key) public static BasicAcl DefaultAcl { get; } = BasicAcl.PublicRW; public static PlacementPolicy DefaultPlacementPolicy { get; } = new PlacementPolicy(true, new Replica(1)); - public Metadata ResponseMetaData => []; + public static Metadata ResponseMetaData => []; protected ResponseVerificationHeader GetResponseVerificationHeader(IResponse response) { @@ -58,7 +58,7 @@ public abstract class ServiceBase(string key) public ResponseMetaHeader ResponseMetaHeader => new() { - Version = Version.ToGrpcMessage(), + Version = Version.ToMessage(), Epoch = 100, Ttl = 1 }; diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs index aff566d..34f6173 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs @@ -19,7 +19,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) { var mock = new Mock(); - var grpcVersion = Version.ToGrpcMessage(); + var grpcVersion = Version.ToMessage(); PutResponse putResponse = new() { @@ -32,7 +32,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) }, MetaHeader = new ResponseMetaHeader { - Version = Version is null ? DefaultVersion.ToGrpcMessage() : Version.ToGrpcMessage(), + Version = (Version is null ? DefaultVersion : Version).ToMessage(), Epoch = 100, Ttl = 1 } @@ -69,7 +69,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) Version = grpcVersion, Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()), BasicAcl = (uint)Acl, - PlacementPolicy = PlacementPolicy.ToGrpcMessage() + PlacementPolicy = PlacementPolicy.ToMessage() } }, MetaHeader = ResponseMetaHeader diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index db4df1e..099398d 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -38,12 +38,12 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) HeadResponse ??= new Header { CreationEpoch = 99, - ContainerId = ObjectHeader.ContainerId.ToGrpcMessage(), + ContainerId = ObjectHeader.ContainerId.ToMessage(), ObjectType = ObjectType.Regular, - OwnerId = ObjectHeader.OwnerId!.ToGrpcMessage(), + OwnerId = ObjectHeader.OwnerId!.ToMessage(), PayloadLength = 1, PayloadHash = new Refs.Checksum { Type = Refs.ChecksumType.Sha256, Sum = ByteString.CopyFrom(SHA256.HashData([0xff])) }, - Version = ObjectHeader.Version!.ToGrpcMessage() + Version = ObjectHeader.Version!.ToMessage() }; HeadResponse headResponse = new() @@ -89,7 +89,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) } - if (ResultObjectId != null) + if (ResultObjectIds != null) { PutResponse putResponse = new() { @@ -97,7 +97,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) { ObjectId = new Refs.ObjectID { - Value = ByteString.CopyFrom(ResultObjectId) + Value = ByteString.CopyFrom(ResultObjectIds.ElementAt(0)) } }, MetaHeader = ResponseMetaHeader, @@ -156,8 +156,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) { Tombstone = new Refs.Address { - ContainerId = ObjectHeader!.ContainerId.ToGrpcMessage(), - ObjectId = ObjectId.ToGrpcMessage() + ContainerId = ObjectHeader!.ContainerId.ToMessage(), + ObjectId = ObjectId.ToMessage() } }, MetaHeader = ResponseMetaHeader @@ -195,7 +195,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) public Header? HeadResponse { get; set; } - public byte[]? ResultObjectId { get; set; } + public List? ResultObjectIds { get; set; } public ClientStreamWriter? ClientStreamWriter { get; private set; } = new (); diff --git a/src/FrostFS.SDK.Tests/ObjectTest.cs b/src/FrostFS.SDK.Tests/ObjectTest.cs index 05573ff..6fb3f0a 100644 --- a/src/FrostFS.SDK.Tests/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/ObjectTest.cs @@ -7,6 +7,7 @@ using FrostFS.SDK.Cryptography; using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2.Enums; using FrostFS.SDK.ModelsV2.Netmap; +using FrostFS.SDK.ProtosV2.Interfaces; using Google.Protobuf; using Microsoft.Extensions.Options; using System.Security.Cryptography; @@ -82,7 +83,7 @@ public class ObjectTest : ObjectTestsBase Assert.NotNull(result); Assert.Equal(Mocker.ObjectHeader!.ContainerId.Value, result.Header.ContainerId.Value); - Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), result.Header.OwnerId!.Value); + Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, result.Header.OwnerId!.Value); Assert.Equal(Mocker.ObjectHeader.PayloadLength, result.Header.PayloadLength); Assert.Single(result.Header.Attributes); Assert.Equal(Mocker.ObjectHeader.Attributes[0].Key, result.Header.Attributes[0].Key); @@ -92,7 +93,7 @@ public class ObjectTest : ObjectTestsBase [Fact] public async void PutObjectTest() { - Mocker.ResultObjectId = SHA256.HashData([]); + Mocker.ResultObjectIds = new([SHA256.HashData([])]); Random rnd = new(); var bytes = new byte[1024]; @@ -113,7 +114,7 @@ public class ObjectTest : ObjectTestsBase var body2 = sentMessages.ElementAt(1).GetBody() as Object.PutRequest.Types.Body; Assert.NotNull(result); - Assert.Equal(Mocker.ResultObjectId, result.ToHash()); + Assert.Equal(Mocker.ResultObjectIds.First(), result.ToHash()); Assert.True(Mocker.ClientStreamWriter.CompletedTask); @@ -137,57 +138,85 @@ public class ObjectTest : ObjectTestsBase { Header = Mocker.ObjectHeader, Payload = new MemoryStream(bytes), + BufferMaxSize = 1024, ClientCut = true }; + Random rnd = new(); + + List objIds = new([new byte[32], new byte[32], new byte[32]]); + rnd.NextBytes(objIds.ElementAt(0)); + rnd.NextBytes(objIds.ElementAt(1)); + rnd.NextBytes(objIds.ElementAt(2)); + + Mocker.ResultObjectIds = objIds; + var result = await GetClient().PutObjectAsync(param); - var sentMessages = Mocker.PutSingleRequests.ToArray(); + var singleObjects = Mocker.PutSingleRequests.ToArray(); - Assert.Equal(4, sentMessages.Length); + var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray(); - var object_0 = sentMessages[0].Body.Object; - var object_1 = sentMessages[1].Body.Object; - var object_2 = sentMessages[2].Body.Object; - var object_3 = sentMessages[3].Body.Object; + Assert.Single(singleObjects); - Assert.NotNull(object_0.Header.Split.SplitId); - Assert.Null(object_0.Header.Split.Previous); - Assert.Equal(blockSize, (int)object_0.Header.PayloadLength); - Assert.Equal(bytes[..blockSize], object_0.Payload); - Assert.True(object_0.Header.Attributes.Count == 0); + Assert.Equal(11, streamObjects.Length); - Assert.Equal(object_0.Header.Split.SplitId, object_1.Header.Split.SplitId); - Assert.Equal(object_0.ObjectId, object_1.Header.Split.Previous); - Assert.Equal(blockSize, (int)object_1.Header.PayloadLength); - Assert.Equal(bytes[blockSize..(blockSize * 2)], object_1.Payload); - Assert.True(object_1.Header.Attributes.Count == 0); + var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray(); + + Assert.Equal(3, bodies.Count(b => b.Init != null)); + Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024)); + + Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length); + + var linkObject = singleObjects[0].Body.Object; + var header1 = bodies[0].Init.Header; + var header2 = bodies[4].Init.Header; + var header3 = bodies[8].Init.Header; + + var payload1 = bodies[1].Chunk + .Concat(bodies[2].Chunk) + .Concat(bodies[3].Chunk) + .ToArray(); + + var payload2 = bodies[5].Chunk + .Concat(bodies[6].Chunk) + .Concat(bodies[7].Chunk) + .ToArray(); + + var payload3 = bodies[9].Chunk + .Concat(bodies[10].Chunk) + .ToArray(); + + Assert.NotNull(header1.Split.SplitId); + Assert.Null(header1.Split.Previous); + Assert.Equal(bytes[..blockSize], payload1); + Assert.True(header1.Attributes.Count == 0); + + Assert.Equal(header1.Split.SplitId, header2.Split.SplitId); + Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value); + Assert.Equal(bytes[blockSize..(blockSize * 2)], payload2); + Assert.True(header2.Attributes.Count == 0); // last part - Assert.NotNull(object_2.Header.Split.Parent); - Assert.NotNull(object_2.Header.Split.ParentHeader); - Assert.NotNull(object_2.Header.Split.ParentSignature); - Assert.Equal(object_1.Header.Split.SplitId, object_2.Header.Split.SplitId); - Assert.Equal(object_1.ObjectId, object_2.Header.Split.Previous); - Assert.Equal(fileLength%blockSize, (int)object_2.Header.PayloadLength); - Assert.Equal(bytes[((fileLength/blockSize) * blockSize)..fileLength], object_2.Payload); - Assert.True(object_2.Header.Attributes.Count == 0); + Assert.NotNull(header3.Split.Parent); + Assert.NotNull(header3.Split.ParentHeader); + Assert.NotNull(header3.Split.ParentSignature); + Assert.Equal(header2.Split.SplitId, header3.Split.SplitId); + Assert.Equal(bytes[((fileLength / blockSize) * blockSize)..fileLength], payload3); + Assert.True(header3.Attributes.Count == 0); - // link object - Assert.Equal(object_2.Header.Split.Parent, object_3.Header.Split.Parent); - Assert.Equal(object_2.Header.Split.ParentHeader, object_3.Header.Split.ParentHeader); - Assert.Equal(object_2.Header.Split.SplitId, object_3.Header.Split.SplitId); - Assert.Equal(0, (int)object_3.Header.PayloadLength); - Assert.Contains(object_0.ObjectId, object_3.Header.Split.Children); - Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children); - Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children); - Assert.True(object_2.Header.Attributes.Count == 0); + //link object + Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent); + Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader); + Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId); + Assert.Equal(0, (int)linkObject.Header.PayloadLength); + Assert.True(header3.Attributes.Count == 0); - Assert.Single(object_3.Header.Split.ParentHeader.Attributes); - Assert.Equal("k", object_3.Header.Split.ParentHeader.Attributes[0].Key); - Assert.Equal("v", object_3.Header.Split.ParentHeader.Attributes[0].Value); + Assert.Single(linkObject.Header.Split.ParentHeader.Attributes); + Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key); + Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value); - var modelObjId = ObjectId.FromHash(object_3.Header.Split.Parent.Value.ToByteArray()); + var modelObjId = ObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray()); Assert.Equal(result.Value, modelObjId.ToString()); } @@ -201,8 +230,8 @@ public class ObjectTest : ObjectTestsBase var request = Mocker.DeleteRequests.FirstOrDefault(); Assert.NotNull(request); - Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId); - Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId); + Assert.Equal(ContainerId.ToMessage().Value, request.Body.Address.ContainerId.Value); + Assert.Equal(Mocker.ObjectId.ToMessage().Value, request.Body.Address.ObjectId.Value); } [Fact] @@ -214,13 +243,13 @@ public class ObjectTest : ObjectTestsBase var request = Mocker.HeadRequests.FirstOrDefault(); Assert.NotNull(request); - Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId); - Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId); + Assert.Equal(ContainerId.ToMessage(), request.Body.Address.ContainerId); + Assert.Equal(Mocker.ObjectId.ToMessage(), request.Body.Address.ObjectId); Assert.NotNull(response); Assert.Equal(ContainerId.Value, response.ContainerId.Value); - Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), response.OwnerId!.Value); + Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, response.OwnerId!.Value); Assert.Equal(Mocker.ObjectHeader!.Version!.ToString(), response.Version!.ToString()); Assert.Equal(Mocker.HeadResponse!.PayloadLength, response.PayloadLength); diff --git a/src/FrostFS.SDK.Tests/SmokeTests.cs b/src/FrostFS.SDK.Tests/SmokeTests.cs index f827677..8b89531 100644 --- a/src/FrostFS.SDK.Tests/SmokeTests.cs +++ b/src/FrostFS.SDK.Tests/SmokeTests.cs @@ -72,13 +72,14 @@ public class SmokeTests [Fact] public async void GetSessionTest() { + var ecdsaKey = this.key.LoadWif(); + using var client = Client.GetInstance(GetOptions(this.key, this.url)); var token = await client.CreateSessionAsync(new PrmSessionCreate(100)); var session = new Session.SessionToken().Deserialize(token.Token); - - var ecdsaKey = this.key.LoadWif(); + var owner = OwnerId.FromKey(ecdsaKey); var ownerHash = Base58.Decode(owner.Value); @@ -217,9 +218,10 @@ public class SmokeTests await foreach (var objId in client.SearchObjectsAsync(searchParam)) { resultObjectsCount++; + var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId)); } - Assert.True(1 == resultObjectsCount, $"Filter for {filter.Key} doesn't work"); + Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); } [Theory] @@ -403,14 +405,14 @@ public class SmokeTests [InlineData(2 * 64 * 1024 * 1024 + 256)] [InlineData(200)] public async void ClientCutScenarioTest(int objectSize) - { + { using var client = Client.GetInstance(GetOptions(this.key, this.url)); await Cleanup(client); var createContainerParam = new PrmContainerCreate(new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1)))) { - WaitParams = lightWait + WaitParams = lightWait }; var containerId = await client.CreateContainerAsync(createContainerParam); @@ -449,8 +451,8 @@ public class SmokeTests var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId)); Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); + Assert.Equal("fileName", objHeader.Attributes[0].Key); + Assert.Equal("test", objHeader.Attributes[0].Value); } Assert.True(hasObject); @@ -542,7 +544,7 @@ public class MetricsInterceptor() : Interceptor watch.Stop(); - // Do somethins with call info + // Do something with call info // var elapsed = watch.ElapsedTicks * 1_000_000/Stopwatch.Frequency; return response; -- 2.45.2 From 608383458271d4bbd89c9ed887f774cffa90833b Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Thu, 1 Aug 2024 18:01:58 +0300 Subject: [PATCH 2/2] [#20] Client: Fix typo Signed-off-by: Pavel Gross --- src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs index ac9634d..65199ca 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs @@ -15,14 +15,14 @@ public static class ContainerIdMapper public static ContainerID ToMessage(this ContainerId model) { - if (!Cache.Owners.TryGetValue(model, out ContainerID? message)) + if (!Cache.Containers.TryGetValue(model, out ContainerID? message)) { message = new ContainerID { Value = ByteString.CopyFrom(Base58.Decode(model.Value)) }; - Cache.Owners.Set(model, message, _oneHourExpiration); + Cache.Containers.Set(model, message, _oneHourExpiration); } return message!; -- 2.45.2