From 87fe8db674b1100711e4b9fb906656b8183d3ed5 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Wed, 26 Mar 2025 16:51:42 +0300 Subject: [PATCH] [#43] Client: Memory optimization Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/ApeRules/Actions.cs | 2 +- src/FrostFS.SDK.Client/ApeRules/Condition.cs | 2 +- src/FrostFS.SDK.Client/ApeRules/Resources.cs | 2 +- src/FrostFS.SDK.Client/AssemblyInfo.cs | 4 +- src/FrostFS.SDK.Client/Caches.cs | 9 - .../Exceptions/FrostFsResponseException.cs | 4 +- .../Extensions/FrostFsExtensions.cs | 72 +- .../FrostFS.SDK.Client.csproj | 2 +- src/FrostFS.SDK.Client/FrostFSClient.cs | 35 +- .../Interfaces/IFrostFSClient.cs | 2 - src/FrostFS.SDK.Client/Mappers/ContainerId.cs | 16 +- .../Mappers/Object/ObjectHeaderMapper.cs | 33 +- .../Mappers/Object/ObjectId.cs | 2 +- src/FrostFS.SDK.Client/Mappers/OwnerId.cs | 2 +- .../Mappers/Session/SessionMapper.cs | 28 - .../Mappers/SignatureMapper.cs | 4 +- .../Models/Containers/FrostFsContainerInfo.cs | 5 +- .../Models/Misc/CheckSum.cs | 2 +- .../Models/Netmap/Placement/Context.cs | 2 +- .../Models/Object/FrostFsObject.cs | 23 - .../Models/Object/FrostFsSplit.cs | 26 + .../Models/Object/SplitId.cs | 13 +- .../Models/Response/FrostFsResponseStatus.cs | 4 +- .../Parameters/PrmApeChainRemove.cs | 2 +- .../Pool/ClientStatusMonitor.cs | 163 ----- src/FrostFS.SDK.Client/Pool/ClientWrapper.cs | 135 ---- src/FrostFS.SDK.Client/Pool/HealthyStatus.cs | 18 - src/FrostFS.SDK.Client/Pool/IClientStatus.cs | 28 - src/FrostFS.SDK.Client/Pool/InitParameters.cs | 45 -- src/FrostFS.SDK.Client/Pool/InnerPool.cs | 47 -- src/FrostFS.SDK.Client/Pool/MethodIndex.cs | 24 - src/FrostFS.SDK.Client/Pool/MethodStatus.cs | 19 - src/FrostFS.SDK.Client/Pool/NodeParam.cs | 12 - src/FrostFS.SDK.Client/Pool/NodeStatistic.cs | 12 - src/FrostFS.SDK.Client/Pool/NodesParam.cs | 12 - src/FrostFS.SDK.Client/Pool/Pool.cs | 677 ----------------- .../Pool/RebalanceParameters.cs | 16 - src/FrostFS.SDK.Client/Pool/RequestInfo.cs | 14 - src/FrostFS.SDK.Client/Pool/Sampler.cs | 85 --- src/FrostFS.SDK.Client/Pool/Statistic.cs | 12 - src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs | 8 - src/FrostFS.SDK.Client/Pool/WorkList.cs | 26 - src/FrostFS.SDK.Client/Pool/WrapperPrm.cs | 39 - .../Services/ApeManagerServiceProvider.cs | 2 - .../Services/ContainerServiceProvider.cs | 7 +- .../Services/ObjectServiceProvider.cs | 25 +- .../{Pool => Services/Shared}/SessionCache.cs | 14 +- src/FrostFS.SDK.Client/Tools/ClientContext.cs | 2 +- src/FrostFS.SDK.Client/Tools/ObjectTools.cs | 25 +- src/FrostFS.SDK.Client/Tools/RequestSigner.cs | 22 +- src/FrostFS.SDK.Client/Tools/Verifier.cs | 66 +- src/FrostFS.SDK.Cryptography/ArrayHelper.cs | 14 + src/FrostFS.SDK.Cryptography/AssemblyInfo.cs | 4 +- src/FrostFS.SDK.Cryptography/Base58.cs | 32 +- src/FrostFS.SDK.Cryptography/DataHasher.cs | 121 ++++ src/FrostFS.SDK.Cryptography/Extentions.cs | 79 -- .../FrostFS.SDK.Cryptography.csproj | 1 - src/FrostFS.SDK.Cryptography/HashStream.cs | 4 +- src/FrostFS.SDK.Cryptography/Key.cs | 36 +- src/FrostFS.SDK.Protos/AssemblyInfo.cs | 4 +- .../Mocks/AsyncStreamReaderMock.cs | 3 +- .../ContainerServiceBase.cs | 2 - .../ContainerServiceMocks/GetContainerMock.cs | 7 +- src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs | 1 - .../Client/ContainerTests/ContainerTests.cs | 41 +- .../Smoke/Client/NetworkTests/NetworkTests.cs | 10 +- .../Smoke/Client/ObjectTests/ObjectTests.cs | 6 +- .../Multithread/MultiThreadTestsBase.cs | 42 -- .../Multithread/MultithreadPoolSmokeTests.cs | 544 -------------- .../MultithreadSmokeClientTests.cs | 684 ------------------ .../Smoke/PoolTests/PoolSmokeTests.cs | 546 -------------- src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs | 16 +- src/FrostFS.SDK.Tests/Unit/ContainerTest.cs | 10 +- src/FrostFS.SDK.Tests/Unit/ObjectTest.cs | 4 +- src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs | 2 +- src/FrostFS.SDK.Tests/Unit/SessionTests.cs | 3 +- 76 files changed, 399 insertions(+), 3668 deletions(-) delete mode 100644 src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/ClientWrapper.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/HealthyStatus.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/IClientStatus.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/InitParameters.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/InnerPool.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/MethodIndex.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/MethodStatus.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/NodeParam.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/NodeStatistic.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/NodesParam.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/Pool.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/RequestInfo.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/Sampler.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/Statistic.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/WorkList.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/WrapperPrm.cs rename src/FrostFS.SDK.Client/{Pool => Services/Shared}/SessionCache.cs (72%) create mode 100644 src/FrostFS.SDK.Cryptography/DataHasher.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs diff --git a/src/FrostFS.SDK.Client/ApeRules/Actions.cs b/src/FrostFS.SDK.Client/ApeRules/Actions.cs index 71888b9..8bae303 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Actions.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Actions.cs @@ -8,7 +8,7 @@ public struct Actions(bool inverted, string[] names) : System.IEquatable public override bool Equals(object obj) { - if (obj == null || obj is not Condition) + if (obj == null || obj is not Condition) return false; return Equals((Condition)obj); diff --git a/src/FrostFS.SDK.Client/ApeRules/Resources.cs b/src/FrostFS.SDK.Client/ApeRules/Resources.cs index ef06b4b..47ff3e1 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Resources.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Resources.cs @@ -8,7 +8,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable _ownersCache; - - internal static IMemoryCache Containers => _containersCache; } diff --git a/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs b/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs index 0e64db5..87799e2 100644 --- a/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs +++ b/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs @@ -10,8 +10,8 @@ public class FrostFsResponseException : FrostFsException { } - public FrostFsResponseException(FrostFsResponseStatus status) - : base(status != null ? status.Message != null ? "" : "" : "") + public FrostFsResponseException(FrostFsResponseStatus status) + : base(status != null ? status.Message != null ? "" : "" : "") { Status = status; } diff --git a/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs b/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs index a986a87..87728d5 100644 --- a/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs +++ b/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs @@ -1,14 +1,17 @@ using System; - +using System.Security.Cryptography; using Google.Protobuf; namespace FrostFS.SDK.Cryptography; public static class FrostFsExtensions { - public static ByteString Sha256(this IMessage data) + public static byte[] Sha256(this IMessage data) { - return ByteString.CopyFrom(data.ToByteArray().Sha256()); + using var sha256 = SHA256.Create(); + using HashStream stream = new(sha256); + data.WriteTo(stream); + return stream.Hash(); } public static Guid ToUuid(this ByteString id) @@ -16,9 +19,18 @@ public static class FrostFsExtensions if (id == null) throw new ArgumentNullException(nameof(id)); - var orderedBytes = GetGuidBytesDirectOrder(id.Span); - - return new Guid(orderedBytes); + return new Guid( + (id[0] << 24) + (id[1] << 16) + (id[2] << 8) + id[3], + (short)((id[4] << 8) + id[5]), + (short)((id[6] << 8) + id[7]), + id[8], + id[9], + id[10], + id[11], + id[12], + id[13], + id[14], + id[15]); } /// @@ -26,37 +38,25 @@ public static class FrostFsExtensions /// /// /// - public static byte[] ToBytes(this Guid id) + public unsafe static void ToBytes(this Guid id, Span span) { - var bytes = id.ToByteArray(); + var pGuid = (byte*)&id; - var orderedBytes = GetGuidBytesDirectOrder(bytes); - - return orderedBytes; - } - - private static byte[] GetGuidBytesDirectOrder(ReadOnlySpan source) - { - if (source.Length != 16) - throw new ArgumentException("Wrong uuid binary format"); - - return [ - source[3], - source[2], - source[1], - source[0], - source[5], - source[4], - source[7], - source[6], - source[8], - source[9], - source[10], - source[11], - source[12], - source[13], - source[14], - source[15] - ]; + span[0] = pGuid[3]; + span[1] = pGuid[2]; + span[2] = pGuid[1]; + span[3] = pGuid[0]; + span[4] = pGuid[5]; + span[5] = pGuid[4]; + span[6] = pGuid[7]; + span[7] = pGuid[6]; + span[8] = pGuid[8]; + span[9] = pGuid[9]; + span[10] = pGuid[10]; + span[11] = pGuid[11]; + span[12] = pGuid[12]; + span[13] = pGuid[13]; + span[14] = pGuid[14]; + span[15] = pGuid[15]; } } diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index 2205b7e..e94d3f8 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -31,10 +31,10 @@ false + True - all diff --git a/src/FrostFS.SDK.Client/FrostFSClient.cs b/src/FrostFS.SDK.Client/FrostFSClient.cs index f72265c..860d346 100644 --- a/src/FrostFS.SDK.Client/FrostFSClient.cs +++ b/src/FrostFS.SDK.Client/FrostFSClient.cs @@ -162,25 +162,7 @@ public class FrostFSClient : IFrostFSClient Callback = settings.Value.Callback, Interceptors = settings.Value.Interceptors }; - - // TODO: define timeout logic - // CheckFrostFsVersionSupport(new Context { Timeout = TimeSpan.FromSeconds(20) }); - } - - internal FrostFSClient(WrapperPrm prm, SessionCache cache) - { - ClientCtx = new ClientContext( - client: this, - key: new ClientKey(prm.Key), - owner: FrostFsOwner.FromKey(prm.Key!), - channel: prm.GrpcChannelFactory(prm.Address), - version: new FrostFsVersion(2, 13)) - { - SessionCache = cache, - Interceptors = prm.Interceptors, - Callback = prm.Callback - }; - } + } #region ApeManagerImplementation public Task> AddChainAsync(PrmApeChainAdd args, CallContext ctx) @@ -447,18 +429,5 @@ public class FrostFSClient : IFrostFSClient } return ObjectServiceProvider; - } - - public async Task Dial(CallContext ctx) - { - var service = GetAccouningService(); - _ = await service.GetBallance(ctx).ConfigureAwait(false); - - return null; - } - - public bool RestartIfUnhealthy(CallContext ctx) - { - throw new NotImplementedException(); - } + } } diff --git a/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs b/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs index d546156..43dc3ac 100644 --- a/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs +++ b/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs @@ -64,6 +64,4 @@ public interface IFrostFSClient #region Account Task GetBalanceAsync(CallContext ctx); #endregion - - public Task Dial(CallContext ctx); } diff --git a/src/FrostFS.SDK.Client/Mappers/ContainerId.cs b/src/FrostFS.SDK.Client/Mappers/ContainerId.cs index df27320..55b6179 100644 --- a/src/FrostFS.SDK.Client/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.Client/Mappers/ContainerId.cs @@ -5,16 +5,10 @@ using FrostFS.SDK.Cryptography; using Google.Protobuf; -using Microsoft.Extensions.Caching.Memory; - namespace FrostFS.SDK.Client.Mappers.GRPC; public static class ContainerIdMapper { - private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions() - .SetSlidingExpiration(TimeSpan.FromHours(1)) - .SetSize(1); - public static ContainerID ToMessage(this FrostFsContainerId model) { if (model is null) @@ -24,15 +18,11 @@ public static class ContainerIdMapper var containerId = model.GetValue() ?? throw new ArgumentNullException(nameof(model)); - if (!Caches.Containers.TryGetValue(containerId, out ContainerID? message)) + var message = new ContainerID { - message = new ContainerID - { - Value = ByteString.CopyFrom(Base58.Decode(containerId)) - }; + Value = UnsafeByteOperations.UnsafeWrap(Base58.Decode(containerId)) + }; - Caches.Containers.Set(containerId, message, _oneHourExpiration); - } return message!; } diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs index 317b31c..115f9e2 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs @@ -24,25 +24,14 @@ public static class ObjectHeaderMapper _ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.") }; - FrostFsSplit? split = null; - - if (header.Split != null) - { - var children = header.Split.Children.Count != 0 ? new ReadOnlyCollection( - header.Split.Children.Select(x => x.ToModel()).ToList()) : null; - - split = new FrostFsSplit(new SplitId(header.Split.SplitId.ToUuid()), - header.Split.Previous?.ToModel(), - header.Split.Parent?.ToModel(), - header.Split.ParentHeader?.ToModel(), - null, - children); - } + FrostFsSplit? split = header!.Split != null + ? header.Split.ToModel() + : null; var model = new FrostFsObjectHeader( new FrostFsContainerId(Base58.Encode(header.ContainerId.Value.Span)), objTypeName, - header.Attributes.Select(attribute => attribute.ToModel()).ToArray(), + [.. header.Attributes.Select(attribute => attribute.ToModel())], split, header.OwnerId.ToModel(), header.Version.ToModel()) @@ -52,4 +41,18 @@ public static class ObjectHeaderMapper return model; } + + public static FrostFsSplit ToModel(this Header.Types.Split split) + { + var children = split!.Children.Count != 0 + ? new ReadOnlyCollection([.. split.Children.Select(x => x.ToModel())]) + : null; + + return new FrostFsSplit(new SplitId(split.SplitId.ToUuid()), + split.Previous?.ToModel(), + split.Parent?.ToModel(), + split.ParentHeader?.ToModel(), + null, + children); + } } diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs index 343e3de..5562be5 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs @@ -17,7 +17,7 @@ public static class ObjectIdMapper return new ObjectID { - Value = ByteString.CopyFrom(objectId.ToHash()) + Value = UnsafeByteOperations.UnsafeWrap(objectId.ToHash()) }; } diff --git a/src/FrostFS.SDK.Client/Mappers/OwnerId.cs b/src/FrostFS.SDK.Client/Mappers/OwnerId.cs index 6739a0b..54f7a21 100644 --- a/src/FrostFS.SDK.Client/Mappers/OwnerId.cs +++ b/src/FrostFS.SDK.Client/Mappers/OwnerId.cs @@ -26,7 +26,7 @@ public static class OwnerIdMapper { message = new OwnerID { - Value = ByteString.CopyFrom(model.ToHash()) + Value = UnsafeByteOperations.UnsafeWrap(model.ToHash()) }; Caches.Owners.Set(model, message, _oneHourExpiration); diff --git a/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs b/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs deleted file mode 100644 index d1307e8..0000000 --- a/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; - -using Google.Protobuf; - -namespace FrostFS.SDK.Client; - -public static class SessionMapper -{ - public static byte[] Serialize(this Session.SessionToken token) - { - if (token is null) - { - throw new ArgumentNullException(nameof(token)); - } - - byte[] bytes = new byte[token.CalculateSize()]; - using CodedOutputStream stream = new(bytes); - token.WriteTo(stream); - - return bytes; - } - - public static Session.SessionToken Deserialize(this Session.SessionToken token, byte[] bytes) - { - token.MergeFrom(bytes); - return token; - } -} diff --git a/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs b/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs index 88b4ac8..ffae746 100644 --- a/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs @@ -23,9 +23,9 @@ public static class SignatureMapper return new Refs.Signature { - Key = ByteString.CopyFrom(signature.Key), + Key = UnsafeByteOperations.UnsafeWrap(signature.Key), Scheme = scheme, - Sign = ByteString.CopyFrom(signature.Sign) + Sign = UnsafeByteOperations.UnsafeWrap(signature.Sign) }; } } diff --git a/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs b/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs index af4ffdc..c7dc782 100644 --- a/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs +++ b/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs @@ -91,10 +91,13 @@ public class FrostFsContainerInfo throw new ArgumentNullException("PlacementPolicy is null"); } + Span nonce = stackalloc byte[16]; + Nonce.ToBytes(nonce); + this.container = new Container.Container() { PlacementPolicy = PlacementPolicy.Value.GetPolicy(), - Nonce = ByteString.CopyFrom(Nonce.ToBytes()), + Nonce = ByteString.CopyFrom(nonce), OwnerId = Owner?.OwnerID, Version = Version?.VersionID }; diff --git a/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs b/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs index fdeac99..2241227 100644 --- a/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs +++ b/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs @@ -11,7 +11,7 @@ public class CheckSum public static CheckSum CreateCheckSum(byte[] content) { - return new CheckSum { hash = content.Sha256() }; + return new CheckSum { hash = DataHasher.Sha256(content.AsSpan()) }; } public override string ToString() diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs index cd287d0..aedfd35 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs @@ -353,7 +353,7 @@ internal struct Context var start = hasPrefix ? likeWildcard.Length : 0; var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length; - var str = f.Value.Substring(start, end-start); + var str = f.Value.Substring(start, end - start); if (hasPrefix && hasSuffix) return nodeInfo.Attributes[f.Key].Contains(str); diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs index c39a587..e22016a 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs @@ -4,10 +4,6 @@ namespace FrostFS.SDK; public class FrostFsObject { - // private byte[]? _payloadBytes; - // private ReadOnlyMemory _payloadMemory; - // private bool _isInitPayloadMemory; - /// /// Creates new instance from ObjectHeader /// @@ -49,25 +45,6 @@ public class FrostFsObject public ReadOnlyMemory SingleObjectPayload { get; set; } - //public ReadOnlyMemory SingleObjectPayloadMemory - //{ - // get - // { - // if (!_isInitPayloadMemory) - // { - // _payloadMemory = _payloadBytes.AsMemory(); - // _isInitPayloadMemory = true; - // } - - // return _payloadMemory; - // } - // set - // { - // _payloadMemory = value; - // _isInitPayloadMemory = true; - // } - //} - /// /// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic /// diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs index bd850cb..5afd46b 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs @@ -1,4 +1,7 @@ using System.Collections.ObjectModel; +using System.Linq; +using FrostFS.Object; +using FrostFS.SDK.Client.Mappers.GRPC; namespace FrostFS.SDK; @@ -9,6 +12,8 @@ public class FrostFsSplit(SplitId splitId, FrostFsSignature? parentSignature = null, ReadOnlyCollection? children = null) { + private Header.Types.Split? _split; + public FrostFsSplit() : this(new SplitId()) { } @@ -24,4 +29,25 @@ public class FrostFsSplit(SplitId splitId, public FrostFsObjectHeader? ParentHeader { get; set; } = parentHeader; public ReadOnlyCollection? Children { get; } = children; + + public Header.Types.Split GetSplit() + { + if (_split == null) + { + _split = new Header.Types.Split + { + SplitId = SplitId?.GetSplitId(), + Parent = Parent?.ToMessage(), + ParentHeader = ParentHeader?.GetHeader(), + ParentSignature = ParentSignature?.ToMessage() + }; + + if (Children != null) + { + _split.Children.AddRange(Children.Select(x => x.ToMessage())); + } + } + + return _split; + } } diff --git a/src/FrostFS.SDK.Client/Models/Object/SplitId.cs b/src/FrostFS.SDK.Client/Models/Object/SplitId.cs index 177817e..a113361 100644 --- a/src/FrostFS.SDK.Client/Models/Object/SplitId.cs +++ b/src/FrostFS.SDK.Client/Models/Object/SplitId.cs @@ -47,16 +47,11 @@ public class SplitId return this.id.ToString(); } - public byte[]? ToBinary() - { - if (this.id == Guid.Empty) - return null; - - return this.id.ToBytes(); - } - public ByteString? GetSplitId() { - return this.message ??= ByteString.CopyFrom(ToBinary()); + Span span = stackalloc byte[16]; + id.ToBytes(span); + + return this.message ??= ByteString.CopyFrom(span); } } diff --git a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs index 2f48ba3..9e1a846 100644 --- a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs +++ b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs @@ -4,9 +4,9 @@ public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = nul { public FrostFsStatusCode Code { get; set; } = code; public string Message { get; set; } = message ?? string.Empty; - + public string Details { get; set; } = details ?? string.Empty; - + public bool IsSuccess => Code == FrostFsStatusCode.Success; public override string ToString() diff --git a/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs b/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs index 720c3c4..2303776 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs @@ -25,7 +25,7 @@ public readonly struct PrmApeChainRemove( public readonly bool Equals(PrmApeChainRemove other) { return Target == other.Target - && ChainId.Equals(other.ChainId) + && ChainId.Equals(other.ChainId) && XHeaders == other.XHeaders; } diff --git a/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs b/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs deleted file mode 100644 index c05c24c..0000000 --- a/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System; -using System.Threading; - -using Microsoft.Extensions.Logging; - -namespace FrostFS.SDK.Client; - -// clientStatusMonitor count error rate and other statistics for connection. -public class ClientStatusMonitor : IClientStatus -{ - private static readonly MethodIndex[] MethodIndexes = - [ - MethodIndex.methodBalanceGet, - MethodIndex.methodContainerPut, - MethodIndex.methodContainerGet, - MethodIndex.methodContainerList, - MethodIndex.methodContainerDelete, - MethodIndex.methodEndpointInfo, - MethodIndex.methodNetworkInfo, - MethodIndex.methodNetMapSnapshot, - MethodIndex.methodObjectPut, - MethodIndex.methodObjectDelete, - MethodIndex.methodObjectGet, - MethodIndex.methodObjectHead, - MethodIndex.methodObjectRange, - MethodIndex.methodObjectPatch, - MethodIndex.methodSessionCreate, - MethodIndex.methodAPEManagerAddChain, - MethodIndex.methodAPEManagerRemoveChain, - MethodIndex.methodAPEManagerListChains - ]; - - public static string GetMethodName(MethodIndex index) - { - return index switch - { - MethodIndex.methodBalanceGet => "BalanceGet", - MethodIndex.methodContainerPut => "ContainerPut", - MethodIndex.methodContainerGet => "ContainerGet", - MethodIndex.methodContainerList => "ContainerList", - MethodIndex.methodContainerDelete => "ContainerDelete", - MethodIndex.methodEndpointInfo => "EndpointInfo", - MethodIndex.methodNetworkInfo => "NetworkInfo", - MethodIndex.methodNetMapSnapshot => "NetMapSnapshot", - MethodIndex.methodObjectPut => "ObjectPut", - MethodIndex.methodObjectDelete => "ObjectDelete", - MethodIndex.methodObjectGet => "ObjectGet", - MethodIndex.methodObjectHead => "ObjectHead", - MethodIndex.methodObjectRange => "ObjectRange", - MethodIndex.methodObjectPatch => "ObjectPatch", - MethodIndex.methodSessionCreate => "SessionCreate", - MethodIndex.methodAPEManagerAddChain => "APEManagerAddChain", - MethodIndex.methodAPEManagerRemoveChain => "APEManagerRemoveChain", - MethodIndex.methodAPEManagerListChains => "APEManagerListChains", - _ => throw new ArgumentException("Unknown method", nameof(index)), - }; - } - - private readonly object _lock = new(); - - private readonly ILogger? logger; - private int healthy; - - public ClientStatusMonitor(ILogger? logger, string address) - { - this.logger = logger; - healthy = (int)HealthyStatus.Healthy; - - Address = address; - Methods = new MethodStatus[MethodIndexes.Length]; - - for (int i = 0; i < MethodIndexes.Length; i++) - { - Methods[i] = new MethodStatus(GetMethodName(MethodIndexes[i])); - } - } - - public string Address { get; } - - internal uint ErrorThreshold { get; set; } - - public uint CurrentErrorCount { get; set; } - - public ulong OverallErrorCount { get; set; } - - public MethodStatus[] Methods { get; private set; } - - public bool IsHealthy() - { - var res = Interlocked.CompareExchange(ref healthy, -1, -1) == (int)HealthyStatus.Healthy; - return res; - } - - public bool IsDialed() - { - return Interlocked.CompareExchange(ref healthy, -1, -1) != (int)HealthyStatus.UnhealthyOnDial; - } - - public void SetHealthy() - { - Interlocked.Exchange(ref healthy, (int)HealthyStatus.Healthy); - } - public void SetUnhealthy() - { - Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnRequest); - } - - public void SetUnhealthyOnDial() - { - Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnDial); - } - - public void IncErrorRate() - { - bool thresholdReached; - lock (_lock) - { - CurrentErrorCount++; - OverallErrorCount++; - - thresholdReached = CurrentErrorCount >= ErrorThreshold; - - if (thresholdReached) - { - SetUnhealthy(); - CurrentErrorCount = 0; - } - } - - if (thresholdReached && logger != null) - { - FrostFsMessages.ErrorЕhresholdReached(logger, Address, ErrorThreshold); - } - } - - public uint GetCurrentErrorRate() - { - lock (_lock) - { - return CurrentErrorCount; - } - } - - public ulong GetOverallErrorRate() - { - lock (_lock) - { - return OverallErrorCount; - } - } - - public StatusSnapshot[] MethodsStatus() - { - var result = new StatusSnapshot[Methods.Length]; - - for (int i = 0; i < result.Length; i++) - { - result[i] = Methods[i].Snapshot!; - } - - return result; - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs b/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs deleted file mode 100644 index 552ee2a..0000000 --- a/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs +++ /dev/null @@ -1,135 +0,0 @@ -using System; -using System.Threading.Tasks; - -using Grpc.Core; - -namespace FrostFS.SDK.Client; - -// clientWrapper is used by default, alternative implementations are intended for testing purposes only. -public class ClientWrapper : ClientStatusMonitor -{ - private readonly object _lock = new(); - private SessionCache sessionCache; - - internal ClientWrapper(WrapperPrm wrapperPrm, Pool pool) : base(wrapperPrm.Logger, wrapperPrm.Address) - { - WrapperPrm = wrapperPrm; - ErrorThreshold = wrapperPrm.ErrorThreshold; - - sessionCache = pool.SessionCache; - Client = new FrostFSClient(WrapperPrm, sessionCache); - } - - internal FrostFSClient? Client { get; private set; } - - internal WrapperPrm WrapperPrm { get; } - - internal FrostFSClient? GetClient() - { - lock (_lock) - { - if (IsHealthy()) - { - return Client; - } - - return null; - } - } - - // dial establishes a connection to the server from the FrostFS network. - // Returns an error describing failure reason. If failed, the client - // SHOULD NOT be used. - internal async Task Dial(CallContext ctx) - { - var client = GetClient() ?? throw new FrostFsInvalidObjectException("pool client unhealthy"); - - await client.Dial(ctx).ConfigureAwait(false); - } - - internal void HandleError(Exception ex) - { - if (ex is FrostFsResponseException responseException && responseException.Status != null) - { - switch (responseException.Status.Code) - { - case FrostFsStatusCode.Internal: - case FrostFsStatusCode.WrongMagicNumber: - case FrostFsStatusCode.SignatureVerificationFailure: - case FrostFsStatusCode.NodeUnderMaintenance: - IncErrorRate(); - return; - } - } - - IncErrorRate(); - } - - private async Task ScheduleGracefulClose() - { - if (Client == null) - return; - - await Task.Delay((int)WrapperPrm.GracefulCloseOnSwitchTimeout).ConfigureAwait(false); - } - - // restartIfUnhealthy checks healthy status of client and recreate it if status is unhealthy. - // Indicating if status was changed by this function call and returns error that caused unhealthy status. - internal async Task RestartIfUnhealthy(CallContext ctx) - { - bool wasHealthy; - - try - { - var response = await Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false); - return false; - } - catch (RpcException) - { - wasHealthy = true; - } - - // if connection is dialed before, to avoid routine/connection leak, - // pool has to close it and then initialize once again. - if (IsDialed()) - { - await ScheduleGracefulClose().ConfigureAwait(false); - } - - FrostFSClient? client = new(WrapperPrm, sessionCache); - - var error = await client.Dial(ctx).ConfigureAwait(false); - if (!string.IsNullOrEmpty(error)) - { - SetUnhealthyOnDial(); - return wasHealthy; - } - - lock (_lock) - { - Client = client; - } - - try - { - var res = await Client.GetNodeInfoAsync(ctx).ConfigureAwait(false); - } - catch (FrostFsException) - { - SetUnhealthy(); - - return wasHealthy; - } - - SetHealthy(); - return !wasHealthy; - } - - internal void IncRequests(ulong elapsed, MethodIndex method) - { - var methodStat = Methods[(int)method]; - - methodStat.IncRequests(elapsed); - } -} - diff --git a/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs b/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs deleted file mode 100644 index d02e6bb..0000000 --- a/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace FrostFS.SDK.Client; - -// values for healthy status of clientStatusMonitor. -public enum HealthyStatus -{ - // statusUnhealthyOnDial is set when dialing to the endpoint is failed, - // so there is no connection to the endpoint, and pool should not close it - // before re-establishing connection once again. - UnhealthyOnDial, - - // statusUnhealthyOnRequest is set when communication after dialing to the - // endpoint is failed due to immediate or accumulated errors, connection is - // available and pool should close it before re-establishing connection once again. - UnhealthyOnRequest, - - // statusHealthy is set when connection is ready to be used by the pool. - Healthy -} diff --git a/src/FrostFS.SDK.Client/Pool/IClientStatus.cs b/src/FrostFS.SDK.Client/Pool/IClientStatus.cs deleted file mode 100644 index 0c08fac..0000000 --- a/src/FrostFS.SDK.Client/Pool/IClientStatus.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace FrostFS.SDK.Client; - -public interface IClientStatus -{ - // isHealthy checks if the connection can handle requests. - bool IsHealthy(); - - // isDialed checks if the connection was created. - bool IsDialed(); - - // setUnhealthy marks client as unhealthy. - void SetUnhealthy(); - - // address return address of endpoint. - string Address { get; } - - // currentErrorRate returns current errors rate. - // After specific threshold connection is considered as unhealthy. - // Pool.startRebalance routine can make this connection healthy again. - uint GetCurrentErrorRate(); - - // overallErrorRate returns the number of all happened errors. - ulong GetOverallErrorRate(); - - // methodsStatus returns statistic for all used methods. - StatusSnapshot[] MethodsStatus(); -} - diff --git a/src/FrostFS.SDK.Client/Pool/InitParameters.cs b/src/FrostFS.SDK.Client/Pool/InitParameters.cs deleted file mode 100644 index 66da23f..0000000 --- a/src/FrostFS.SDK.Client/Pool/InitParameters.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.ObjectModel; -using System.Security.Cryptography; - -using Grpc.Core; -using Grpc.Core.Interceptors; -using Grpc.Net.Client; - -using Microsoft.Extensions.Logging; - -namespace FrostFS.SDK.Client; - -// InitParameters contains values used to initialize connection Pool. -public class InitParameters(Func grpcChannelFactory) -{ - public ECDsa? Key { get; set; } - - public ulong NodeDialTimeout { get; set; } - - public ulong NodeStreamTimeout { get; set; } - - public ulong HealthcheckTimeout { get; set; } - - public ulong ClientRebalanceInterval { get; set; } - - public ulong SessionExpirationDuration { get; set; } - - public uint ErrorThreshold { get; set; } - - public NodeParam[]? NodeParams { get; set; } - - public GrpcChannelOptions[]? DialOptions { get; set; } - - public Func? ClientBuilder { get; set; } - - public ulong GracefulCloseOnSwitchTimeout { get; set; } - - public ILogger? Logger { get; set; } - - public Action? Callback { get; set; } - - public Collection Interceptors { get; } = []; - - public Func GrpcChannelFactory { get; set; } = grpcChannelFactory; -} diff --git a/src/FrostFS.SDK.Client/Pool/InnerPool.cs b/src/FrostFS.SDK.Client/Pool/InnerPool.cs deleted file mode 100644 index f712552..0000000 --- a/src/FrostFS.SDK.Client/Pool/InnerPool.cs +++ /dev/null @@ -1,47 +0,0 @@ -using FrostFS.SDK.Client; - -internal sealed class InnerPool -{ - private readonly object _lock = new(); - - internal InnerPool(Sampler sampler, ClientWrapper[] clients) - { - Clients = clients; - Sampler = sampler; - } - - internal Sampler Sampler { get; set; } - - internal ClientWrapper[] Clients { get; } - - internal ClientWrapper? Connection() - { - lock (_lock) - { - if (Clients.Length == 1) - { - var client = Clients[0]; - if (client.IsHealthy()) - { - return client; - } - } - else - { - var attempts = 3 * Clients.Length; - - for (int i = 0; i < attempts; i++) - { - int index = Sampler.Next(); - - if (Clients[index].IsHealthy()) - { - return Clients[index]; - } - } - } - - return null; - } - } -} diff --git a/src/FrostFS.SDK.Client/Pool/MethodIndex.cs b/src/FrostFS.SDK.Client/Pool/MethodIndex.cs deleted file mode 100644 index 53e5430..0000000 --- a/src/FrostFS.SDK.Client/Pool/MethodIndex.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace FrostFS.SDK.Client; - -public enum MethodIndex -{ - methodBalanceGet, - methodContainerPut, - methodContainerGet, - methodContainerList, - methodContainerDelete, - methodEndpointInfo, - methodNetworkInfo, - methodNetMapSnapshot, - methodObjectPut, - methodObjectDelete, - methodObjectGet, - methodObjectHead, - methodObjectRange, - methodObjectPatch, - methodSessionCreate, - methodAPEManagerAddChain, - methodAPEManagerRemoveChain, - methodAPEManagerListChains, - methodLast -} diff --git a/src/FrostFS.SDK.Client/Pool/MethodStatus.cs b/src/FrostFS.SDK.Client/Pool/MethodStatus.cs deleted file mode 100644 index 8f40f3c..0000000 --- a/src/FrostFS.SDK.Client/Pool/MethodStatus.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class MethodStatus(string name) -{ - private readonly object _lock = new(); - - public string Name { get; } = name; - - public StatusSnapshot Snapshot { get; set; } = new StatusSnapshot(); - - internal void IncRequests(ulong elapsed) - { - lock (_lock) - { - Snapshot.AllTime += elapsed; - Snapshot.AllRequests++; - } - } -} diff --git a/src/FrostFS.SDK.Client/Pool/NodeParam.cs b/src/FrostFS.SDK.Client/Pool/NodeParam.cs deleted file mode 100644 index 92c2560..0000000 --- a/src/FrostFS.SDK.Client/Pool/NodeParam.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace FrostFS.SDK.Client; - -// NodeParam groups parameters of remote node. -[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "")] -public readonly struct NodeParam(int priority, string address, float weight) -{ - public int Priority { get; } = priority; - - public string Address { get; } = address; - - public float Weight { get; } = weight; -} diff --git a/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs b/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs deleted file mode 100644 index 9323d93..0000000 --- a/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class NodeStatistic -{ - public string? Address { get; internal set; } - - public StatusSnapshot[]? Methods { get; internal set; } - - public ulong OverallErrors { get; internal set; } - - public uint CurrentErrors { get; internal set; } -} diff --git a/src/FrostFS.SDK.Client/Pool/NodesParam.cs b/src/FrostFS.SDK.Client/Pool/NodesParam.cs deleted file mode 100644 index be9f012..0000000 --- a/src/FrostFS.SDK.Client/Pool/NodesParam.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Collections.ObjectModel; - -namespace FrostFS.SDK.Client; - -public class NodesParam(int priority) -{ - public int Priority { get; } = priority; - - public Collection Addresses { get; } = []; - - public Collection Weights { get; } = []; -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Pool/Pool.cs b/src/FrostFS.SDK.Client/Pool/Pool.cs deleted file mode 100644 index cd09d30..0000000 --- a/src/FrostFS.SDK.Client/Pool/Pool.cs +++ /dev/null @@ -1,677 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography; -using System.Threading; -using System.Threading.Tasks; - -using FrostFS.Refs; -using FrostFS.SDK.Client.Interfaces; -using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; - -using Grpc.Core; - -using Microsoft.Extensions.Logging; - - -namespace FrostFS.SDK.Client; - -public partial class Pool : IFrostFSClient -{ - const int defaultSessionTokenExpirationDuration = 100; // in epochs - - const int defaultErrorThreshold = 100; - - const int defaultGracefulCloseOnSwitchTimeout = 10; //Seconds; - const int defaultRebalanceInterval = 15; //Seconds; - const int defaultHealthcheckTimeout = 4; //Seconds; - const int defaultDialTimeout = 5; //Seconds; - const int defaultStreamTimeout = 10; //Seconds; - - private readonly object _lock = new(); - - private InnerPool[]? InnerPools { get; set; } - - private ClientKey Key { get; set; } - - private OwnerID? _ownerId; - - private FrostFsVersion _version; - - private FrostFsOwner? _owner; - - private FrostFsOwner Owner - { - get - { - _owner ??= new FrostFsOwner(Key.ECDsaKey.PublicKey().PublicKeyToAddress()); - return _owner; - } - } - - private OwnerID OwnerId - { - get - { - if (_ownerId == null) - { - _owner = Key.Owner; - _ownerId = _owner.ToMessage(); - } - return _ownerId; - } - } - - internal CancellationTokenSource CancellationTokenSource { get; } = new CancellationTokenSource(); - - internal SessionCache SessionCache { get; set; } - - private ulong SessionTokenDuration { get; set; } - - private RebalanceParameters RebalanceParams { get; set; } - - private Func ClientBuilder; - - private bool disposedValue; - - private ILogger? logger { get; set; } - - private ulong MaxObjectSize { get; set; } - - public IClientStatus? ClientStatus { get; } - - // NewPool creates connection pool using parameters. - public Pool(InitParameters options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (options.Key == null) - { - throw new ArgumentException($"Missed required parameter {nameof(options.Key)}"); - } - - _version = new FrostFsVersion(2, 13); - - var nodesParams = AdjustNodeParams(options.NodeParams); - - var cache = new SessionCache(options.SessionExpirationDuration); - - FillDefaultInitParams(options, this); - - Key = new ClientKey(options.Key); - - SessionCache = cache; - logger = options.Logger; - SessionTokenDuration = options.SessionExpirationDuration; - - RebalanceParams = new RebalanceParameters( - nodesParams.ToArray(), - options.HealthcheckTimeout, - options.ClientRebalanceInterval, - options.SessionExpirationDuration); - - ClientBuilder = options.ClientBuilder!; - - // ClientContext.PoolErrorHandler = client.HandleError; - - } - - // Dial establishes a connection to the servers from the FrostFS network. - // It also starts a routine that checks the health of the nodes and - // updates the weights of the nodes for balancing. - // Returns an error describing failure reason. - // - // If failed, the Pool SHOULD NOT be used. - // - // See also InitParameters.SetClientRebalanceInterval. - public async Task Dial(CallContext ctx) - { - var inner = new InnerPool[RebalanceParams.NodesParams.Length]; - - bool atLeastOneHealthy = false; - int i = 0; - foreach (var nodeParams in RebalanceParams.NodesParams) - { - var wrappers = new ClientWrapper[nodeParams.Weights.Count]; - - for (int j = 0; j < nodeParams.Addresses.Count; j++) - { - ClientWrapper? wrapper = null; - bool dialed = false; - try - { - wrapper = wrappers[j] = ClientBuilder(nodeParams.Addresses[j]); - - await wrapper.Dial(ctx).ConfigureAwait(false); - dialed = true; - - var token = await InitSessionForDuration(ctx, wrapper, RebalanceParams.SessionExpirationDuration, Key.ECDsaKey, false) - .ConfigureAwait(false); - - var key = FormCacheKey(nodeParams.Addresses[j], Key.PublicKey); - SessionCache.SetValue(key, token); - - atLeastOneHealthy = true; - } - catch (RpcException ex) - { - if (!dialed) - wrapper!.SetUnhealthyOnDial(); - else - wrapper!.SetUnhealthy(); - - if (logger != null) - { - FrostFsMessages.SessionCreationError(logger, wrapper!.WrapperPrm.Address, ex.Message); - } - } - catch (FrostFsInvalidObjectException) - { - break; - } - } - - var sampler = new Sampler(nodeParams.Weights.ToArray()); - - inner[i] = new InnerPool(sampler, wrappers); - - i++; - } - - if (!atLeastOneHealthy) - return "At least one node must be healthy"; - - InnerPools = inner; - - var res = await GetNetworkSettingsAsync(default).ConfigureAwait(false); - - MaxObjectSize = res.MaxObjectSize; - - StartRebalance(ctx); - - return null; - } - - private static IEnumerable AdjustNodeParams(NodeParam[]? nodeParams) - { - if (nodeParams == null || nodeParams.Length == 0) - { - throw new ArgumentException("No FrostFS peers configured"); - } - - Dictionary nodesParamsDict = new(nodeParams.Length); - foreach (var nodeParam in nodeParams) - { - if (!nodesParamsDict.TryGetValue(nodeParam.Priority, out var nodes)) - { - nodes = new NodesParam(nodeParam.Priority); - nodesParamsDict[nodeParam.Priority] = nodes; - } - - nodes.Addresses.Add(nodeParam.Address); - nodes.Weights.Add(nodeParam.Weight); - } - - var nodesParams = new List(nodesParamsDict.Count); - - foreach (var key in nodesParamsDict.Keys) - { - var nodes = nodesParamsDict[key]; - var newWeights = AdjustWeights([.. nodes.Weights]); - nodes.Weights.Clear(); - foreach (var weight in newWeights) - { - nodes.Weights.Add(weight); - } - - nodesParams.Add(nodes); - } - - return nodesParams.OrderBy(n => n.Priority); - } - - private static double[] AdjustWeights(double[] weights) - { - var adjusted = new double[weights.Length]; - - var sum = weights.Sum(); - - if (sum > 0) - { - for (int i = 0; i < weights.Length; i++) - { - adjusted[i] = weights[i] / sum; - } - } - - return adjusted; - } - - private static void FillDefaultInitParams(InitParameters parameters, Pool pool) - { - if (parameters.SessionExpirationDuration == 0) - parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration; - - if (parameters.ErrorThreshold == 0) - parameters.ErrorThreshold = defaultErrorThreshold; - - if (parameters.ClientRebalanceInterval <= 0) - parameters.ClientRebalanceInterval = defaultRebalanceInterval; - - if (parameters.GracefulCloseOnSwitchTimeout <= 0) - parameters.GracefulCloseOnSwitchTimeout = defaultGracefulCloseOnSwitchTimeout; - - if (parameters.HealthcheckTimeout <= 0) - parameters.HealthcheckTimeout = defaultHealthcheckTimeout; - - if (parameters.NodeDialTimeout <= 0) - parameters.NodeDialTimeout = defaultDialTimeout; - - if (parameters.NodeStreamTimeout <= 0) - parameters.NodeStreamTimeout = defaultStreamTimeout; - - if (parameters.SessionExpirationDuration == 0) - parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration; - - parameters.ClientBuilder ??= new Func((address) => - { - var wrapperPrm = new WrapperPrm() - { - Address = address, - Key = parameters.Key!, - Logger = parameters.Logger, - DialTimeout = parameters.NodeDialTimeout, - StreamTimeout = parameters.NodeStreamTimeout, - ErrorThreshold = parameters.ErrorThreshold, - GracefulCloseOnSwitchTimeout = parameters.GracefulCloseOnSwitchTimeout, - Callback = parameters.Callback, - Interceptors = parameters.Interceptors, - GrpcChannelFactory = parameters.GrpcChannelFactory - - }; - - return new ClientWrapper(wrapperPrm, pool); - } - ); - } - - private ClientWrapper Connection() - { - foreach (var pool in InnerPools!) - { - var client = pool.Connection(); - if (client != null) - { - return client; - } - } - - throw new FrostFsException("Cannot find alive client"); - } - - private static async Task InitSessionForDuration(CallContext ctx, ClientWrapper cw, ulong duration, ECDsa key, bool clientCut) - { - var client = cw.Client; - var networkInfo = await client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - - var epoch = networkInfo.Epoch; - - ulong exp = ulong.MaxValue - epoch < duration - ? ulong.MaxValue - : epoch + duration; - - var prmSessionCreate = new PrmSessionCreate(exp); - - return await client.CreateSessionAsync(prmSessionCreate, ctx).ConfigureAwait(false); - } - - internal static string FormCacheKey(string address, string key) - { - return $"{address}{key}"; - } - - public void Close() - { - CancellationTokenSource.Cancel(); - - //if (InnerPools != null) - //{ - // // close all clients - // foreach (var innerPool in InnerPools) - // foreach (var client in innerPool.Clients) - // if (client.IsDialed()) - // client.Client?.Close(); - //} - } - - // startRebalance runs loop to monitor connection healthy status. - internal void StartRebalance(CallContext ctx) - { - var buffers = new double[RebalanceParams.NodesParams.Length][]; - - for (int i = 0; i < RebalanceParams.NodesParams.Length; i++) - { - var parameters = RebalanceParams.NodesParams[i]; - buffers[i] = new double[parameters.Weights.Count]; - - Task.Run(async () => - { - await Task.Delay((int)RebalanceParams.ClientRebalanceInterval).ConfigureAwait(false); - UpdateNodesHealth(ctx, buffers); - }); - } - } - - private void UpdateNodesHealth(CallContext ctx, double[][] buffers) - { - var tasks = new Task[InnerPools!.Length]; - - for (int i = 0; i < InnerPools.Length; i++) - { - var bufferWeights = buffers[i]; - - tasks[i] = Task.Run(() => UpdateInnerNodesHealth(ctx, i, bufferWeights)); - } - - Task.WaitAll(tasks); - } - - private async ValueTask UpdateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights) - { - if (poolIndex > InnerPools!.Length - 1) - { - return; - } - - var pool = InnerPools[poolIndex]; - - var options = RebalanceParams; - - int healthyChanged = 0; - - var tasks = new Task[pool.Clients.Length]; - - for (int j = 0; j < pool.Clients.Length; j++) - { - var client = pool.Clients[j]; - var healthy = false; - string? error = null; - var changed = false; - - try - { - // check timeout settings - changed = await client!.RestartIfUnhealthy(ctx).ConfigureAwait(false); - healthy = true; - bufferWeights[j] = options.NodesParams[poolIndex].Weights[j]; - } - // TODO: specify - catch (FrostFsException e) - { - error = e.Message; - bufferWeights[j] = 0; - - SessionCache.DeleteByPrefix(client.Address); - } - - if (changed) - { - if (!string.IsNullOrEmpty(error)) - { - if (logger != null) - { - FrostFsMessages.HealthChanged(logger, client.Address, healthy, error!); - } - - Interlocked.Exchange(ref healthyChanged, 1); - } - } - - await Task.WhenAll(tasks).ConfigureAwait(false); - - if (Interlocked.CompareExchange(ref healthyChanged, -1, -1) == 1) - { - var probabilities = AdjustWeights(bufferWeights); - - lock (_lock) - { - pool.Sampler = new Sampler(probabilities); - } - } - } - } - - - // TODO: remove - private bool CheckSessionTokenErr(Exception error, string address) - { - if (error == null) - { - return false; - } - - if (error is SessionNotFoundException || error is SessionExpiredException) - { - this.SessionCache.DeleteByPrefix(address); - return true; - } - - return false; - } - - public Statistic Statistic() - { - if (InnerPools == null) - { - throw new FrostFsInvalidObjectException(nameof(Pool)); - } - - var statistics = new Statistic(); - - foreach (var inner in InnerPools) - { - int valueIndex = 0; - var nodes = new string[inner.Clients.Length]; - - lock (_lock) - { - foreach (var client in inner.Clients) - { - if (client.IsHealthy()) - { - nodes[valueIndex] = client.Address; - } - - var node = new NodeStatistic - { - Address = client.Address, - Methods = client.MethodsStatus(), - OverallErrors = client.GetOverallErrorRate(), - CurrentErrors = client.GetCurrentErrorRate() - }; - - statistics.Nodes.Add(node); - - valueIndex++; - - statistics.OverallErrors += node.OverallErrors; - } - - if (statistics.CurrentNodes == null || statistics.CurrentNodes.Length == 0) - { - statistics.CurrentNodes = nodes; - } - } - } - - return statistics; - } - - public async Task GetNetmapSnapshotAsync(CallContext ctx) - { - var client = Connection(); - - return await client.Client!.GetNetmapSnapshotAsync(ctx).ConfigureAwait(false); - } - - public async Task GetNodeInfoAsync(CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false); - } - - public async Task GetNetworkSettingsAsync(CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - } - - public async Task CreateSessionAsync(PrmSessionCreate args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.CreateSessionAsync(args, ctx).ConfigureAwait(false); - } - - public async Task> AddChainAsync(PrmApeChainAdd args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.AddChainAsync(args, ctx).ConfigureAwait(false); - } - - public async Task RemoveChainAsync(PrmApeChainRemove args, CallContext ctx) - { - var client = Connection(); - await client.Client!.RemoveChainAsync(args, ctx).ConfigureAwait(false); - } - - public async Task ListChainAsync(PrmApeChainList args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.ListChainAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetContainerAsync(PrmContainerGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetContainerAsync(args, ctx).ConfigureAwait(false); - } - - public IAsyncEnumerable ListContainersAsync(PrmContainerGetAll args, CallContext ctx) - { - var client = Connection(); - return client.Client!.ListContainersAsync(args, ctx); - } - - [Obsolete("Use PutContainerAsync method")] - public async Task CreateContainerAsync(PrmContainerCreate args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutContainerAsync(PrmContainerCreate args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false); - } - - public async Task DeleteContainerAsync(PrmContainerDelete args, CallContext ctx) - { - var client = Connection(); - await client.Client!.DeleteContainerAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetObjectHeadAsync(PrmObjectHeadGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetObjectHeadAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetObjectAsync(PrmObjectGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutObjectAsync(PrmObjectPut args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutClientCutObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutSingleObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PatchObjectAsync(PrmObjectPatch args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetRangeAsync(PrmRangeGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetRangeAsync(args, ctx).ConfigureAwait(false); - } - - public async Task[]> GetRangeHashAsync(PrmRangeHashGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetRangeHashAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PatchAsync(PrmObjectPatch args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task DeleteObjectAsync(PrmObjectDelete args, CallContext ctx) - { - var client = Connection(); - await client.Client!.DeleteObjectAsync(args, ctx).ConfigureAwait(false); - } - - public IAsyncEnumerable SearchObjectsAsync(PrmObjectSearch args, CallContext ctx) - { - var client = Connection(); - return client.Client!.SearchObjectsAsync(args, ctx); - } - - public async Task GetBalanceAsync(CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetBalanceAsync(ctx).ConfigureAwait(false); - } - - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - Close(); - } - - disposedValue = true; - } - } - - public void Dispose() - { - Dispose(disposing: true); - } -} diff --git a/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs b/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs deleted file mode 100644 index 988002c..0000000 --- a/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class RebalanceParameters( - NodesParam[] nodesParams, - ulong nodeRequestTimeout, - ulong clientRebalanceInterval, - ulong sessionExpirationDuration) -{ - public NodesParam[] NodesParams { get; set; } = nodesParams; - - public ulong NodeRequestTimeout { get; set; } = nodeRequestTimeout; - - public ulong ClientRebalanceInterval { get; set; } = clientRebalanceInterval; - - public ulong SessionExpirationDuration { get; set; } = sessionExpirationDuration; -} diff --git a/src/FrostFS.SDK.Client/Pool/RequestInfo.cs b/src/FrostFS.SDK.Client/Pool/RequestInfo.cs deleted file mode 100644 index 3b70650..0000000 --- a/src/FrostFS.SDK.Client/Pool/RequestInfo.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace FrostFS.SDK.Client; - -// RequestInfo groups info about pool request. -struct RequestInfo -{ - public string Address { get; set; } - - public MethodIndex MethodIndex { get; set; } - - public TimeSpan Elapsed { get; set; } -} - diff --git a/src/FrostFS.SDK.Client/Pool/Sampler.cs b/src/FrostFS.SDK.Client/Pool/Sampler.cs deleted file mode 100644 index 275f3f0..0000000 --- a/src/FrostFS.SDK.Client/Pool/Sampler.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; - -namespace FrostFS.SDK.Client; - -internal sealed class Sampler -{ - private readonly object _lock = new(); - - private Random random = new(); - - internal double[] Probabilities { get; set; } - internal int[] Alias { get; set; } - - internal Sampler(double[] probabilities) - { - var small = new WorkList(); - var large = new WorkList(); - - var n = probabilities.Length; - - // sampler.randomGenerator = rand.New(source) - Probabilities = new double[n]; - Alias = new int[n]; - - // Compute scaled probabilities. - var p = new double[n]; - - for (int i = 0; i < n; i++) - { - p[i] = probabilities[i] * n; - if (p[i] < 1) - small.Add(i); - else - large.Add(i); - } - - while (small.Length > 0 && large.Length > 0) - { - var l = small.Remove(); - var g = large.Remove(); - - Probabilities[l] = p[l]; - Alias[l] = g; - - p[g] = p[g] + p[l] - 1; - - if (p[g] < 1) - small.Add(g); - else - large.Add(g); - } - - while (large.Length > 0) - { - var g = large.Remove(); - Probabilities[g] = 1; - } - - while (small.Length > 0) - { - var l = small.Remove(); - probabilities[l] = 1; - } - } - - internal int Next() - { - var n = Alias.Length; - - int i; - double f; - lock (_lock) - { - i = random.Next(0, n - 1); - f = random.NextDouble(); - } - - if (f < Probabilities[i]) - { - return i; - } - - return Alias[i]; - } -} diff --git a/src/FrostFS.SDK.Client/Pool/Statistic.cs b/src/FrostFS.SDK.Client/Pool/Statistic.cs deleted file mode 100644 index c4977d5..0000000 --- a/src/FrostFS.SDK.Client/Pool/Statistic.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Collections.ObjectModel; - -namespace FrostFS.SDK.Client; - -public sealed class Statistic -{ - public ulong OverallErrors { get; internal set; } - - public Collection Nodes { get; } = []; - - public string[]? CurrentNodes { get; internal set; } -} diff --git a/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs b/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs deleted file mode 100644 index 2156f99..0000000 --- a/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class StatusSnapshot() -{ - public ulong AllTime { get; internal set; } - - public ulong AllRequests { get; internal set; } -} diff --git a/src/FrostFS.SDK.Client/Pool/WorkList.cs b/src/FrostFS.SDK.Client/Pool/WorkList.cs deleted file mode 100644 index 7796e86..0000000 --- a/src/FrostFS.SDK.Client/Pool/WorkList.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace FrostFS.SDK.Client; - -internal sealed class WorkList -{ - private readonly List elements = []; - - internal int Length - { - get { return elements.Count; } - } - - internal void Add(int element) - { - elements.Add(element); - } - - internal int Remove() - { - int last = elements.LastOrDefault(); - elements.RemoveAt(elements.Count - 1); - return last; - } -} diff --git a/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs b/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs deleted file mode 100644 index 83ac869..0000000 --- a/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.ObjectModel; -using System.Security.Cryptography; - -using Grpc.Core; -using Grpc.Core.Interceptors; - -using Microsoft.Extensions.Logging; - -namespace FrostFS.SDK.Client; - -[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "")] -public struct WrapperPrm -{ - internal ILogger? Logger { get; set; } - - internal string Address { get; set; } - - internal ECDsa Key { get; set; } - - internal ulong DialTimeout { get; set; } - - internal ulong StreamTimeout { get; set; } - - internal uint ErrorThreshold { get; set; } - - internal Action ResponseInfoCallback { get; set; } - - internal Action PoolRequestInfoCallback { get; set; } - - internal Func GrpcChannelFactory { get; set; } - - internal ulong GracefulCloseOnSwitchTimeout { get; set; } - - internal Action? Callback { get; set; } - - internal Collection? Interceptors { get; set; } -} - diff --git a/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs index 8ad35d5..ebd60de 100644 --- a/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs @@ -20,8 +20,6 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor { var binary = RuleSerializer.Serialize(args.Chain); - var base64 = Convert.ToBase64String(binary); - AddChainRequest request = new() { Body = new() diff --git a/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs index 59783cf..c0f8b1a 100644 --- a/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Security.Cryptography; using System.Threading.Tasks; using FrostFS.Container; @@ -83,7 +82,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService Body = new PutRequest.Types.Body { Container = grpcContainer, - Signature = ClientContext.Key.ECDsaKey.SignRFC6979(grpcContainer) + Signature = ClientContext.Key.SignRFC6979(grpcContainer) } }; @@ -113,8 +112,8 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService { Body = new DeleteRequest.Types.Body { - ContainerId = args.ContainerId.ToMessage(), - Signature = ClientContext.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value) + ContainerId = args.ContainerId.GetContainerID(), + Signature = ClientContext.Key.SignRFC6979(args.ContainerId.GetContainerID().Value) } }; diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index 3ce543d..ea3b727 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -96,7 +96,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() } } @@ -124,7 +124,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() }, Range = new Object.Range @@ -159,7 +159,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() }, Type = ChecksumType.Sha256, @@ -204,7 +204,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() } } @@ -231,7 +231,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Body = new SearchRequest.Types.Body { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), Version = 1 // TODO: clarify this param } }; @@ -296,18 +296,17 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { var chunkSize = args.MaxChunkLength; Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null"); - + var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken); byte[]? chunkBuffer = null; try { - // common chunkBuffer = ArrayPool.Shared.Rent(chunkSize); bool isFirstChunk = true; ulong currentPos = args.Range.Offset; - + var address = new Address { ObjectId = args.Address.ObjectId, @@ -327,11 +326,11 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Body = new() { - Address = address, + Address = address, Patch = new PatchRequest.Types.Body.Types.Patch { Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), - SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } + SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount } } } }; @@ -385,7 +384,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return response.Body.ObjectId.ToModel(); } - + internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { var stream = args.Payload!; @@ -567,7 +566,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl break; sentBytes += bytesCount; - + var chunkRequest = new PutRequest { Body = new PutRequest.Types.Body @@ -622,7 +621,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var initRequest = new PutRequest { Body = new PutRequest.Types.Body - { + { Init = new PutRequest.Types.Body.Types.Init { Header = grpcHeader, diff --git a/src/FrostFS.SDK.Client/Pool/SessionCache.cs b/src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs similarity index 72% rename from src/FrostFS.SDK.Client/Pool/SessionCache.cs rename to src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs index fae3de1..a02436f 100644 --- a/src/FrostFS.SDK.Client/Pool/SessionCache.cs +++ b/src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Concurrent; +using System.Collections.Concurrent; namespace FrostFS.SDK.Client; @@ -36,15 +35,4 @@ internal sealed class SessionCache(ulong sessionExpirationDuration) _cache[key] = value; } } - - internal void DeleteByPrefix(string prefix) - { - foreach (var key in _cache.Keys) - { - if (key.StartsWith(prefix, StringComparison.Ordinal)) - { - _cache.TryRemove(key, out var _); - } - } - } } diff --git a/src/FrostFS.SDK.Client/Tools/ClientContext.cs b/src/FrostFS.SDK.Client/Tools/ClientContext.cs index e2522c9..104cb3d 100644 --- a/src/FrostFS.SDK.Client/Tools/ClientContext.cs +++ b/src/FrostFS.SDK.Client/Tools/ClientContext.cs @@ -39,7 +39,7 @@ public class ClientContext(FrostFSClient client, ClientKey key, FrostFsOwner own { if (sessionKey == null && Key != null && Address != null) { - sessionKey = Pool.FormCacheKey(Address, Key.PublicKey); + sessionKey = $"{Address}{Key}"; } return sessionKey; diff --git a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs index ac5d019..c8c5caf 100644 --- a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs +++ b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs @@ -1,6 +1,6 @@ using System; using System.Linq; - +using System.Security.Cryptography; using FrostFS.Object; using FrostFS.Refs; using FrostFS.SDK.Client.Mappers.GRPC; @@ -44,7 +44,11 @@ public static class ObjectTools if (header.Split != null) SetSplitValues(grpcHeader, header.Split, owner, version, key); - return new ObjectID { Value = grpcHeader.Sha256() }.ToModel(); + using var sha256 = SHA256.Create(); + using HashStream stream = new(sha256); + grpcHeader.WriteTo(stream); + + return new FrostFsObjectId(Base58.Encode(stream.Hash())); } internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx) @@ -75,7 +79,7 @@ public static class ObjectTools var obj = new Object.Object { Header = grpcHeader, - ObjectId = new ObjectID { Value = grpcHeader.Sha256() }, + ObjectId = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcHeader.Sha256()) }, Payload = UnsafeByteOperations.UnsafeWrap(@object.SingleObjectPayload) }; @@ -117,9 +121,9 @@ public static class ObjectTools if (split.ParentHeader is not null) { - var grpcParentHeader = CreateHeader(split.ParentHeader, Array.Empty().Sha256(), owner, version); + var grpcParentHeader = CreateHeader(split.ParentHeader, DataHasher.Sha256([]), owner, version); - grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() }; + grpcHeader.Split.Parent = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcParentHeader.Sha256()) }; grpcHeader.Split.ParentHeader = grpcParentHeader; grpcHeader.Split.ParentSignature = new Signature { @@ -147,21 +151,12 @@ public static class ObjectTools return grpcHeader; } - internal static Checksum Sha256Checksum(byte[] data) - { - return new Checksum - { - Type = ChecksumType.Sha256, - Sum = ByteString.CopyFrom(data.Sha256()) - }; - } - internal static Checksum Sha256Checksum(ReadOnlyMemory data) { return new Checksum { Type = ChecksumType.Sha256, - Sum = ByteString.CopyFrom(data.Sha256()) + Sum = UnsafeByteOperations.UnsafeWrap(DataHasher.Sha256(data)) }; } diff --git a/src/FrostFS.SDK.Client/Tools/RequestSigner.cs b/src/FrostFS.SDK.Client/Tools/RequestSigner.cs index e2aceef..8df1b7c 100644 --- a/src/FrostFS.SDK.Client/Tools/RequestSigner.cs +++ b/src/FrostFS.SDK.Client/Tools/RequestSigner.cs @@ -33,6 +33,7 @@ public static class RequestSigner var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N); var privateKey = new ECPrivateKeyParameters(new BigInteger(1, key.PrivateKey()), ecParameters); var signer = new ECDsaSigner(new HMacDsaKCalculator(digest)); + var hash = new byte[digest.GetDigestSize()]; digest.BlockUpdate(data, 0, data.Length); @@ -54,21 +55,21 @@ public static class RequestSigner return ByteString.CopyFrom(signature); } - internal static SignatureRFC6979 SignRFC6979(this ECDsa key, IMessage message) + internal static SignatureRFC6979 SignRFC6979(this ClientKey key, IMessage message) { return new SignatureRFC6979 { - Key = ByteString.CopyFrom(key.PublicKey()), - Sign = key.SignRFC6979(message.ToByteArray()), + Key = key.PublicKeyProto, + Sign = key.ECDsaKey.SignRFC6979(message.ToByteArray()), }; } - internal static SignatureRFC6979 SignRFC6979(this ECDsa key, ByteString data) + internal static SignatureRFC6979 SignRFC6979(this ClientKey key, ByteString data) { return new SignatureRFC6979 { - Key = ByteString.CopyFrom(key.PublicKey()), - Sign = key.SignRFC6979(data.ToByteArray()), + Key = key.PublicKeyProto, + Sign = key.ECDsaKey.SignRFC6979(data.ToByteArray()), }; } @@ -82,11 +83,11 @@ public static class RequestSigner Span result = stackalloc byte[65]; result[0] = 0x04; - key.SignHash(data.Sha512()).AsSpan().CopyTo(result.Slice(1)); + key.SignHash(DataHasher.Sha512(data)).AsSpan().CopyTo(result.Slice(1)); return ByteString.CopyFrom(result); } - + public static ByteString SignDataByHash(this ECDsa key, byte[] hash) { if (key is null) @@ -112,8 +113,9 @@ public static class RequestSigner Sign = key.ECDsaKey.SignData(ReadOnlyMemory.Empty), }; } - - using HashStream stream = new(); + + using var sha512 = SHA512.Create(); + using HashStream stream = new(sha512); data.WriteTo(stream); var sig = new Signature diff --git a/src/FrostFS.SDK.Client/Tools/Verifier.cs b/src/FrostFS.SDK.Client/Tools/Verifier.cs index c110431..90eb3ce 100644 --- a/src/FrostFS.SDK.Client/Tools/Verifier.cs +++ b/src/FrostFS.SDK.Client/Tools/Verifier.cs @@ -9,61 +9,13 @@ using FrostFS.Session; using Google.Protobuf; -using Org.BouncyCastle.Asn1.Sec; -using Org.BouncyCastle.Crypto.Digests; -using Org.BouncyCastle.Crypto.Parameters; -using Org.BouncyCastle.Crypto.Signers; -using Org.BouncyCastle.Math; - namespace FrostFS.SDK.Client; public static class Verifier { public const int RFC6979SignatureSize = 64; - private static BigInteger[] DecodeSignature(byte[] sig) - { - if (sig.Length != RFC6979SignatureSize) - throw new FormatException($"Wrong signature size, expect={RFC6979SignatureSize}, actual={sig.Length}"); - - var rs = new BigInteger[2]; - rs[0] = new BigInteger(1, sig.AsSpan(0, 32).ToArray()); - rs[1] = new BigInteger(1, sig.AsSpan(32).ToArray()); - - return rs; - } - - public static bool VerifyRFC6979(this byte[] publicKey, byte[] data, byte[] sig) - { - if (publicKey is null || data is null || sig is null) - return false; - - var rs = DecodeSignature(sig); - var digest = new Sha256Digest(); - var signer = new ECDsaSigner(new HMacDsaKCalculator(digest)); - var secp256R1 = SecNamedCurves.GetByName("secp256r1"); - var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N); - var bcPublicKey = new ECPublicKeyParameters(secp256R1.Curve.DecodePoint(publicKey), ecParameters); - var hash = new byte[digest.GetDigestSize()]; - - digest.BlockUpdate(data, 0, data.Length); - digest.DoFinal(hash, 0); - signer.Init(false, bcPublicKey); - - return signer.VerifySignature(hash, rs[0], rs[1]); - } - - public static bool VerifyRFC6979(this SignatureRFC6979 signature, IMessage message) - { - if (signature is null) - { - throw new ArgumentNullException(nameof(signature)); - } - - return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray()); - } - - public static bool VerifyData(this ECDsa key, ReadOnlyMemory data, byte[] sig) + public static bool VerifyData(this ECDsa key, IMessage data, ByteString sig) { if (key is null) throw new ArgumentNullException(nameof(key)); @@ -71,7 +23,18 @@ public static class Verifier if (sig is null) throw new ArgumentNullException(nameof(sig)); - return key.VerifyHash(data.Sha512(), sig.AsSpan(1).ToArray()); + var signature = sig.Span.Slice(1).ToArray(); + using var sha = SHA512.Create(); + + if (data is null) + { + return key.VerifyHash(DataHasher.Sha512(new Span([])), signature); + } + + using var stream = new HashStream(sha); + data.WriteTo(stream); + + return key.VerifyHash(stream.Hash(), signature); } public static bool VerifyMessagePart(this Signature sig, IMessage data) @@ -80,9 +43,8 @@ public static class Verifier return false; using var key = sig.Key.ToByteArray().LoadPublicKey(); - var data2Verify = data is null ? [] : data.ToByteArray(); - return key.VerifyData(data2Verify, sig.Sign.ToByteArray()); + return key.VerifyData(data, sig.Sign); } internal static bool VerifyMatryoskaLevel(IMessage body, IMetaHeader meta, IVerificationHeader verification) diff --git a/src/FrostFS.SDK.Cryptography/ArrayHelper.cs b/src/FrostFS.SDK.Cryptography/ArrayHelper.cs index 00c356d..c88134d 100644 --- a/src/FrostFS.SDK.Cryptography/ArrayHelper.cs +++ b/src/FrostFS.SDK.Cryptography/ArrayHelper.cs @@ -21,4 +21,18 @@ internal static class ArrayHelper return dst; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void GetRevertedArray(ReadOnlySpan source, byte[] data) + { + if (source.Length != 0) + { + int i = 0; + int j = source.Length - 1; + while (i < source.Length) + { + data[i++] = source[j--]; + } + } + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index c03d604..9f33f25 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Cryptography")] -[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyFileVersion("1.0.4.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] -[assembly: AssemblyVersion("1.0.3.0")] +[assembly: AssemblyVersion("1.0.4.0")] diff --git a/src/FrostFS.SDK.Cryptography/Base58.cs b/src/FrostFS.SDK.Cryptography/Base58.cs index 636eb0a..0026bc3 100644 --- a/src/FrostFS.SDK.Cryptography/Base58.cs +++ b/src/FrostFS.SDK.Cryptography/Base58.cs @@ -19,19 +19,20 @@ public static class Base58 if (buffer.Length < 4) throw new FormatException(); - var check = buffer.AsSpan(0, buffer.Length - 4).ToArray(); - byte[] checksum = check.Sha256().Sha256(); + var check = buffer.AsSpan(0, buffer.Length - 4); + byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(check).AsSpan()); if (!buffer.AsSpan(buffer.Length - 4).SequenceEqual(checksum.AsSpan(0, 4))) throw new FormatException(); + var result = check.ToArray(); Array.Clear(buffer, 0, buffer.Length); - return check; + return result; } - public static string Base58CheckEncode(this ReadOnlySpan data) + public static string Base58CheckEncode(this Span data) { - byte[] checksum = data.ToArray().Sha256().Sha256(); + byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan()); ; Span buffer = stackalloc byte[data.Length + 4]; data.CopyTo(buffer); @@ -59,10 +60,9 @@ public static class Base58 } int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count(); - var leadingZeros = new byte[leadingZeroCount]; if (bi.IsZero) - return leadingZeros; + return new byte[leadingZeroCount]; var bytesBigEndian = bi.ToByteArray().Reverse().ToArray(); @@ -73,12 +73,26 @@ public static class Base58 var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray(); - return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros); + var result = new byte[leadingZeroCount + bytesBigEndian.Length - firstNonZeroIndex]; + + int p = 0; + while (p < leadingZeroCount) + result[p++] = 0; + + for (int j = firstNonZeroIndex; j < bytesBigEndian.Length; j++) + result[p++] = bytesBigEndian[j]; + + return result; } public static string Encode(ReadOnlySpan input) { - var data = input.ToArray().Reverse().Concat(new byte[] { 0 }).ToArray(); + var data = new byte[input.Length + 1]; + + ArrayHelper.GetRevertedArray(input, data); + + data[input.Length] = 0; + BigInteger value = new(data); // Encode BigInteger to Base58 string diff --git a/src/FrostFS.SDK.Cryptography/DataHasher.cs b/src/FrostFS.SDK.Cryptography/DataHasher.cs new file mode 100644 index 0000000..fceeeaa --- /dev/null +++ b/src/FrostFS.SDK.Cryptography/DataHasher.cs @@ -0,0 +1,121 @@ +using System; +using System.Buffers; +using System.Security.Cryptography; + +namespace FrostFS.SDK.Cryptography; + +public static class DataHasher +{ + private const int LargeBlockSize = 1024 * 1024; + private const int SmallBlockSize = 256; + + public static byte[] Hash(this ReadOnlyMemory bytes, HashAlgorithm algorithm) + { + if (algorithm is null) + { + throw new ArgumentNullException(nameof(algorithm)); + } + + if (bytes.Length == 0) + { + return algorithm.ComputeHash([]); + } + + int rest, pos = 0; + + var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize; + + byte[] buffer = ArrayPool.Shared.Rent(blockSize); + + try + { + while ((rest = bytes.Length - pos) > 0) + { + var size = Math.Min(rest, blockSize); + + bytes.Slice(pos, size).CopyTo(buffer); + + algorithm.TransformBlock(buffer, 0, size, buffer, 0); + + pos += size; + } + + algorithm.TransformFinalBlock([], 0, 0); + return algorithm.Hash; + } + finally + { + if (buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + public static byte[] Hash(this ReadOnlySpan bytes, HashAlgorithm algorithm) + { + if (algorithm is null) + { + throw new ArgumentNullException(nameof(algorithm)); + } + + if (bytes.Length == 0) + { + return algorithm.ComputeHash([]); + } + + int rest, pos = 0; + + var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize; + + byte[] buffer = ArrayPool.Shared.Rent(blockSize); + + try + { + while ((rest = bytes.Length - pos) > 0) + { + var size = Math.Min(rest, blockSize); + + bytes.Slice(pos, size).CopyTo(buffer); + + algorithm.TransformBlock(buffer, 0, size, buffer, 0); + + pos += size; + } + + algorithm.TransformFinalBlock([], 0, 0); + return algorithm.Hash; + } + finally + { + if (buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + public static byte[] Sha256(ReadOnlyMemory value) + { + using SHA256 sha = SHA256.Create(); + return Hash(value, sha); + } + + public static byte[] Sha256(ReadOnlySpan value) + { + using SHA256 sha = SHA256.Create(); + return Hash(value, sha); + } + + public static byte[] Sha512(ReadOnlyMemory value) + { + using SHA512 sha = SHA512.Create(); + return Hash(value, sha); + } + + public static byte[] Sha512(ReadOnlySpan value) + { + using SHA512 sha = SHA512.Create(); + return Hash(value, sha); + } +} diff --git a/src/FrostFS.SDK.Cryptography/Extentions.cs b/src/FrostFS.SDK.Cryptography/Extentions.cs index 40ad6f0..5f4fdd9 100644 --- a/src/FrostFS.SDK.Cryptography/Extentions.cs +++ b/src/FrostFS.SDK.Cryptography/Extentions.cs @@ -1,20 +1,9 @@ -using System; -using System.IO; -using System.Security.Cryptography; -using System.Threading; -using CommunityToolkit.HighPerformance; using Org.BouncyCastle.Crypto.Digests; namespace FrostFS.SDK.Cryptography; public static class Extentions { - private static readonly SHA256 _sha256 = SHA256.Create(); - private static SpinLock _spinlockSha256; - - private static readonly SHA512 _sha512 = SHA512.Create(); - private static SpinLock _spinlockSha512; - internal static byte[] RIPEMD160(this byte[] value) { var hash = new byte[20]; @@ -24,72 +13,4 @@ public static class Extentions digest.DoFinal(hash, 0); return hash; } - - public static byte[] Sha256(this byte[] value) - { - bool lockTaken = false; - try - { - _spinlockSha256.Enter(ref lockTaken); - - return _sha256.ComputeHash(value); - } - finally - { - if (lockTaken) - { - _spinlockSha256.Exit(false); - } - } - } - - public static byte[] Sha256(this ReadOnlyMemory value) - { - bool lockTaken = false; - try - { - _spinlockSha256.Enter(ref lockTaken); - - return _sha256.ComputeHash(value.AsStream()); - } - finally - { - if (lockTaken) - { - _spinlockSha256.Exit(false); - } - } - } - - public static byte[] Sha512(this ReadOnlyMemory value) - { - bool lockTaken = false; - try - { - _spinlockSha512.Enter(ref lockTaken); - - return _sha512.ComputeHash(value.AsStream()); - } - finally - { - if (lockTaken) - _spinlockSha512.Exit(false); - } - } - - public static byte[] Sha512(this Stream stream) - { - bool lockTaken = false; - try - { - _spinlockSha512.Enter(ref lockTaken); - - return _sha512.ComputeHash(stream); - } - finally - { - if (lockTaken) - _spinlockSha512.Exit(false); - } - } } diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index e4c7824..9dfd370 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -30,7 +30,6 @@ - all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/FrostFS.SDK.Cryptography/HashStream.cs b/src/FrostFS.SDK.Cryptography/HashStream.cs index d2ede1a..2893f26 100644 --- a/src/FrostFS.SDK.Cryptography/HashStream.cs +++ b/src/FrostFS.SDK.Cryptography/HashStream.cs @@ -3,11 +3,11 @@ using System.Security.Cryptography; namespace FrostFS.SDK.Cryptography; -public sealed class HashStream() : Stream +public sealed class HashStream(HashAlgorithm algorithm) : Stream { private long position; - private readonly SHA512 _hash = SHA512.Create(); + private readonly HashAlgorithm _hash = algorithm; public override bool CanRead => false; diff --git a/src/FrostFS.SDK.Cryptography/Key.cs b/src/FrostFS.SDK.Cryptography/Key.cs index 072f155..5f382fa 100644 --- a/src/FrostFS.SDK.Cryptography/Key.cs +++ b/src/FrostFS.SDK.Cryptography/Key.cs @@ -16,7 +16,7 @@ public static class KeyExtension private const int UncompressedPublicKeyLength = 65; private static readonly uint CheckSigDescriptor = - BinaryPrimitives.ReadUInt32LittleEndian(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").Sha256()); + BinaryPrimitives.ReadUInt32LittleEndian(DataHasher.Sha256(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").AsSpan())); public static byte[] Compress(this byte[] publicKey) { @@ -60,10 +60,15 @@ public static class KeyExtension $"expected length={CompressedPublicKeyLength}, actual={publicKey.Length}" ); - var script = new byte[] { 0x0c, CompressedPublicKeyLength }; //PUSHDATA1 33 - script = ArrayHelper.Concat(script, publicKey); - script = ArrayHelper.Concat(script, [0x41]); //SYSCALL - script = ArrayHelper.Concat(script, BitConverter.GetBytes(CheckSigDescriptor)); //Neo_Crypto_CheckSig + var signDescriptor = BitConverter.GetBytes(CheckSigDescriptor); + + var script = new byte[3 + publicKey.Length + signDescriptor.Length]; + + script[0] = 0x0c; + script[1] = CompressedPublicKeyLength; //PUSHDATA1 33 + Buffer.BlockCopy(publicKey, 0, script, 2, publicKey.Length); + script[publicKey.Length + 2] = 0x41; //SYSCALL + Buffer.BlockCopy(signDescriptor, 0, script, publicKey.Length + 3, signDescriptor.Length); return script; } @@ -74,7 +79,7 @@ public static class KeyExtension throw new ArgumentNullException(nameof(publicKey)); var script = publicKey.CreateSignatureRedeemScript(); - return script.Sha256().RIPEMD160(); + return DataHasher.Sha256(script.AsSpan()).RIPEMD160(); } private static string ToAddress(this byte[] scriptHash, byte version) @@ -102,11 +107,6 @@ public static class KeyExtension return privateKey; } - public static string Address(this ECDsa key) - { - return key.PublicKey().PublicKeyToAddress(); - } - public static string PublicKeyToAddress(this byte[] publicKey) { if (publicKey == null) @@ -132,10 +132,12 @@ public static class KeyExtension var pos = 33 - param.Q.X.Length; param.Q.X.CopyTo(pubkey, pos); - if (new BigInteger(param.Q.Y.Reverse().Concat(new byte[] { 0x0 }).ToArray()).IsEven) - pubkey[0] = 0x2; - else - pubkey[0] = 0x3; + + var y = new byte[33]; + ArrayHelper.GetRevertedArray(param.Q.Y, y); + y[32] = 0; + + pubkey[0] = new BigInteger(y).IsEven ? (byte)0x2 : (byte)0x3; return pubkey; } @@ -152,7 +154,9 @@ public static class KeyExtension { var secp256R1 = SecNamedCurves.GetByName("secp256r1"); var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)) - .GetEncoded(false).Skip(1).ToArray(); + .GetEncoded(false) + .Skip(1) + .ToArray(); var key = ECDsa.Create(new ECParameters { diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index 11ace79..c3c813c 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Protos")] -[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyFileVersion("1.0.4.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] -[assembly: AssemblyVersion("1.0.3.0")] +[assembly: AssemblyVersion("1.0.4.0")] diff --git a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs index 0a3aec6..58a9d40 100644 --- a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs @@ -3,7 +3,6 @@ using System.Security.Cryptography; using FrostFS.Object; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; using FrostFS.Session; using Google.Protobuf; @@ -43,7 +42,7 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader) Signature = new Refs.Signature { Key = Key.PublicKeyProto, - Sign = Key.ECDsaKey. SignData(header.ToByteArray()), + Sign = Key.ECDsaKey.SignData(header.ToByteArray()), } } }, diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs index e6f25c2..eafdda5 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs @@ -1,5 +1,3 @@ -using System.Security.Cryptography; - using FrostFS.Container; using FrostFS.Object; using FrostFS.SDK.Client; diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs index 0ad6ab0..7491950 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs @@ -23,13 +23,16 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) var grpcVersion = Version.ToMessage(); + Span ContainerGuidSpan = stackalloc byte[16]; + ContainerGuid.ToBytes(ContainerGuidSpan); + PutResponse putResponse = new() { Body = new PutResponse.Types.Body { ContainerId = new ContainerID { - Value = ByteString.CopyFrom(ContainerGuid.ToBytes()) + Value = ByteString.CopyFrom(ContainerGuidSpan) } }, MetaHeader = new ResponseMetaHeader @@ -69,7 +72,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) Container = new Container.Container { Version = grpcVersion, - Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()), + Nonce = ByteString.CopyFrom(ContainerGuidSpan), PlacementPolicy = PlacementPolicy.GetPolicy() } }, diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index 49a0f6d..f33b9a1 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -4,7 +4,6 @@ using System.Security.Cryptography; using FrostFS.Object; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; using Google.Protobuf; diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs index 37a8c13..04a8b2c 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs @@ -40,9 +40,9 @@ public class ContainerTests : SmokeTestsBase { var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); await Cleanup(client); - + client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - + FrostFsContainerId containerId = await CreateContainer(client, ctx: default, token: null, @@ -50,22 +50,25 @@ public class ContainerTests : SmokeTestsBase backupFactor: 1, selectors: [], filter: [], - containerAttributes: [new ("testKey1", "testValue1")], + containerAttributes: [new("testKey1", "testValue1")], new FrostFsReplica(3)); Assert.NotNull(containerId); var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default); - + Assert.NotNull(container); Assert.NotNull(container.Attributes); - Assert.Equal(2, container.Attributes.Count); Assert.Equal("testKey1", container.Attributes[0].Key); Assert.Equal("testValue1", container.Attributes[0].Value); - Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key); - Assert.Equal("true", container.Attributes[1].Value); - + + //Assert.Equal("true", container.Attributes[1].Value); + + // for cluster + //Assert.Equal(2, container.Attributes.Count); + //Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key); + Assert.True(container.PlacementPolicy.HasValue); Assert.Equal(1u, container.PlacementPolicy.Value.BackupFactor); @@ -82,7 +85,7 @@ public class ContainerTests : SmokeTestsBase Assert.Equal(OwnerId!.ToString(), container.Owner!.Value); Assert.NotNull(container.Version); - + Assert.Equal(Version!.Major, container.Version.Major); Assert.Equal(Version.Minor, container.Version.Minor); } @@ -101,8 +104,8 @@ public class ContainerTests : SmokeTestsBase new ("filter2", "filterKey2", 2, "testValue2", [new ("subFilter2", "subFilterKey2", 3, "testValue3",[])]) ]; - Collection selectors = [ - new ("selector1") { + Collection selectors = [ + new ("selector1") { Count = 1, Clause = 1, Attribute = "attribute1", @@ -129,19 +132,19 @@ public class ContainerTests : SmokeTestsBase Assert.NotNull(containerId); var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default); - + Assert.NotNull(container); Assert.NotNull(container.Attributes); - Assert.Single(container.Attributes); - Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key); - Assert.Equal("true", container.Attributes[0].Value); + //Assert.Single(container.Attributes); + //Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key); + //Assert.Equal("true", container.Attributes[0].Value); Assert.True(container.PlacementPolicy.HasValue); Assert.Equal(2u, container.PlacementPolicy.Value.BackupFactor); Assert.False(container.PlacementPolicy.Value.Unique); - + Assert.NotEmpty(container.PlacementPolicy.Value.Selectors); Assert.Equal(2, container.PlacementPolicy.Value.Selectors.Count); @@ -196,7 +199,7 @@ public class ContainerTests : SmokeTestsBase Assert.Equal(OwnerId!.ToString(), container.Owner!.Value); Assert.NotNull(container.Version); - + Assert.Equal(Version!.Major, container.Version.Major); Assert.Equal(Version.Minor, container.Version.Minor); } @@ -233,12 +236,12 @@ public class ContainerTests : SmokeTestsBase }); } - #pragma warning disable xUnit1031 // Timeout is used +#pragma warning disable xUnit1031 // Timeout is used if (!Task.WaitAll(tasks, 20000)) { Assert.Fail("cannot create containers"); } - #pragma warning restore xUnit1031 +#pragma warning restore xUnit1031 var containers = client.ListContainersAsync(new PrmContainerGetAll(), default); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs index f1fce3c..a3d25af 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs @@ -32,8 +32,10 @@ public class SmokeClientTests : SmokeTestsBase Assert.Equal(13, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); - Assert.Equal(2, result.Addresses.Count); - Assert.Equal(11, result.Attributes.Count); + Assert.Single(result.Addresses); + Assert.Equal(9, result.Attributes.Count); + //Assert.Equal(2, result.Addresses.Count); + //Assert.Equal(11, result.Attributes.Count); } [Fact] @@ -65,13 +67,13 @@ public class SmokeClientTests : SmokeTestsBase Assert.True(result.HomomorphicHashingDisabled); Assert.True(result.MaintenanceModeAllowed); Assert.True(0u < result.MagicNumber); - + Assert.Equal(0u, result.AuditFee); Assert.Equal(0u, result.BasicIncomeRate); Assert.Equal(0u, result.ContainerAliasFee); Assert.Equal(0u, result.ContainerFee); Assert.Equal(75u, result.EpochDuration); - Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee); + Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee); Assert.Equal(12u, result.MaxECDataCount); Assert.Equal(4u, result.MaxECParityCount); Assert.Equal(5242880u, result.MaxObjectSize); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs index 03a9169..a0f501f 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs @@ -58,7 +58,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase int[] objectSizes = [1, 257, 5 * 1024 * 1024, 20 * 1024 * 1024]; string[] objectTypes = [clientCut, serverCut, singleObject]; - + foreach (var objectSize in objectSizes) { _testOutputHelper.WriteLine($"test set for object size {objectSize}"); @@ -157,7 +157,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.NotNull(x); Assert.True(x.Length > 0); - // Assert.True(expectedHash.SequenceEqual(h.ToArray())); + // Assert.True(expectedHash.SequenceEqual(h.ToArray())); } } @@ -201,7 +201,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId) { - if (bytes.Length < 1024 + 64 || bytes.Length > 5900) + if (bytes.Length < 1024 + 64 || bytes.Length > 5900) return; var patch = new byte[1024]; diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs deleted file mode 100644 index 831f37e..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Smoke; - -public abstract class MultiThreadTestsBase -{ - private TestNodeInfo[] nodes; - - protected CallContext? Ctx { get; } - - protected MultiThreadTestsBase() - { - nodes = new TestNodeInfo[4]; - - nodes[0] = new(new Uri(""), ""); - nodes[1] = new(new Uri(""), ""); - nodes[2] = new(new Uri(""), ""); - nodes[3] = new(new Uri(""), ""); - } -} - -public class TestNodeInfo -{ - internal Uri Uri; - - protected ECDsa? Key { get; } - - protected FrostFsOwner? OwnerId { get; } - - protected FrostFsVersion? Version { get; } - - public TestNodeInfo(Uri uri, string keyString) - { - Uri = uri; - Key = keyString.LoadWif(); - OwnerId = FrostFsOwner.FromKey(Key); - Version = new FrostFsVersion(2, 13); - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs deleted file mode 100644 index 194ef98..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs +++ /dev/null @@ -1,544 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Smoke; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] -public class MultithreadPoolSmokeTests : SmokeTestsBase -{ - private InitParameters GetDefaultParams() - { - return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [new(1, url, 100.0f)], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null - }; - } - - [Fact] - public async void NetworkMapTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNetmapSnapshotAsync(default); - - Assert.True(result.Epoch > 0); - Assert.Single(result.NodeInfoCollection); - - var item = result.NodeInfoCollection[0]; - Assert.Equal(2, item.Version.Major); - Assert.Equal(13, item.Version.Minor); - Assert.Equal(NodeState.Online, item.State); - Assert.True(item.PublicKey.Length > 0); - Assert.Single(item.Addresses); - Assert.Equal(9, item.Attributes.Count); - } - - [Fact] - public async void NodeInfoTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } - - [Fact] - public async void NodeInfoStatisticsTwoNodesTest() - { - var callbackText = string.Empty; - - var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [ - new(1, url, 100.0f), - new(2, url.Replace('0', '1'), 100.0f) - ], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null, - Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds" - }; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - var statistics = pool.Statistic(); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void NodeInfoStatisticsTest() - { - var options = GetDefaultParams(); - - var callbackText = string.Empty; - options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void GetSessionTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var prm = new PrmSessionCreate(100); - - var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true); - - var ownerHash = Base58.Decode(OwnerId!.Value); - - Assert.NotNull(token); - Assert.NotEqual(Guid.Empty, token.Id); - Assert.Equal(33, token.SessionKey.Length); - } - - [Fact] - public async void CreateObjectWithSessionToken() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["key1", "value1"]); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = GetRandomBytes(1024); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - } - - [Fact] - public async void FilterTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = new byte[] { 1, 2, 3 }; - - var ParentHeader = new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular) - { - PayloadLength = 3 - }; - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")], - new FrostFsSplit())); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId, false), default); - - var ecdsaKey = keyString.LoadWif(); - - var networkInfo = await pool.GetNetmapSnapshotAsync(default); - - await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); - - await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); - - await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); - - await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); - - await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); - - await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); - - await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); - - await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); - - var checkSum = CheckSum.CreateCheckSum(bytes); - - await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); - - await CheckFilter(pool, containerId, new FilterByPhysicallyStored()); - } - - private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter) - { - var resultObjectsCount = 0; - - PrmObjectSearch searchParam = new(containerId, null, [], filter); - - await foreach (var objId in pool.SearchObjectsAsync(searchParam, default)) - { - resultObjectsCount++; - var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); - } - - Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioTest(int objectSize) - { - bool callbackInvoked = false; - - var options = GetDefaultParams(); - - options.Callback = new((cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await pool.PutContainerAsync(createContainerParam, default); - - var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - Assert.True(callbackInvoked); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes!); - Assert.Equal("fileName", objHeader.Attributes!.First().Key); - Assert.Equal("test", objHeader.Attributes!.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioWithSessionTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0)); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - await Cleanup(pool); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams); - - var container = await pool.PutContainerAsync(createContainerParam, ctx); - - var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx); - Assert.NotNull(containerInfo); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: container, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB - [InlineData(64 * 1024 * 1024 - 1)] - [InlineData(64 * 1024 * 1024 + 1)] - [InlineData(2 * 64 * 1024 * 1024 + 256)] - [InlineData(200)] - public async void ClientCutScenarioTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var ctx = new CallContext(TimeSpan.FromSeconds(10)); - - // ctx.Interceptors.Add(new CallbackInterceptor()); - - var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx); - - Assert.NotNull(container); - - byte[] bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectClientCutPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - payload: new MemoryStream(bytes)); - - var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes[0].Key); - Assert.Equal("test", objHeader.Attributes[0].Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await CheckFilter(pool, containerId, new FilterByRootObject()); - - await Cleanup(pool); - - await foreach (var cid in pool.ListContainersAsync(default, default)) - { - Assert.Fail($"Container {cid.GetValue()} exist"); - } - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs deleted file mode 100644 index 7de8155..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs +++ /dev/null @@ -1,684 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Client.Interfaces; -using FrostFS.SDK.Cryptography; -using FrostFS.SDK.SmokeTests; - -namespace FrostFS.SDK.Tests.Smoke; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] -public class MultithreadSmokeClientTests : SmokeTestsBase -{ - [Fact] - public async void AccountTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var result = await client.GetBalanceAsync(default); - - Assert.NotNull(result); - Assert.True(result.Value == 0); - } - - [Fact] - public async void NetworkMapTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var result = await client.GetNetmapSnapshotAsync(default); - - Assert.True(result.Epoch > 0); - Assert.Single(result.NodeInfoCollection); - - var item = result.NodeInfoCollection[0]; - Assert.Equal(2, item.Version.Major); - Assert.Equal(13, item.Version.Minor); - Assert.Equal(NodeState.Online, item.State); - Assert.True(item.PublicKey.Length > 0); - Assert.Single(item.Addresses); - Assert.Equal(9, item.Attributes.Count); - } - - - [Fact] - public async void NodeInfoTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var result = await client.GetNodeInfoAsync(default); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } - - [Fact] - public async void NodeInfoStatisticsTest() - { - var options = ClientOptions; - - var callbackContent = string.Empty; - options.Value.Callback = (cs) => callbackContent = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - var result = await client.GetNodeInfoAsync(default); - - Assert.NotEmpty(callbackContent); - } - - [Fact] - public async void GetSessionTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var token = await client.CreateSessionAsync(new(100), default); - - Assert.NotNull(token); - Assert.NotEqual(Guid.Empty, token.Id); - Assert.Equal(33, token.SessionKey.Length); - } - - [Fact] - public async void CreateObjectWithSessionToken() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["key1", "value1"]); - - var containerId = await client.PutContainerAsync(createContainerParam, default); - - var bytes = GetRandomBytes(1024); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default) - .ConfigureAwait(true); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk().ConfigureAwait(true)) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client).ConfigureAwait(true); - } - - [Fact] - public async void FilterTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await client.PutContainerAsync(createContainerParam, default); - - var bytes = new byte[] { 1, 2, 3 }; - - var ParentHeader = new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular) - { - PayloadLength = 3 - }; - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")], - new FrostFsSplit())); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var head = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var ecdsaKey = keyString.LoadWif(); - - var networkInfo = await client.GetNetmapSnapshotAsync(default); - - await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); - - await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); - - await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); - - await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); - - await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); - - await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); - - await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); - - await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); - - var checkSum = CheckSum.CreateCheckSum(bytes); - - await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); - - await CheckFilter(client, containerId, new FilterByPhysicallyStored()); - } - - private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter) - { - var resultObjectsCount = 0; - - PrmObjectSearch searchParam = new(containerId, null, [], filter); - - await foreach (var objId in client.SearchObjectsAsync(searchParam, default)) - { - resultObjectsCount++; - var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); - } - - Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioTest(int objectSize) - { - bool callbackInvoked = false; - - var options = ClientOptions; - - options.Value.Callback = new((cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }); - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - await Cleanup(client); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, ctx); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - Assert.True(callbackInvoked); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) - { - hasObject = true; - - var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Fact] - public async void PatchTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, default); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - - var bytes = new byte[1024]; - for (int i = 0; i < 1024; i++) - { - bytes[i] = 31; - } - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var patch = new byte[16]; - for (int i = 0; i < 16; i++) - { - patch[i] = 32; - } - - var range = new FrostFsRange(8, (ulong)patch.Length); - - var patchParams = new PrmObjectPatch( - new FrostFsAddress(createdContainer, objectId), - payload: new MemoryStream(patch), - maxChunkLength: 32, - range: range); - - var newIbjId = await client.PatchObjectAsync(patchParams, default); - - var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - for (int i = 0; i < (int)range.Offset; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); - - var rangeEnd = range.Offset + range.Length; - - for (int i = (int)range.Offset; i < (int)rangeEnd; i++) - Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); - - for (int i = (int)rangeEnd; i < bytes.Length; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Fact] - public async void RangeTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, default); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - - var bytes = new byte[256]; - for (int i = 0; i < 256; i++) - { - bytes[i] = (byte)i; - } - - var param = new PrmObjectPut( - new FrostFsObjectHeader(containerId: createdContainer, type: FrostFsObjectType.Regular)); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64)); - - var rangeReader = await client.GetRangeAsync(rangeParam, default); - - var downloadedBytes = new byte[rangeParam.Range.Length]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await rangeReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Fact] - public async void RangeHashTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, default); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - - var bytes = new byte[256]; - for (int i = 0; i < 256; i++) - { - bytes[i] = (byte)i; - } - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular)); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [new FrostFsRange(100, 64)], bytes); - - var hashes = await client.GetRangeHashAsync(rangeParam, default); - - foreach (var hash in hashes) - { - var x = hash[..32].ToArray(); - } - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioWithSessionTest(int objectSize) - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - //Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0)) - var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - await Cleanup(client); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams); - - var container = await client.PutContainerAsync(createContainerParam, ctx); - - var containerInfo = await client.GetContainerAsync(new PrmContainerGet(container), ctx); - Assert.NotNull(containerInfo); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: container, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync( - new PrmObjectSearch(container, token, [], filter), default)) - { - hasObject = true; - - var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await client.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB - [InlineData(64 * 1024 * 1024 - 1)] - [InlineData(64 * 1024 * 1024 + 1)] - [InlineData(2 * 64 * 1024 * 1024 + 256)] - [InlineData(200)] - public async void ClientCutScenarioTest(int objectSize) - { - var options = ClientOptions; - options.Value.Interceptors.Add(new CallbackInterceptor()); - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await client.PutContainerAsync(createContainerParam, default); - - var ctx = new CallContext(TimeSpan.FromSeconds(10)); - - var container = await client.GetContainerAsync(new PrmContainerGet(containerId), ctx); - - Assert.NotNull(container); - - byte[] bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectClientCutPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - payload: new MemoryStream(bytes)); - - var objectId = await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) - { - hasObject = true; - - var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes[0].Key); - Assert.Equal("test", objHeader.Attributes[0].Value); - } - - Assert.True(hasObject); - - var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await CheckFilter(client, containerId, new FilterByRootObject()); - - await Cleanup(client); - - var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5)); - - IAsyncEnumerator? enumerator = null; - do - { - if (deadline <= DateTime.UtcNow) - { - Assert.Fail("Containers exist"); - break; - } - - enumerator = client.ListContainersAsync(default, default).GetAsyncEnumerator(); - await Task.Delay(500); - } - while (await enumerator!.MoveNextAsync()); - } - - [Fact] - public async void NodeInfoCallbackAndInterceptorTest() - { - bool callbackInvoked = false; - bool intercepterInvoked = false; - - var options = ClientOptions; - options.Value.Callback = (cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }; - - options.Value.Interceptors.Add(new CallbackInterceptor(s => intercepterInvoked = true)); - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - var result = await client.GetNodeInfoAsync(default); - - Assert.True(callbackInvoked); - Assert.True(intercepterInvoked); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs deleted file mode 100644 index 9e7ea80..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs +++ /dev/null @@ -1,546 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Smoke; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] -public class PoolSmokeTests : SmokeTestsBase -{ - private InitParameters GetDefaultParams() - { - return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [new(1, url, 100.0f)], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null - }; - } - - [Fact] - public async void NetworkMapTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNetmapSnapshotAsync(default); - - Assert.True(result.Epoch > 0); - Assert.Single(result.NodeInfoCollection); - - var item = result.NodeInfoCollection[0]; - Assert.Equal(2, item.Version.Major); - Assert.Equal(13, item.Version.Minor); - Assert.Equal(NodeState.Online, item.State); - Assert.True(item.PublicKey.Length > 0); - Assert.Single(item.Addresses); - Assert.Equal(9, item.Attributes.Count); - } - - [Fact] - public async void NodeInfoTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } - - [Fact] - public async void NodeInfoStatisticsTwoNodesTest() - { - var callbackText = string.Empty; - - var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [ - new(1, url, 100.0f), - new(2, url.Replace('0', '1'), 100.0f) - ], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null, - Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds" - }; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - var statistics = pool.Statistic(); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void NodeInfoStatisticsTest() - { - var options = GetDefaultParams(); - - var callbackText = string.Empty; - options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void GetSessionTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var prm = new PrmSessionCreate(100); - - var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true); - - var ownerHash = Base58.Decode(OwnerId!.Value); - - Assert.NotNull(token); - Assert.NotEqual(Guid.Empty, token.Id); - Assert.Equal(33, token.SessionKey.Length); - } - - [Fact] - public async void CreateObjectWithSessionToken() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["key1", "value1"]); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = GetRandomBytes(1024); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - } - - [Fact] - public async void FilterTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = new byte[] { 1, 2, 3 }; - - var ParentHeader = new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular) - { - PayloadLength = 3 - }; - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")], - new FrostFsSplit())); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var ecdsaKey = keyString.LoadWif(); - - var networkInfo = await pool.GetNetmapSnapshotAsync(default); - - await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); - - await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); - - await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); - - await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); - - await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); - - await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); - - await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); - - await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); - - var checkSum = CheckSum.CreateCheckSum(bytes); - - await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); - - await CheckFilter(pool, containerId, new FilterByPhysicallyStored()); - } - - private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter) - { - var resultObjectsCount = 0; - - PrmObjectSearch searchParam = new(containerId, null, [], filter); - - await foreach (var objId in pool.SearchObjectsAsync(searchParam, default)) - { - resultObjectsCount++; - var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); - } - - Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioTest(int objectSize) - { - bool callbackInvoked = false; - - var options = GetDefaultParams(); - - options.Callback = new((cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var ctx = new CallContext(TimeSpan.Zero); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await pool.PutContainerAsync(createContainerParam, ctx); - - var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - Assert.True(callbackInvoked); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes!); - Assert.Equal("fileName", objHeader.Attributes!.First().Key); - Assert.Equal("test", objHeader.Attributes!.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioWithSessionTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0)); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - await Cleanup(pool); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams); - - var container = await pool.PutContainerAsync(createContainerParam, ctx); - - var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx); - Assert.NotNull(containerInfo); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: container, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - var objs = pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default); - await foreach (var objId in objs) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB - [InlineData(64 * 1024 * 1024 - 1)] - [InlineData(64 * 1024 * 1024 + 1)] - [InlineData(2 * 64 * 1024 * 1024 + 256)] - [InlineData(200)] - public async void ClientCutScenarioTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var ctx = new CallContext(TimeSpan.FromSeconds(10)); - - // ctx.Interceptors.Add(new CallbackInterceptor()); - - var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx); - - Assert.NotNull(container); - - byte[] bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectClientCutPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - payload: new MemoryStream(bytes)); - - var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes[0].Key); - Assert.Equal("test", objHeader.Attributes[0].Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await CheckFilter(pool, containerId, new FilterByRootObject()); - - await Cleanup(pool); - - await foreach (var cid in pool.ListContainersAsync(default, default)) - { - Assert.Fail($"Container {cid.GetValue()} exist"); - } - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs b/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs index 59099a9..5e6998b 100644 --- a/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs @@ -16,19 +16,13 @@ namespace FrostFS.SDK.Tests.Smoke; [SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] public abstract class SmokeTestsBase { - // cluster Ori - // internal readonly string url = "http://10.78.128.207:8080"; - // internal readonly string keyString = "L4JWLdedUd4b21sriRHtCPGkjG2Mryz2AWLiVqTBSNyxxyAUcc7s"; - // cluster - internal readonly string url = "http://10.78.128.190:8080"; - internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba"; + // internal readonly string url = "http://10.78.128.190:8080"; + // internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba"; // WSL2 - // internal readonly string url = "http://172.29.238.97:8080"; - // internal readonly string keyString = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; // "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; - - //"KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; + internal readonly string url = "http://172.29.238.97:8080"; // "http://172.20.8.23:8080"; + internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; protected ECDsa? Key { get; } @@ -77,7 +71,7 @@ public abstract class SmokeTestsBase if (networkSettings.HomomorphicHashingDisabled) attributes.Add(new("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", "true")); - + var containerInfo = new FrostFsContainerInfo( new FrostFsPlacementPolicy(unique, backupFactor, selectors, filter, replicas), [.. attributes]); diff --git a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs index db1e193..cf4bdfa 100644 --- a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; - using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; using FrostFS.SDK.Cryptography; @@ -20,13 +19,16 @@ public class ContainerTest : ContainerTestsBase Assert.NotNull(result); Assert.NotNull(result.GetValue()); - Assert.True(Base58.Encode(Mocker.ContainerGuid.ToBytes()) == result.GetValue()); + + var bytes = Mocker.ContainerGuid.ToByteArray(true); + + Assert.True(Base58.Encode(new Span(bytes)) == result.GetValue()); } [Fact] public async void GetContainerTest() { - var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); + var cid = new FrostFsContainerId(Base58.Encode(new Span(Mocker.ContainerGuid.ToByteArray(true)))); var result = await GetClient().GetContainerAsync(new PrmContainerGet(cid), default); @@ -61,7 +63,7 @@ public class ContainerTest : ContainerTestsBase public async void DeleteContainerAsyncTest() { Mocker.ReturnContainerRemoved = true; - var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); + var cid = new FrostFsContainerId(Base58.Encode(new Span(Mocker.ContainerGuid.ToByteArray(true)))); await GetClient().DeleteContainerAsync(new PrmContainerDelete(cid, PrmWait.DefaultParams), default); diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs index 479c6ba..7a595e6 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs @@ -88,7 +88,7 @@ public class ObjectTest : ObjectTestsBase // PART1 Assert.Equal(blockSize, objects[0].Payload.Length); Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload)); - + Assert.NotNull(objects[0].Header.Split.SplitId); Assert.Null(objects[0].Header.Split.Previous); Assert.True(objects[0].Header.Attributes.Count == 0); @@ -104,7 +104,7 @@ public class ObjectTest : ObjectTestsBase // last part Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); - Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload)); + Assert.True(bytes.AsMemory(2 * blockSize).ToArray().SequenceEqual(objects[2].Payload)); Assert.NotNull(objects[3].Header.Split.Parent); Assert.NotNull(objects[3].Header.Split.ParentHeader); diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs index 18e3981..c3681ce 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs @@ -35,7 +35,7 @@ public abstract class ObjectTestsBase ContainerGuid = Guid.NewGuid() }; - ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); + ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToByteArray(true))); Mocker.ObjectHeader = new( ContainerId, diff --git a/src/FrostFS.SDK.Tests/Unit/SessionTests.cs b/src/FrostFS.SDK.Tests/Unit/SessionTests.cs index b2f1b78..be2e072 100644 --- a/src/FrostFS.SDK.Tests/Unit/SessionTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/SessionTests.cs @@ -2,7 +2,6 @@ using System.Diagnostics.CodeAnalysis; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; namespace FrostFS.SDK.Tests.Unit; @@ -38,7 +37,7 @@ public class SessionTest : SessionTestsBase Assert.NotNull(result); Assert.NotEqual(Guid.Empty, result.Id); - Assert.Equal(Mocker.SessionId, result.Id.ToBytes()); + Assert.Equal(Mocker.SessionId, result.Id.ToByteArray(true)); Assert.Equal(Mocker.SessionKey, result.SessionKey.ToArray()); //Assert.Equal(OwnerId.ToMessage(), result.Token.Body.OwnerId); -- 2.45.3