From 87fe8db674b1100711e4b9fb906656b8183d3ed5 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Wed, 26 Mar 2025 16:51:42 +0300 Subject: [PATCH 01/12] [#43] Client: Memory optimization Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/ApeRules/Actions.cs | 2 +- src/FrostFS.SDK.Client/ApeRules/Condition.cs | 2 +- src/FrostFS.SDK.Client/ApeRules/Resources.cs | 2 +- src/FrostFS.SDK.Client/AssemblyInfo.cs | 4 +- src/FrostFS.SDK.Client/Caches.cs | 9 - .../Exceptions/FrostFsResponseException.cs | 4 +- .../Extensions/FrostFsExtensions.cs | 72 +- .../FrostFS.SDK.Client.csproj | 2 +- src/FrostFS.SDK.Client/FrostFSClient.cs | 35 +- .../Interfaces/IFrostFSClient.cs | 2 - src/FrostFS.SDK.Client/Mappers/ContainerId.cs | 16 +- .../Mappers/Object/ObjectHeaderMapper.cs | 33 +- .../Mappers/Object/ObjectId.cs | 2 +- src/FrostFS.SDK.Client/Mappers/OwnerId.cs | 2 +- .../Mappers/Session/SessionMapper.cs | 28 - .../Mappers/SignatureMapper.cs | 4 +- .../Models/Containers/FrostFsContainerInfo.cs | 5 +- .../Models/Misc/CheckSum.cs | 2 +- .../Models/Netmap/Placement/Context.cs | 2 +- .../Models/Object/FrostFsObject.cs | 23 - .../Models/Object/FrostFsSplit.cs | 26 + .../Models/Object/SplitId.cs | 13 +- .../Models/Response/FrostFsResponseStatus.cs | 4 +- .../Parameters/PrmApeChainRemove.cs | 2 +- .../Pool/ClientStatusMonitor.cs | 163 ----- src/FrostFS.SDK.Client/Pool/ClientWrapper.cs | 135 ---- src/FrostFS.SDK.Client/Pool/HealthyStatus.cs | 18 - src/FrostFS.SDK.Client/Pool/IClientStatus.cs | 28 - src/FrostFS.SDK.Client/Pool/InitParameters.cs | 45 -- src/FrostFS.SDK.Client/Pool/InnerPool.cs | 47 -- src/FrostFS.SDK.Client/Pool/MethodIndex.cs | 24 - src/FrostFS.SDK.Client/Pool/MethodStatus.cs | 19 - src/FrostFS.SDK.Client/Pool/NodeParam.cs | 12 - src/FrostFS.SDK.Client/Pool/NodeStatistic.cs | 12 - src/FrostFS.SDK.Client/Pool/NodesParam.cs | 12 - src/FrostFS.SDK.Client/Pool/Pool.cs | 677 ----------------- .../Pool/RebalanceParameters.cs | 16 - src/FrostFS.SDK.Client/Pool/RequestInfo.cs | 14 - src/FrostFS.SDK.Client/Pool/Sampler.cs | 85 --- src/FrostFS.SDK.Client/Pool/Statistic.cs | 12 - src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs | 8 - src/FrostFS.SDK.Client/Pool/WorkList.cs | 26 - src/FrostFS.SDK.Client/Pool/WrapperPrm.cs | 39 - .../Services/ApeManagerServiceProvider.cs | 2 - .../Services/ContainerServiceProvider.cs | 7 +- .../Services/ObjectServiceProvider.cs | 25 +- .../{Pool => Services/Shared}/SessionCache.cs | 14 +- src/FrostFS.SDK.Client/Tools/ClientContext.cs | 2 +- src/FrostFS.SDK.Client/Tools/ObjectTools.cs | 25 +- src/FrostFS.SDK.Client/Tools/RequestSigner.cs | 22 +- src/FrostFS.SDK.Client/Tools/Verifier.cs | 66 +- src/FrostFS.SDK.Cryptography/ArrayHelper.cs | 14 + src/FrostFS.SDK.Cryptography/AssemblyInfo.cs | 4 +- src/FrostFS.SDK.Cryptography/Base58.cs | 32 +- src/FrostFS.SDK.Cryptography/DataHasher.cs | 121 ++++ src/FrostFS.SDK.Cryptography/Extentions.cs | 79 -- .../FrostFS.SDK.Cryptography.csproj | 1 - src/FrostFS.SDK.Cryptography/HashStream.cs | 4 +- src/FrostFS.SDK.Cryptography/Key.cs | 36 +- src/FrostFS.SDK.Protos/AssemblyInfo.cs | 4 +- .../Mocks/AsyncStreamReaderMock.cs | 3 +- .../ContainerServiceBase.cs | 2 - .../ContainerServiceMocks/GetContainerMock.cs | 7 +- src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs | 1 - .../Client/ContainerTests/ContainerTests.cs | 41 +- .../Smoke/Client/NetworkTests/NetworkTests.cs | 10 +- .../Smoke/Client/ObjectTests/ObjectTests.cs | 6 +- .../Multithread/MultiThreadTestsBase.cs | 42 -- .../Multithread/MultithreadPoolSmokeTests.cs | 544 -------------- .../MultithreadSmokeClientTests.cs | 684 ------------------ .../Smoke/PoolTests/PoolSmokeTests.cs | 546 -------------- src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs | 16 +- src/FrostFS.SDK.Tests/Unit/ContainerTest.cs | 10 +- src/FrostFS.SDK.Tests/Unit/ObjectTest.cs | 4 +- src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs | 2 +- src/FrostFS.SDK.Tests/Unit/SessionTests.cs | 3 +- 76 files changed, 399 insertions(+), 3668 deletions(-) delete mode 100644 src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/ClientWrapper.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/HealthyStatus.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/IClientStatus.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/InitParameters.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/InnerPool.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/MethodIndex.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/MethodStatus.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/NodeParam.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/NodeStatistic.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/NodesParam.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/Pool.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/RequestInfo.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/Sampler.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/Statistic.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/WorkList.cs delete mode 100644 src/FrostFS.SDK.Client/Pool/WrapperPrm.cs rename src/FrostFS.SDK.Client/{Pool => Services/Shared}/SessionCache.cs (72%) create mode 100644 src/FrostFS.SDK.Cryptography/DataHasher.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs delete mode 100644 src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs diff --git a/src/FrostFS.SDK.Client/ApeRules/Actions.cs b/src/FrostFS.SDK.Client/ApeRules/Actions.cs index 71888b9..8bae303 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Actions.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Actions.cs @@ -8,7 +8,7 @@ public struct Actions(bool inverted, string[] names) : System.IEquatable public override bool Equals(object obj) { - if (obj == null || obj is not Condition) + if (obj == null || obj is not Condition) return false; return Equals((Condition)obj); diff --git a/src/FrostFS.SDK.Client/ApeRules/Resources.cs b/src/FrostFS.SDK.Client/ApeRules/Resources.cs index ef06b4b..47ff3e1 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Resources.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Resources.cs @@ -8,7 +8,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable _ownersCache; - - internal static IMemoryCache Containers => _containersCache; } diff --git a/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs b/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs index 0e64db5..87799e2 100644 --- a/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs +++ b/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs @@ -10,8 +10,8 @@ public class FrostFsResponseException : FrostFsException { } - public FrostFsResponseException(FrostFsResponseStatus status) - : base(status != null ? status.Message != null ? "" : "" : "") + public FrostFsResponseException(FrostFsResponseStatus status) + : base(status != null ? status.Message != null ? "" : "" : "") { Status = status; } diff --git a/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs b/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs index a986a87..87728d5 100644 --- a/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs +++ b/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs @@ -1,14 +1,17 @@ using System; - +using System.Security.Cryptography; using Google.Protobuf; namespace FrostFS.SDK.Cryptography; public static class FrostFsExtensions { - public static ByteString Sha256(this IMessage data) + public static byte[] Sha256(this IMessage data) { - return ByteString.CopyFrom(data.ToByteArray().Sha256()); + using var sha256 = SHA256.Create(); + using HashStream stream = new(sha256); + data.WriteTo(stream); + return stream.Hash(); } public static Guid ToUuid(this ByteString id) @@ -16,9 +19,18 @@ public static class FrostFsExtensions if (id == null) throw new ArgumentNullException(nameof(id)); - var orderedBytes = GetGuidBytesDirectOrder(id.Span); - - return new Guid(orderedBytes); + return new Guid( + (id[0] << 24) + (id[1] << 16) + (id[2] << 8) + id[3], + (short)((id[4] << 8) + id[5]), + (short)((id[6] << 8) + id[7]), + id[8], + id[9], + id[10], + id[11], + id[12], + id[13], + id[14], + id[15]); } /// @@ -26,37 +38,25 @@ public static class FrostFsExtensions /// /// /// - public static byte[] ToBytes(this Guid id) + public unsafe static void ToBytes(this Guid id, Span span) { - var bytes = id.ToByteArray(); + var pGuid = (byte*)&id; - var orderedBytes = GetGuidBytesDirectOrder(bytes); - - return orderedBytes; - } - - private static byte[] GetGuidBytesDirectOrder(ReadOnlySpan source) - { - if (source.Length != 16) - throw new ArgumentException("Wrong uuid binary format"); - - return [ - source[3], - source[2], - source[1], - source[0], - source[5], - source[4], - source[7], - source[6], - source[8], - source[9], - source[10], - source[11], - source[12], - source[13], - source[14], - source[15] - ]; + span[0] = pGuid[3]; + span[1] = pGuid[2]; + span[2] = pGuid[1]; + span[3] = pGuid[0]; + span[4] = pGuid[5]; + span[5] = pGuid[4]; + span[6] = pGuid[7]; + span[7] = pGuid[6]; + span[8] = pGuid[8]; + span[9] = pGuid[9]; + span[10] = pGuid[10]; + span[11] = pGuid[11]; + span[12] = pGuid[12]; + span[13] = pGuid[13]; + span[14] = pGuid[14]; + span[15] = pGuid[15]; } } diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index 2205b7e..e94d3f8 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -31,10 +31,10 @@ false + True - all diff --git a/src/FrostFS.SDK.Client/FrostFSClient.cs b/src/FrostFS.SDK.Client/FrostFSClient.cs index f72265c..860d346 100644 --- a/src/FrostFS.SDK.Client/FrostFSClient.cs +++ b/src/FrostFS.SDK.Client/FrostFSClient.cs @@ -162,25 +162,7 @@ public class FrostFSClient : IFrostFSClient Callback = settings.Value.Callback, Interceptors = settings.Value.Interceptors }; - - // TODO: define timeout logic - // CheckFrostFsVersionSupport(new Context { Timeout = TimeSpan.FromSeconds(20) }); - } - - internal FrostFSClient(WrapperPrm prm, SessionCache cache) - { - ClientCtx = new ClientContext( - client: this, - key: new ClientKey(prm.Key), - owner: FrostFsOwner.FromKey(prm.Key!), - channel: prm.GrpcChannelFactory(prm.Address), - version: new FrostFsVersion(2, 13)) - { - SessionCache = cache, - Interceptors = prm.Interceptors, - Callback = prm.Callback - }; - } + } #region ApeManagerImplementation public Task> AddChainAsync(PrmApeChainAdd args, CallContext ctx) @@ -447,18 +429,5 @@ public class FrostFSClient : IFrostFSClient } return ObjectServiceProvider; - } - - public async Task Dial(CallContext ctx) - { - var service = GetAccouningService(); - _ = await service.GetBallance(ctx).ConfigureAwait(false); - - return null; - } - - public bool RestartIfUnhealthy(CallContext ctx) - { - throw new NotImplementedException(); - } + } } diff --git a/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs b/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs index d546156..43dc3ac 100644 --- a/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs +++ b/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs @@ -64,6 +64,4 @@ public interface IFrostFSClient #region Account Task GetBalanceAsync(CallContext ctx); #endregion - - public Task Dial(CallContext ctx); } diff --git a/src/FrostFS.SDK.Client/Mappers/ContainerId.cs b/src/FrostFS.SDK.Client/Mappers/ContainerId.cs index df27320..55b6179 100644 --- a/src/FrostFS.SDK.Client/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.Client/Mappers/ContainerId.cs @@ -5,16 +5,10 @@ using FrostFS.SDK.Cryptography; using Google.Protobuf; -using Microsoft.Extensions.Caching.Memory; - namespace FrostFS.SDK.Client.Mappers.GRPC; public static class ContainerIdMapper { - private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions() - .SetSlidingExpiration(TimeSpan.FromHours(1)) - .SetSize(1); - public static ContainerID ToMessage(this FrostFsContainerId model) { if (model is null) @@ -24,15 +18,11 @@ public static class ContainerIdMapper var containerId = model.GetValue() ?? throw new ArgumentNullException(nameof(model)); - if (!Caches.Containers.TryGetValue(containerId, out ContainerID? message)) + var message = new ContainerID { - message = new ContainerID - { - Value = ByteString.CopyFrom(Base58.Decode(containerId)) - }; + Value = UnsafeByteOperations.UnsafeWrap(Base58.Decode(containerId)) + }; - Caches.Containers.Set(containerId, message, _oneHourExpiration); - } return message!; } diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs index 317b31c..115f9e2 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs @@ -24,25 +24,14 @@ public static class ObjectHeaderMapper _ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.") }; - FrostFsSplit? split = null; - - if (header.Split != null) - { - var children = header.Split.Children.Count != 0 ? new ReadOnlyCollection( - header.Split.Children.Select(x => x.ToModel()).ToList()) : null; - - split = new FrostFsSplit(new SplitId(header.Split.SplitId.ToUuid()), - header.Split.Previous?.ToModel(), - header.Split.Parent?.ToModel(), - header.Split.ParentHeader?.ToModel(), - null, - children); - } + FrostFsSplit? split = header!.Split != null + ? header.Split.ToModel() + : null; var model = new FrostFsObjectHeader( new FrostFsContainerId(Base58.Encode(header.ContainerId.Value.Span)), objTypeName, - header.Attributes.Select(attribute => attribute.ToModel()).ToArray(), + [.. header.Attributes.Select(attribute => attribute.ToModel())], split, header.OwnerId.ToModel(), header.Version.ToModel()) @@ -52,4 +41,18 @@ public static class ObjectHeaderMapper return model; } + + public static FrostFsSplit ToModel(this Header.Types.Split split) + { + var children = split!.Children.Count != 0 + ? new ReadOnlyCollection([.. split.Children.Select(x => x.ToModel())]) + : null; + + return new FrostFsSplit(new SplitId(split.SplitId.ToUuid()), + split.Previous?.ToModel(), + split.Parent?.ToModel(), + split.ParentHeader?.ToModel(), + null, + children); + } } diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs index 343e3de..5562be5 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs @@ -17,7 +17,7 @@ public static class ObjectIdMapper return new ObjectID { - Value = ByteString.CopyFrom(objectId.ToHash()) + Value = UnsafeByteOperations.UnsafeWrap(objectId.ToHash()) }; } diff --git a/src/FrostFS.SDK.Client/Mappers/OwnerId.cs b/src/FrostFS.SDK.Client/Mappers/OwnerId.cs index 6739a0b..54f7a21 100644 --- a/src/FrostFS.SDK.Client/Mappers/OwnerId.cs +++ b/src/FrostFS.SDK.Client/Mappers/OwnerId.cs @@ -26,7 +26,7 @@ public static class OwnerIdMapper { message = new OwnerID { - Value = ByteString.CopyFrom(model.ToHash()) + Value = UnsafeByteOperations.UnsafeWrap(model.ToHash()) }; Caches.Owners.Set(model, message, _oneHourExpiration); diff --git a/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs b/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs deleted file mode 100644 index d1307e8..0000000 --- a/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; - -using Google.Protobuf; - -namespace FrostFS.SDK.Client; - -public static class SessionMapper -{ - public static byte[] Serialize(this Session.SessionToken token) - { - if (token is null) - { - throw new ArgumentNullException(nameof(token)); - } - - byte[] bytes = new byte[token.CalculateSize()]; - using CodedOutputStream stream = new(bytes); - token.WriteTo(stream); - - return bytes; - } - - public static Session.SessionToken Deserialize(this Session.SessionToken token, byte[] bytes) - { - token.MergeFrom(bytes); - return token; - } -} diff --git a/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs b/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs index 88b4ac8..ffae746 100644 --- a/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs @@ -23,9 +23,9 @@ public static class SignatureMapper return new Refs.Signature { - Key = ByteString.CopyFrom(signature.Key), + Key = UnsafeByteOperations.UnsafeWrap(signature.Key), Scheme = scheme, - Sign = ByteString.CopyFrom(signature.Sign) + Sign = UnsafeByteOperations.UnsafeWrap(signature.Sign) }; } } diff --git a/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs b/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs index af4ffdc..c7dc782 100644 --- a/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs +++ b/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs @@ -91,10 +91,13 @@ public class FrostFsContainerInfo throw new ArgumentNullException("PlacementPolicy is null"); } + Span nonce = stackalloc byte[16]; + Nonce.ToBytes(nonce); + this.container = new Container.Container() { PlacementPolicy = PlacementPolicy.Value.GetPolicy(), - Nonce = ByteString.CopyFrom(Nonce.ToBytes()), + Nonce = ByteString.CopyFrom(nonce), OwnerId = Owner?.OwnerID, Version = Version?.VersionID }; diff --git a/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs b/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs index fdeac99..2241227 100644 --- a/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs +++ b/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs @@ -11,7 +11,7 @@ public class CheckSum public static CheckSum CreateCheckSum(byte[] content) { - return new CheckSum { hash = content.Sha256() }; + return new CheckSum { hash = DataHasher.Sha256(content.AsSpan()) }; } public override string ToString() diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs index cd287d0..aedfd35 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs @@ -353,7 +353,7 @@ internal struct Context var start = hasPrefix ? likeWildcard.Length : 0; var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length; - var str = f.Value.Substring(start, end-start); + var str = f.Value.Substring(start, end - start); if (hasPrefix && hasSuffix) return nodeInfo.Attributes[f.Key].Contains(str); diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs index c39a587..e22016a 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs @@ -4,10 +4,6 @@ namespace FrostFS.SDK; public class FrostFsObject { - // private byte[]? _payloadBytes; - // private ReadOnlyMemory _payloadMemory; - // private bool _isInitPayloadMemory; - /// /// Creates new instance from ObjectHeader /// @@ -49,25 +45,6 @@ public class FrostFsObject public ReadOnlyMemory SingleObjectPayload { get; set; } - //public ReadOnlyMemory SingleObjectPayloadMemory - //{ - // get - // { - // if (!_isInitPayloadMemory) - // { - // _payloadMemory = _payloadBytes.AsMemory(); - // _isInitPayloadMemory = true; - // } - - // return _payloadMemory; - // } - // set - // { - // _payloadMemory = value; - // _isInitPayloadMemory = true; - // } - //} - /// /// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic /// diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs index bd850cb..5afd46b 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs @@ -1,4 +1,7 @@ using System.Collections.ObjectModel; +using System.Linq; +using FrostFS.Object; +using FrostFS.SDK.Client.Mappers.GRPC; namespace FrostFS.SDK; @@ -9,6 +12,8 @@ public class FrostFsSplit(SplitId splitId, FrostFsSignature? parentSignature = null, ReadOnlyCollection? children = null) { + private Header.Types.Split? _split; + public FrostFsSplit() : this(new SplitId()) { } @@ -24,4 +29,25 @@ public class FrostFsSplit(SplitId splitId, public FrostFsObjectHeader? ParentHeader { get; set; } = parentHeader; public ReadOnlyCollection? Children { get; } = children; + + public Header.Types.Split GetSplit() + { + if (_split == null) + { + _split = new Header.Types.Split + { + SplitId = SplitId?.GetSplitId(), + Parent = Parent?.ToMessage(), + ParentHeader = ParentHeader?.GetHeader(), + ParentSignature = ParentSignature?.ToMessage() + }; + + if (Children != null) + { + _split.Children.AddRange(Children.Select(x => x.ToMessage())); + } + } + + return _split; + } } diff --git a/src/FrostFS.SDK.Client/Models/Object/SplitId.cs b/src/FrostFS.SDK.Client/Models/Object/SplitId.cs index 177817e..a113361 100644 --- a/src/FrostFS.SDK.Client/Models/Object/SplitId.cs +++ b/src/FrostFS.SDK.Client/Models/Object/SplitId.cs @@ -47,16 +47,11 @@ public class SplitId return this.id.ToString(); } - public byte[]? ToBinary() - { - if (this.id == Guid.Empty) - return null; - - return this.id.ToBytes(); - } - public ByteString? GetSplitId() { - return this.message ??= ByteString.CopyFrom(ToBinary()); + Span span = stackalloc byte[16]; + id.ToBytes(span); + + return this.message ??= ByteString.CopyFrom(span); } } diff --git a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs index 2f48ba3..9e1a846 100644 --- a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs +++ b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs @@ -4,9 +4,9 @@ public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = nul { public FrostFsStatusCode Code { get; set; } = code; public string Message { get; set; } = message ?? string.Empty; - + public string Details { get; set; } = details ?? string.Empty; - + public bool IsSuccess => Code == FrostFsStatusCode.Success; public override string ToString() diff --git a/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs b/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs index 720c3c4..2303776 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs @@ -25,7 +25,7 @@ public readonly struct PrmApeChainRemove( public readonly bool Equals(PrmApeChainRemove other) { return Target == other.Target - && ChainId.Equals(other.ChainId) + && ChainId.Equals(other.ChainId) && XHeaders == other.XHeaders; } diff --git a/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs b/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs deleted file mode 100644 index c05c24c..0000000 --- a/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System; -using System.Threading; - -using Microsoft.Extensions.Logging; - -namespace FrostFS.SDK.Client; - -// clientStatusMonitor count error rate and other statistics for connection. -public class ClientStatusMonitor : IClientStatus -{ - private static readonly MethodIndex[] MethodIndexes = - [ - MethodIndex.methodBalanceGet, - MethodIndex.methodContainerPut, - MethodIndex.methodContainerGet, - MethodIndex.methodContainerList, - MethodIndex.methodContainerDelete, - MethodIndex.methodEndpointInfo, - MethodIndex.methodNetworkInfo, - MethodIndex.methodNetMapSnapshot, - MethodIndex.methodObjectPut, - MethodIndex.methodObjectDelete, - MethodIndex.methodObjectGet, - MethodIndex.methodObjectHead, - MethodIndex.methodObjectRange, - MethodIndex.methodObjectPatch, - MethodIndex.methodSessionCreate, - MethodIndex.methodAPEManagerAddChain, - MethodIndex.methodAPEManagerRemoveChain, - MethodIndex.methodAPEManagerListChains - ]; - - public static string GetMethodName(MethodIndex index) - { - return index switch - { - MethodIndex.methodBalanceGet => "BalanceGet", - MethodIndex.methodContainerPut => "ContainerPut", - MethodIndex.methodContainerGet => "ContainerGet", - MethodIndex.methodContainerList => "ContainerList", - MethodIndex.methodContainerDelete => "ContainerDelete", - MethodIndex.methodEndpointInfo => "EndpointInfo", - MethodIndex.methodNetworkInfo => "NetworkInfo", - MethodIndex.methodNetMapSnapshot => "NetMapSnapshot", - MethodIndex.methodObjectPut => "ObjectPut", - MethodIndex.methodObjectDelete => "ObjectDelete", - MethodIndex.methodObjectGet => "ObjectGet", - MethodIndex.methodObjectHead => "ObjectHead", - MethodIndex.methodObjectRange => "ObjectRange", - MethodIndex.methodObjectPatch => "ObjectPatch", - MethodIndex.methodSessionCreate => "SessionCreate", - MethodIndex.methodAPEManagerAddChain => "APEManagerAddChain", - MethodIndex.methodAPEManagerRemoveChain => "APEManagerRemoveChain", - MethodIndex.methodAPEManagerListChains => "APEManagerListChains", - _ => throw new ArgumentException("Unknown method", nameof(index)), - }; - } - - private readonly object _lock = new(); - - private readonly ILogger? logger; - private int healthy; - - public ClientStatusMonitor(ILogger? logger, string address) - { - this.logger = logger; - healthy = (int)HealthyStatus.Healthy; - - Address = address; - Methods = new MethodStatus[MethodIndexes.Length]; - - for (int i = 0; i < MethodIndexes.Length; i++) - { - Methods[i] = new MethodStatus(GetMethodName(MethodIndexes[i])); - } - } - - public string Address { get; } - - internal uint ErrorThreshold { get; set; } - - public uint CurrentErrorCount { get; set; } - - public ulong OverallErrorCount { get; set; } - - public MethodStatus[] Methods { get; private set; } - - public bool IsHealthy() - { - var res = Interlocked.CompareExchange(ref healthy, -1, -1) == (int)HealthyStatus.Healthy; - return res; - } - - public bool IsDialed() - { - return Interlocked.CompareExchange(ref healthy, -1, -1) != (int)HealthyStatus.UnhealthyOnDial; - } - - public void SetHealthy() - { - Interlocked.Exchange(ref healthy, (int)HealthyStatus.Healthy); - } - public void SetUnhealthy() - { - Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnRequest); - } - - public void SetUnhealthyOnDial() - { - Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnDial); - } - - public void IncErrorRate() - { - bool thresholdReached; - lock (_lock) - { - CurrentErrorCount++; - OverallErrorCount++; - - thresholdReached = CurrentErrorCount >= ErrorThreshold; - - if (thresholdReached) - { - SetUnhealthy(); - CurrentErrorCount = 0; - } - } - - if (thresholdReached && logger != null) - { - FrostFsMessages.ErrorЕhresholdReached(logger, Address, ErrorThreshold); - } - } - - public uint GetCurrentErrorRate() - { - lock (_lock) - { - return CurrentErrorCount; - } - } - - public ulong GetOverallErrorRate() - { - lock (_lock) - { - return OverallErrorCount; - } - } - - public StatusSnapshot[] MethodsStatus() - { - var result = new StatusSnapshot[Methods.Length]; - - for (int i = 0; i < result.Length; i++) - { - result[i] = Methods[i].Snapshot!; - } - - return result; - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs b/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs deleted file mode 100644 index 552ee2a..0000000 --- a/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs +++ /dev/null @@ -1,135 +0,0 @@ -using System; -using System.Threading.Tasks; - -using Grpc.Core; - -namespace FrostFS.SDK.Client; - -// clientWrapper is used by default, alternative implementations are intended for testing purposes only. -public class ClientWrapper : ClientStatusMonitor -{ - private readonly object _lock = new(); - private SessionCache sessionCache; - - internal ClientWrapper(WrapperPrm wrapperPrm, Pool pool) : base(wrapperPrm.Logger, wrapperPrm.Address) - { - WrapperPrm = wrapperPrm; - ErrorThreshold = wrapperPrm.ErrorThreshold; - - sessionCache = pool.SessionCache; - Client = new FrostFSClient(WrapperPrm, sessionCache); - } - - internal FrostFSClient? Client { get; private set; } - - internal WrapperPrm WrapperPrm { get; } - - internal FrostFSClient? GetClient() - { - lock (_lock) - { - if (IsHealthy()) - { - return Client; - } - - return null; - } - } - - // dial establishes a connection to the server from the FrostFS network. - // Returns an error describing failure reason. If failed, the client - // SHOULD NOT be used. - internal async Task Dial(CallContext ctx) - { - var client = GetClient() ?? throw new FrostFsInvalidObjectException("pool client unhealthy"); - - await client.Dial(ctx).ConfigureAwait(false); - } - - internal void HandleError(Exception ex) - { - if (ex is FrostFsResponseException responseException && responseException.Status != null) - { - switch (responseException.Status.Code) - { - case FrostFsStatusCode.Internal: - case FrostFsStatusCode.WrongMagicNumber: - case FrostFsStatusCode.SignatureVerificationFailure: - case FrostFsStatusCode.NodeUnderMaintenance: - IncErrorRate(); - return; - } - } - - IncErrorRate(); - } - - private async Task ScheduleGracefulClose() - { - if (Client == null) - return; - - await Task.Delay((int)WrapperPrm.GracefulCloseOnSwitchTimeout).ConfigureAwait(false); - } - - // restartIfUnhealthy checks healthy status of client and recreate it if status is unhealthy. - // Indicating if status was changed by this function call and returns error that caused unhealthy status. - internal async Task RestartIfUnhealthy(CallContext ctx) - { - bool wasHealthy; - - try - { - var response = await Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false); - return false; - } - catch (RpcException) - { - wasHealthy = true; - } - - // if connection is dialed before, to avoid routine/connection leak, - // pool has to close it and then initialize once again. - if (IsDialed()) - { - await ScheduleGracefulClose().ConfigureAwait(false); - } - - FrostFSClient? client = new(WrapperPrm, sessionCache); - - var error = await client.Dial(ctx).ConfigureAwait(false); - if (!string.IsNullOrEmpty(error)) - { - SetUnhealthyOnDial(); - return wasHealthy; - } - - lock (_lock) - { - Client = client; - } - - try - { - var res = await Client.GetNodeInfoAsync(ctx).ConfigureAwait(false); - } - catch (FrostFsException) - { - SetUnhealthy(); - - return wasHealthy; - } - - SetHealthy(); - return !wasHealthy; - } - - internal void IncRequests(ulong elapsed, MethodIndex method) - { - var methodStat = Methods[(int)method]; - - methodStat.IncRequests(elapsed); - } -} - diff --git a/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs b/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs deleted file mode 100644 index d02e6bb..0000000 --- a/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace FrostFS.SDK.Client; - -// values for healthy status of clientStatusMonitor. -public enum HealthyStatus -{ - // statusUnhealthyOnDial is set when dialing to the endpoint is failed, - // so there is no connection to the endpoint, and pool should not close it - // before re-establishing connection once again. - UnhealthyOnDial, - - // statusUnhealthyOnRequest is set when communication after dialing to the - // endpoint is failed due to immediate or accumulated errors, connection is - // available and pool should close it before re-establishing connection once again. - UnhealthyOnRequest, - - // statusHealthy is set when connection is ready to be used by the pool. - Healthy -} diff --git a/src/FrostFS.SDK.Client/Pool/IClientStatus.cs b/src/FrostFS.SDK.Client/Pool/IClientStatus.cs deleted file mode 100644 index 0c08fac..0000000 --- a/src/FrostFS.SDK.Client/Pool/IClientStatus.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace FrostFS.SDK.Client; - -public interface IClientStatus -{ - // isHealthy checks if the connection can handle requests. - bool IsHealthy(); - - // isDialed checks if the connection was created. - bool IsDialed(); - - // setUnhealthy marks client as unhealthy. - void SetUnhealthy(); - - // address return address of endpoint. - string Address { get; } - - // currentErrorRate returns current errors rate. - // After specific threshold connection is considered as unhealthy. - // Pool.startRebalance routine can make this connection healthy again. - uint GetCurrentErrorRate(); - - // overallErrorRate returns the number of all happened errors. - ulong GetOverallErrorRate(); - - // methodsStatus returns statistic for all used methods. - StatusSnapshot[] MethodsStatus(); -} - diff --git a/src/FrostFS.SDK.Client/Pool/InitParameters.cs b/src/FrostFS.SDK.Client/Pool/InitParameters.cs deleted file mode 100644 index 66da23f..0000000 --- a/src/FrostFS.SDK.Client/Pool/InitParameters.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.ObjectModel; -using System.Security.Cryptography; - -using Grpc.Core; -using Grpc.Core.Interceptors; -using Grpc.Net.Client; - -using Microsoft.Extensions.Logging; - -namespace FrostFS.SDK.Client; - -// InitParameters contains values used to initialize connection Pool. -public class InitParameters(Func grpcChannelFactory) -{ - public ECDsa? Key { get; set; } - - public ulong NodeDialTimeout { get; set; } - - public ulong NodeStreamTimeout { get; set; } - - public ulong HealthcheckTimeout { get; set; } - - public ulong ClientRebalanceInterval { get; set; } - - public ulong SessionExpirationDuration { get; set; } - - public uint ErrorThreshold { get; set; } - - public NodeParam[]? NodeParams { get; set; } - - public GrpcChannelOptions[]? DialOptions { get; set; } - - public Func? ClientBuilder { get; set; } - - public ulong GracefulCloseOnSwitchTimeout { get; set; } - - public ILogger? Logger { get; set; } - - public Action? Callback { get; set; } - - public Collection Interceptors { get; } = []; - - public Func GrpcChannelFactory { get; set; } = grpcChannelFactory; -} diff --git a/src/FrostFS.SDK.Client/Pool/InnerPool.cs b/src/FrostFS.SDK.Client/Pool/InnerPool.cs deleted file mode 100644 index f712552..0000000 --- a/src/FrostFS.SDK.Client/Pool/InnerPool.cs +++ /dev/null @@ -1,47 +0,0 @@ -using FrostFS.SDK.Client; - -internal sealed class InnerPool -{ - private readonly object _lock = new(); - - internal InnerPool(Sampler sampler, ClientWrapper[] clients) - { - Clients = clients; - Sampler = sampler; - } - - internal Sampler Sampler { get; set; } - - internal ClientWrapper[] Clients { get; } - - internal ClientWrapper? Connection() - { - lock (_lock) - { - if (Clients.Length == 1) - { - var client = Clients[0]; - if (client.IsHealthy()) - { - return client; - } - } - else - { - var attempts = 3 * Clients.Length; - - for (int i = 0; i < attempts; i++) - { - int index = Sampler.Next(); - - if (Clients[index].IsHealthy()) - { - return Clients[index]; - } - } - } - - return null; - } - } -} diff --git a/src/FrostFS.SDK.Client/Pool/MethodIndex.cs b/src/FrostFS.SDK.Client/Pool/MethodIndex.cs deleted file mode 100644 index 53e5430..0000000 --- a/src/FrostFS.SDK.Client/Pool/MethodIndex.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace FrostFS.SDK.Client; - -public enum MethodIndex -{ - methodBalanceGet, - methodContainerPut, - methodContainerGet, - methodContainerList, - methodContainerDelete, - methodEndpointInfo, - methodNetworkInfo, - methodNetMapSnapshot, - methodObjectPut, - methodObjectDelete, - methodObjectGet, - methodObjectHead, - methodObjectRange, - methodObjectPatch, - methodSessionCreate, - methodAPEManagerAddChain, - methodAPEManagerRemoveChain, - methodAPEManagerListChains, - methodLast -} diff --git a/src/FrostFS.SDK.Client/Pool/MethodStatus.cs b/src/FrostFS.SDK.Client/Pool/MethodStatus.cs deleted file mode 100644 index 8f40f3c..0000000 --- a/src/FrostFS.SDK.Client/Pool/MethodStatus.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class MethodStatus(string name) -{ - private readonly object _lock = new(); - - public string Name { get; } = name; - - public StatusSnapshot Snapshot { get; set; } = new StatusSnapshot(); - - internal void IncRequests(ulong elapsed) - { - lock (_lock) - { - Snapshot.AllTime += elapsed; - Snapshot.AllRequests++; - } - } -} diff --git a/src/FrostFS.SDK.Client/Pool/NodeParam.cs b/src/FrostFS.SDK.Client/Pool/NodeParam.cs deleted file mode 100644 index 92c2560..0000000 --- a/src/FrostFS.SDK.Client/Pool/NodeParam.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace FrostFS.SDK.Client; - -// NodeParam groups parameters of remote node. -[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "")] -public readonly struct NodeParam(int priority, string address, float weight) -{ - public int Priority { get; } = priority; - - public string Address { get; } = address; - - public float Weight { get; } = weight; -} diff --git a/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs b/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs deleted file mode 100644 index 9323d93..0000000 --- a/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class NodeStatistic -{ - public string? Address { get; internal set; } - - public StatusSnapshot[]? Methods { get; internal set; } - - public ulong OverallErrors { get; internal set; } - - public uint CurrentErrors { get; internal set; } -} diff --git a/src/FrostFS.SDK.Client/Pool/NodesParam.cs b/src/FrostFS.SDK.Client/Pool/NodesParam.cs deleted file mode 100644 index be9f012..0000000 --- a/src/FrostFS.SDK.Client/Pool/NodesParam.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Collections.ObjectModel; - -namespace FrostFS.SDK.Client; - -public class NodesParam(int priority) -{ - public int Priority { get; } = priority; - - public Collection Addresses { get; } = []; - - public Collection Weights { get; } = []; -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Pool/Pool.cs b/src/FrostFS.SDK.Client/Pool/Pool.cs deleted file mode 100644 index cd09d30..0000000 --- a/src/FrostFS.SDK.Client/Pool/Pool.cs +++ /dev/null @@ -1,677 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography; -using System.Threading; -using System.Threading.Tasks; - -using FrostFS.Refs; -using FrostFS.SDK.Client.Interfaces; -using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; - -using Grpc.Core; - -using Microsoft.Extensions.Logging; - - -namespace FrostFS.SDK.Client; - -public partial class Pool : IFrostFSClient -{ - const int defaultSessionTokenExpirationDuration = 100; // in epochs - - const int defaultErrorThreshold = 100; - - const int defaultGracefulCloseOnSwitchTimeout = 10; //Seconds; - const int defaultRebalanceInterval = 15; //Seconds; - const int defaultHealthcheckTimeout = 4; //Seconds; - const int defaultDialTimeout = 5; //Seconds; - const int defaultStreamTimeout = 10; //Seconds; - - private readonly object _lock = new(); - - private InnerPool[]? InnerPools { get; set; } - - private ClientKey Key { get; set; } - - private OwnerID? _ownerId; - - private FrostFsVersion _version; - - private FrostFsOwner? _owner; - - private FrostFsOwner Owner - { - get - { - _owner ??= new FrostFsOwner(Key.ECDsaKey.PublicKey().PublicKeyToAddress()); - return _owner; - } - } - - private OwnerID OwnerId - { - get - { - if (_ownerId == null) - { - _owner = Key.Owner; - _ownerId = _owner.ToMessage(); - } - return _ownerId; - } - } - - internal CancellationTokenSource CancellationTokenSource { get; } = new CancellationTokenSource(); - - internal SessionCache SessionCache { get; set; } - - private ulong SessionTokenDuration { get; set; } - - private RebalanceParameters RebalanceParams { get; set; } - - private Func ClientBuilder; - - private bool disposedValue; - - private ILogger? logger { get; set; } - - private ulong MaxObjectSize { get; set; } - - public IClientStatus? ClientStatus { get; } - - // NewPool creates connection pool using parameters. - public Pool(InitParameters options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (options.Key == null) - { - throw new ArgumentException($"Missed required parameter {nameof(options.Key)}"); - } - - _version = new FrostFsVersion(2, 13); - - var nodesParams = AdjustNodeParams(options.NodeParams); - - var cache = new SessionCache(options.SessionExpirationDuration); - - FillDefaultInitParams(options, this); - - Key = new ClientKey(options.Key); - - SessionCache = cache; - logger = options.Logger; - SessionTokenDuration = options.SessionExpirationDuration; - - RebalanceParams = new RebalanceParameters( - nodesParams.ToArray(), - options.HealthcheckTimeout, - options.ClientRebalanceInterval, - options.SessionExpirationDuration); - - ClientBuilder = options.ClientBuilder!; - - // ClientContext.PoolErrorHandler = client.HandleError; - - } - - // Dial establishes a connection to the servers from the FrostFS network. - // It also starts a routine that checks the health of the nodes and - // updates the weights of the nodes for balancing. - // Returns an error describing failure reason. - // - // If failed, the Pool SHOULD NOT be used. - // - // See also InitParameters.SetClientRebalanceInterval. - public async Task Dial(CallContext ctx) - { - var inner = new InnerPool[RebalanceParams.NodesParams.Length]; - - bool atLeastOneHealthy = false; - int i = 0; - foreach (var nodeParams in RebalanceParams.NodesParams) - { - var wrappers = new ClientWrapper[nodeParams.Weights.Count]; - - for (int j = 0; j < nodeParams.Addresses.Count; j++) - { - ClientWrapper? wrapper = null; - bool dialed = false; - try - { - wrapper = wrappers[j] = ClientBuilder(nodeParams.Addresses[j]); - - await wrapper.Dial(ctx).ConfigureAwait(false); - dialed = true; - - var token = await InitSessionForDuration(ctx, wrapper, RebalanceParams.SessionExpirationDuration, Key.ECDsaKey, false) - .ConfigureAwait(false); - - var key = FormCacheKey(nodeParams.Addresses[j], Key.PublicKey); - SessionCache.SetValue(key, token); - - atLeastOneHealthy = true; - } - catch (RpcException ex) - { - if (!dialed) - wrapper!.SetUnhealthyOnDial(); - else - wrapper!.SetUnhealthy(); - - if (logger != null) - { - FrostFsMessages.SessionCreationError(logger, wrapper!.WrapperPrm.Address, ex.Message); - } - } - catch (FrostFsInvalidObjectException) - { - break; - } - } - - var sampler = new Sampler(nodeParams.Weights.ToArray()); - - inner[i] = new InnerPool(sampler, wrappers); - - i++; - } - - if (!atLeastOneHealthy) - return "At least one node must be healthy"; - - InnerPools = inner; - - var res = await GetNetworkSettingsAsync(default).ConfigureAwait(false); - - MaxObjectSize = res.MaxObjectSize; - - StartRebalance(ctx); - - return null; - } - - private static IEnumerable AdjustNodeParams(NodeParam[]? nodeParams) - { - if (nodeParams == null || nodeParams.Length == 0) - { - throw new ArgumentException("No FrostFS peers configured"); - } - - Dictionary nodesParamsDict = new(nodeParams.Length); - foreach (var nodeParam in nodeParams) - { - if (!nodesParamsDict.TryGetValue(nodeParam.Priority, out var nodes)) - { - nodes = new NodesParam(nodeParam.Priority); - nodesParamsDict[nodeParam.Priority] = nodes; - } - - nodes.Addresses.Add(nodeParam.Address); - nodes.Weights.Add(nodeParam.Weight); - } - - var nodesParams = new List(nodesParamsDict.Count); - - foreach (var key in nodesParamsDict.Keys) - { - var nodes = nodesParamsDict[key]; - var newWeights = AdjustWeights([.. nodes.Weights]); - nodes.Weights.Clear(); - foreach (var weight in newWeights) - { - nodes.Weights.Add(weight); - } - - nodesParams.Add(nodes); - } - - return nodesParams.OrderBy(n => n.Priority); - } - - private static double[] AdjustWeights(double[] weights) - { - var adjusted = new double[weights.Length]; - - var sum = weights.Sum(); - - if (sum > 0) - { - for (int i = 0; i < weights.Length; i++) - { - adjusted[i] = weights[i] / sum; - } - } - - return adjusted; - } - - private static void FillDefaultInitParams(InitParameters parameters, Pool pool) - { - if (parameters.SessionExpirationDuration == 0) - parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration; - - if (parameters.ErrorThreshold == 0) - parameters.ErrorThreshold = defaultErrorThreshold; - - if (parameters.ClientRebalanceInterval <= 0) - parameters.ClientRebalanceInterval = defaultRebalanceInterval; - - if (parameters.GracefulCloseOnSwitchTimeout <= 0) - parameters.GracefulCloseOnSwitchTimeout = defaultGracefulCloseOnSwitchTimeout; - - if (parameters.HealthcheckTimeout <= 0) - parameters.HealthcheckTimeout = defaultHealthcheckTimeout; - - if (parameters.NodeDialTimeout <= 0) - parameters.NodeDialTimeout = defaultDialTimeout; - - if (parameters.NodeStreamTimeout <= 0) - parameters.NodeStreamTimeout = defaultStreamTimeout; - - if (parameters.SessionExpirationDuration == 0) - parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration; - - parameters.ClientBuilder ??= new Func((address) => - { - var wrapperPrm = new WrapperPrm() - { - Address = address, - Key = parameters.Key!, - Logger = parameters.Logger, - DialTimeout = parameters.NodeDialTimeout, - StreamTimeout = parameters.NodeStreamTimeout, - ErrorThreshold = parameters.ErrorThreshold, - GracefulCloseOnSwitchTimeout = parameters.GracefulCloseOnSwitchTimeout, - Callback = parameters.Callback, - Interceptors = parameters.Interceptors, - GrpcChannelFactory = parameters.GrpcChannelFactory - - }; - - return new ClientWrapper(wrapperPrm, pool); - } - ); - } - - private ClientWrapper Connection() - { - foreach (var pool in InnerPools!) - { - var client = pool.Connection(); - if (client != null) - { - return client; - } - } - - throw new FrostFsException("Cannot find alive client"); - } - - private static async Task InitSessionForDuration(CallContext ctx, ClientWrapper cw, ulong duration, ECDsa key, bool clientCut) - { - var client = cw.Client; - var networkInfo = await client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - - var epoch = networkInfo.Epoch; - - ulong exp = ulong.MaxValue - epoch < duration - ? ulong.MaxValue - : epoch + duration; - - var prmSessionCreate = new PrmSessionCreate(exp); - - return await client.CreateSessionAsync(prmSessionCreate, ctx).ConfigureAwait(false); - } - - internal static string FormCacheKey(string address, string key) - { - return $"{address}{key}"; - } - - public void Close() - { - CancellationTokenSource.Cancel(); - - //if (InnerPools != null) - //{ - // // close all clients - // foreach (var innerPool in InnerPools) - // foreach (var client in innerPool.Clients) - // if (client.IsDialed()) - // client.Client?.Close(); - //} - } - - // startRebalance runs loop to monitor connection healthy status. - internal void StartRebalance(CallContext ctx) - { - var buffers = new double[RebalanceParams.NodesParams.Length][]; - - for (int i = 0; i < RebalanceParams.NodesParams.Length; i++) - { - var parameters = RebalanceParams.NodesParams[i]; - buffers[i] = new double[parameters.Weights.Count]; - - Task.Run(async () => - { - await Task.Delay((int)RebalanceParams.ClientRebalanceInterval).ConfigureAwait(false); - UpdateNodesHealth(ctx, buffers); - }); - } - } - - private void UpdateNodesHealth(CallContext ctx, double[][] buffers) - { - var tasks = new Task[InnerPools!.Length]; - - for (int i = 0; i < InnerPools.Length; i++) - { - var bufferWeights = buffers[i]; - - tasks[i] = Task.Run(() => UpdateInnerNodesHealth(ctx, i, bufferWeights)); - } - - Task.WaitAll(tasks); - } - - private async ValueTask UpdateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights) - { - if (poolIndex > InnerPools!.Length - 1) - { - return; - } - - var pool = InnerPools[poolIndex]; - - var options = RebalanceParams; - - int healthyChanged = 0; - - var tasks = new Task[pool.Clients.Length]; - - for (int j = 0; j < pool.Clients.Length; j++) - { - var client = pool.Clients[j]; - var healthy = false; - string? error = null; - var changed = false; - - try - { - // check timeout settings - changed = await client!.RestartIfUnhealthy(ctx).ConfigureAwait(false); - healthy = true; - bufferWeights[j] = options.NodesParams[poolIndex].Weights[j]; - } - // TODO: specify - catch (FrostFsException e) - { - error = e.Message; - bufferWeights[j] = 0; - - SessionCache.DeleteByPrefix(client.Address); - } - - if (changed) - { - if (!string.IsNullOrEmpty(error)) - { - if (logger != null) - { - FrostFsMessages.HealthChanged(logger, client.Address, healthy, error!); - } - - Interlocked.Exchange(ref healthyChanged, 1); - } - } - - await Task.WhenAll(tasks).ConfigureAwait(false); - - if (Interlocked.CompareExchange(ref healthyChanged, -1, -1) == 1) - { - var probabilities = AdjustWeights(bufferWeights); - - lock (_lock) - { - pool.Sampler = new Sampler(probabilities); - } - } - } - } - - - // TODO: remove - private bool CheckSessionTokenErr(Exception error, string address) - { - if (error == null) - { - return false; - } - - if (error is SessionNotFoundException || error is SessionExpiredException) - { - this.SessionCache.DeleteByPrefix(address); - return true; - } - - return false; - } - - public Statistic Statistic() - { - if (InnerPools == null) - { - throw new FrostFsInvalidObjectException(nameof(Pool)); - } - - var statistics = new Statistic(); - - foreach (var inner in InnerPools) - { - int valueIndex = 0; - var nodes = new string[inner.Clients.Length]; - - lock (_lock) - { - foreach (var client in inner.Clients) - { - if (client.IsHealthy()) - { - nodes[valueIndex] = client.Address; - } - - var node = new NodeStatistic - { - Address = client.Address, - Methods = client.MethodsStatus(), - OverallErrors = client.GetOverallErrorRate(), - CurrentErrors = client.GetCurrentErrorRate() - }; - - statistics.Nodes.Add(node); - - valueIndex++; - - statistics.OverallErrors += node.OverallErrors; - } - - if (statistics.CurrentNodes == null || statistics.CurrentNodes.Length == 0) - { - statistics.CurrentNodes = nodes; - } - } - } - - return statistics; - } - - public async Task GetNetmapSnapshotAsync(CallContext ctx) - { - var client = Connection(); - - return await client.Client!.GetNetmapSnapshotAsync(ctx).ConfigureAwait(false); - } - - public async Task GetNodeInfoAsync(CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false); - } - - public async Task GetNetworkSettingsAsync(CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - } - - public async Task CreateSessionAsync(PrmSessionCreate args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.CreateSessionAsync(args, ctx).ConfigureAwait(false); - } - - public async Task> AddChainAsync(PrmApeChainAdd args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.AddChainAsync(args, ctx).ConfigureAwait(false); - } - - public async Task RemoveChainAsync(PrmApeChainRemove args, CallContext ctx) - { - var client = Connection(); - await client.Client!.RemoveChainAsync(args, ctx).ConfigureAwait(false); - } - - public async Task ListChainAsync(PrmApeChainList args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.ListChainAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetContainerAsync(PrmContainerGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetContainerAsync(args, ctx).ConfigureAwait(false); - } - - public IAsyncEnumerable ListContainersAsync(PrmContainerGetAll args, CallContext ctx) - { - var client = Connection(); - return client.Client!.ListContainersAsync(args, ctx); - } - - [Obsolete("Use PutContainerAsync method")] - public async Task CreateContainerAsync(PrmContainerCreate args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutContainerAsync(PrmContainerCreate args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false); - } - - public async Task DeleteContainerAsync(PrmContainerDelete args, CallContext ctx) - { - var client = Connection(); - await client.Client!.DeleteContainerAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetObjectHeadAsync(PrmObjectHeadGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetObjectHeadAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetObjectAsync(PrmObjectGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutObjectAsync(PrmObjectPut args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutClientCutObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PutSingleObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PatchObjectAsync(PrmObjectPatch args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task GetRangeAsync(PrmRangeGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetRangeAsync(args, ctx).ConfigureAwait(false); - } - - public async Task[]> GetRangeHashAsync(PrmRangeHashGet args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetRangeHashAsync(args, ctx).ConfigureAwait(false); - } - - public async Task PatchAsync(PrmObjectPatch args, CallContext ctx) - { - var client = Connection(); - return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false); - } - - public async Task DeleteObjectAsync(PrmObjectDelete args, CallContext ctx) - { - var client = Connection(); - await client.Client!.DeleteObjectAsync(args, ctx).ConfigureAwait(false); - } - - public IAsyncEnumerable SearchObjectsAsync(PrmObjectSearch args, CallContext ctx) - { - var client = Connection(); - return client.Client!.SearchObjectsAsync(args, ctx); - } - - public async Task GetBalanceAsync(CallContext ctx) - { - var client = Connection(); - return await client.Client!.GetBalanceAsync(ctx).ConfigureAwait(false); - } - - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - Close(); - } - - disposedValue = true; - } - } - - public void Dispose() - { - Dispose(disposing: true); - } -} diff --git a/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs b/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs deleted file mode 100644 index 988002c..0000000 --- a/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class RebalanceParameters( - NodesParam[] nodesParams, - ulong nodeRequestTimeout, - ulong clientRebalanceInterval, - ulong sessionExpirationDuration) -{ - public NodesParam[] NodesParams { get; set; } = nodesParams; - - public ulong NodeRequestTimeout { get; set; } = nodeRequestTimeout; - - public ulong ClientRebalanceInterval { get; set; } = clientRebalanceInterval; - - public ulong SessionExpirationDuration { get; set; } = sessionExpirationDuration; -} diff --git a/src/FrostFS.SDK.Client/Pool/RequestInfo.cs b/src/FrostFS.SDK.Client/Pool/RequestInfo.cs deleted file mode 100644 index 3b70650..0000000 --- a/src/FrostFS.SDK.Client/Pool/RequestInfo.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace FrostFS.SDK.Client; - -// RequestInfo groups info about pool request. -struct RequestInfo -{ - public string Address { get; set; } - - public MethodIndex MethodIndex { get; set; } - - public TimeSpan Elapsed { get; set; } -} - diff --git a/src/FrostFS.SDK.Client/Pool/Sampler.cs b/src/FrostFS.SDK.Client/Pool/Sampler.cs deleted file mode 100644 index 275f3f0..0000000 --- a/src/FrostFS.SDK.Client/Pool/Sampler.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; - -namespace FrostFS.SDK.Client; - -internal sealed class Sampler -{ - private readonly object _lock = new(); - - private Random random = new(); - - internal double[] Probabilities { get; set; } - internal int[] Alias { get; set; } - - internal Sampler(double[] probabilities) - { - var small = new WorkList(); - var large = new WorkList(); - - var n = probabilities.Length; - - // sampler.randomGenerator = rand.New(source) - Probabilities = new double[n]; - Alias = new int[n]; - - // Compute scaled probabilities. - var p = new double[n]; - - for (int i = 0; i < n; i++) - { - p[i] = probabilities[i] * n; - if (p[i] < 1) - small.Add(i); - else - large.Add(i); - } - - while (small.Length > 0 && large.Length > 0) - { - var l = small.Remove(); - var g = large.Remove(); - - Probabilities[l] = p[l]; - Alias[l] = g; - - p[g] = p[g] + p[l] - 1; - - if (p[g] < 1) - small.Add(g); - else - large.Add(g); - } - - while (large.Length > 0) - { - var g = large.Remove(); - Probabilities[g] = 1; - } - - while (small.Length > 0) - { - var l = small.Remove(); - probabilities[l] = 1; - } - } - - internal int Next() - { - var n = Alias.Length; - - int i; - double f; - lock (_lock) - { - i = random.Next(0, n - 1); - f = random.NextDouble(); - } - - if (f < Probabilities[i]) - { - return i; - } - - return Alias[i]; - } -} diff --git a/src/FrostFS.SDK.Client/Pool/Statistic.cs b/src/FrostFS.SDK.Client/Pool/Statistic.cs deleted file mode 100644 index c4977d5..0000000 --- a/src/FrostFS.SDK.Client/Pool/Statistic.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Collections.ObjectModel; - -namespace FrostFS.SDK.Client; - -public sealed class Statistic -{ - public ulong OverallErrors { get; internal set; } - - public Collection Nodes { get; } = []; - - public string[]? CurrentNodes { get; internal set; } -} diff --git a/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs b/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs deleted file mode 100644 index 2156f99..0000000 --- a/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace FrostFS.SDK.Client; - -public class StatusSnapshot() -{ - public ulong AllTime { get; internal set; } - - public ulong AllRequests { get; internal set; } -} diff --git a/src/FrostFS.SDK.Client/Pool/WorkList.cs b/src/FrostFS.SDK.Client/Pool/WorkList.cs deleted file mode 100644 index 7796e86..0000000 --- a/src/FrostFS.SDK.Client/Pool/WorkList.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace FrostFS.SDK.Client; - -internal sealed class WorkList -{ - private readonly List elements = []; - - internal int Length - { - get { return elements.Count; } - } - - internal void Add(int element) - { - elements.Add(element); - } - - internal int Remove() - { - int last = elements.LastOrDefault(); - elements.RemoveAt(elements.Count - 1); - return last; - } -} diff --git a/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs b/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs deleted file mode 100644 index 83ac869..0000000 --- a/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.ObjectModel; -using System.Security.Cryptography; - -using Grpc.Core; -using Grpc.Core.Interceptors; - -using Microsoft.Extensions.Logging; - -namespace FrostFS.SDK.Client; - -[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "")] -public struct WrapperPrm -{ - internal ILogger? Logger { get; set; } - - internal string Address { get; set; } - - internal ECDsa Key { get; set; } - - internal ulong DialTimeout { get; set; } - - internal ulong StreamTimeout { get; set; } - - internal uint ErrorThreshold { get; set; } - - internal Action ResponseInfoCallback { get; set; } - - internal Action PoolRequestInfoCallback { get; set; } - - internal Func GrpcChannelFactory { get; set; } - - internal ulong GracefulCloseOnSwitchTimeout { get; set; } - - internal Action? Callback { get; set; } - - internal Collection? Interceptors { get; set; } -} - diff --git a/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs index 8ad35d5..ebd60de 100644 --- a/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs @@ -20,8 +20,6 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor { var binary = RuleSerializer.Serialize(args.Chain); - var base64 = Convert.ToBase64String(binary); - AddChainRequest request = new() { Body = new() diff --git a/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs index 59783cf..c0f8b1a 100644 --- a/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Security.Cryptography; using System.Threading.Tasks; using FrostFS.Container; @@ -83,7 +82,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService Body = new PutRequest.Types.Body { Container = grpcContainer, - Signature = ClientContext.Key.ECDsaKey.SignRFC6979(grpcContainer) + Signature = ClientContext.Key.SignRFC6979(grpcContainer) } }; @@ -113,8 +112,8 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService { Body = new DeleteRequest.Types.Body { - ContainerId = args.ContainerId.ToMessage(), - Signature = ClientContext.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value) + ContainerId = args.ContainerId.GetContainerID(), + Signature = ClientContext.Key.SignRFC6979(args.ContainerId.GetContainerID().Value) } }; diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index 3ce543d..ea3b727 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -96,7 +96,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() } } @@ -124,7 +124,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() }, Range = new Object.Range @@ -159,7 +159,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() }, Type = ChecksumType.Sha256, @@ -204,7 +204,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), ObjectId = args.ObjectId.ToMessage() } } @@ -231,7 +231,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Body = new SearchRequest.Types.Body { - ContainerId = args.ContainerId.ToMessage(), + ContainerId = args.ContainerId.GetContainerID(), Version = 1 // TODO: clarify this param } }; @@ -296,18 +296,17 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { var chunkSize = args.MaxChunkLength; Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null"); - + var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken); byte[]? chunkBuffer = null; try { - // common chunkBuffer = ArrayPool.Shared.Rent(chunkSize); bool isFirstChunk = true; ulong currentPos = args.Range.Offset; - + var address = new Address { ObjectId = args.Address.ObjectId, @@ -327,11 +326,11 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Body = new() { - Address = address, + Address = address, Patch = new PatchRequest.Types.Body.Types.Patch { Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), - SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } + SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount } } } }; @@ -385,7 +384,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return response.Body.ObjectId.ToModel(); } - + internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { var stream = args.Payload!; @@ -567,7 +566,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl break; sentBytes += bytesCount; - + var chunkRequest = new PutRequest { Body = new PutRequest.Types.Body @@ -622,7 +621,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var initRequest = new PutRequest { Body = new PutRequest.Types.Body - { + { Init = new PutRequest.Types.Body.Types.Init { Header = grpcHeader, diff --git a/src/FrostFS.SDK.Client/Pool/SessionCache.cs b/src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs similarity index 72% rename from src/FrostFS.SDK.Client/Pool/SessionCache.cs rename to src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs index fae3de1..a02436f 100644 --- a/src/FrostFS.SDK.Client/Pool/SessionCache.cs +++ b/src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Concurrent; +using System.Collections.Concurrent; namespace FrostFS.SDK.Client; @@ -36,15 +35,4 @@ internal sealed class SessionCache(ulong sessionExpirationDuration) _cache[key] = value; } } - - internal void DeleteByPrefix(string prefix) - { - foreach (var key in _cache.Keys) - { - if (key.StartsWith(prefix, StringComparison.Ordinal)) - { - _cache.TryRemove(key, out var _); - } - } - } } diff --git a/src/FrostFS.SDK.Client/Tools/ClientContext.cs b/src/FrostFS.SDK.Client/Tools/ClientContext.cs index e2522c9..104cb3d 100644 --- a/src/FrostFS.SDK.Client/Tools/ClientContext.cs +++ b/src/FrostFS.SDK.Client/Tools/ClientContext.cs @@ -39,7 +39,7 @@ public class ClientContext(FrostFSClient client, ClientKey key, FrostFsOwner own { if (sessionKey == null && Key != null && Address != null) { - sessionKey = Pool.FormCacheKey(Address, Key.PublicKey); + sessionKey = $"{Address}{Key}"; } return sessionKey; diff --git a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs index ac5d019..c8c5caf 100644 --- a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs +++ b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs @@ -1,6 +1,6 @@ using System; using System.Linq; - +using System.Security.Cryptography; using FrostFS.Object; using FrostFS.Refs; using FrostFS.SDK.Client.Mappers.GRPC; @@ -44,7 +44,11 @@ public static class ObjectTools if (header.Split != null) SetSplitValues(grpcHeader, header.Split, owner, version, key); - return new ObjectID { Value = grpcHeader.Sha256() }.ToModel(); + using var sha256 = SHA256.Create(); + using HashStream stream = new(sha256); + grpcHeader.WriteTo(stream); + + return new FrostFsObjectId(Base58.Encode(stream.Hash())); } internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx) @@ -75,7 +79,7 @@ public static class ObjectTools var obj = new Object.Object { Header = grpcHeader, - ObjectId = new ObjectID { Value = grpcHeader.Sha256() }, + ObjectId = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcHeader.Sha256()) }, Payload = UnsafeByteOperations.UnsafeWrap(@object.SingleObjectPayload) }; @@ -117,9 +121,9 @@ public static class ObjectTools if (split.ParentHeader is not null) { - var grpcParentHeader = CreateHeader(split.ParentHeader, Array.Empty().Sha256(), owner, version); + var grpcParentHeader = CreateHeader(split.ParentHeader, DataHasher.Sha256([]), owner, version); - grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() }; + grpcHeader.Split.Parent = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcParentHeader.Sha256()) }; grpcHeader.Split.ParentHeader = grpcParentHeader; grpcHeader.Split.ParentSignature = new Signature { @@ -147,21 +151,12 @@ public static class ObjectTools return grpcHeader; } - internal static Checksum Sha256Checksum(byte[] data) - { - return new Checksum - { - Type = ChecksumType.Sha256, - Sum = ByteString.CopyFrom(data.Sha256()) - }; - } - internal static Checksum Sha256Checksum(ReadOnlyMemory data) { return new Checksum { Type = ChecksumType.Sha256, - Sum = ByteString.CopyFrom(data.Sha256()) + Sum = UnsafeByteOperations.UnsafeWrap(DataHasher.Sha256(data)) }; } diff --git a/src/FrostFS.SDK.Client/Tools/RequestSigner.cs b/src/FrostFS.SDK.Client/Tools/RequestSigner.cs index e2aceef..8df1b7c 100644 --- a/src/FrostFS.SDK.Client/Tools/RequestSigner.cs +++ b/src/FrostFS.SDK.Client/Tools/RequestSigner.cs @@ -33,6 +33,7 @@ public static class RequestSigner var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N); var privateKey = new ECPrivateKeyParameters(new BigInteger(1, key.PrivateKey()), ecParameters); var signer = new ECDsaSigner(new HMacDsaKCalculator(digest)); + var hash = new byte[digest.GetDigestSize()]; digest.BlockUpdate(data, 0, data.Length); @@ -54,21 +55,21 @@ public static class RequestSigner return ByteString.CopyFrom(signature); } - internal static SignatureRFC6979 SignRFC6979(this ECDsa key, IMessage message) + internal static SignatureRFC6979 SignRFC6979(this ClientKey key, IMessage message) { return new SignatureRFC6979 { - Key = ByteString.CopyFrom(key.PublicKey()), - Sign = key.SignRFC6979(message.ToByteArray()), + Key = key.PublicKeyProto, + Sign = key.ECDsaKey.SignRFC6979(message.ToByteArray()), }; } - internal static SignatureRFC6979 SignRFC6979(this ECDsa key, ByteString data) + internal static SignatureRFC6979 SignRFC6979(this ClientKey key, ByteString data) { return new SignatureRFC6979 { - Key = ByteString.CopyFrom(key.PublicKey()), - Sign = key.SignRFC6979(data.ToByteArray()), + Key = key.PublicKeyProto, + Sign = key.ECDsaKey.SignRFC6979(data.ToByteArray()), }; } @@ -82,11 +83,11 @@ public static class RequestSigner Span result = stackalloc byte[65]; result[0] = 0x04; - key.SignHash(data.Sha512()).AsSpan().CopyTo(result.Slice(1)); + key.SignHash(DataHasher.Sha512(data)).AsSpan().CopyTo(result.Slice(1)); return ByteString.CopyFrom(result); } - + public static ByteString SignDataByHash(this ECDsa key, byte[] hash) { if (key is null) @@ -112,8 +113,9 @@ public static class RequestSigner Sign = key.ECDsaKey.SignData(ReadOnlyMemory.Empty), }; } - - using HashStream stream = new(); + + using var sha512 = SHA512.Create(); + using HashStream stream = new(sha512); data.WriteTo(stream); var sig = new Signature diff --git a/src/FrostFS.SDK.Client/Tools/Verifier.cs b/src/FrostFS.SDK.Client/Tools/Verifier.cs index c110431..90eb3ce 100644 --- a/src/FrostFS.SDK.Client/Tools/Verifier.cs +++ b/src/FrostFS.SDK.Client/Tools/Verifier.cs @@ -9,61 +9,13 @@ using FrostFS.Session; using Google.Protobuf; -using Org.BouncyCastle.Asn1.Sec; -using Org.BouncyCastle.Crypto.Digests; -using Org.BouncyCastle.Crypto.Parameters; -using Org.BouncyCastle.Crypto.Signers; -using Org.BouncyCastle.Math; - namespace FrostFS.SDK.Client; public static class Verifier { public const int RFC6979SignatureSize = 64; - private static BigInteger[] DecodeSignature(byte[] sig) - { - if (sig.Length != RFC6979SignatureSize) - throw new FormatException($"Wrong signature size, expect={RFC6979SignatureSize}, actual={sig.Length}"); - - var rs = new BigInteger[2]; - rs[0] = new BigInteger(1, sig.AsSpan(0, 32).ToArray()); - rs[1] = new BigInteger(1, sig.AsSpan(32).ToArray()); - - return rs; - } - - public static bool VerifyRFC6979(this byte[] publicKey, byte[] data, byte[] sig) - { - if (publicKey is null || data is null || sig is null) - return false; - - var rs = DecodeSignature(sig); - var digest = new Sha256Digest(); - var signer = new ECDsaSigner(new HMacDsaKCalculator(digest)); - var secp256R1 = SecNamedCurves.GetByName("secp256r1"); - var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N); - var bcPublicKey = new ECPublicKeyParameters(secp256R1.Curve.DecodePoint(publicKey), ecParameters); - var hash = new byte[digest.GetDigestSize()]; - - digest.BlockUpdate(data, 0, data.Length); - digest.DoFinal(hash, 0); - signer.Init(false, bcPublicKey); - - return signer.VerifySignature(hash, rs[0], rs[1]); - } - - public static bool VerifyRFC6979(this SignatureRFC6979 signature, IMessage message) - { - if (signature is null) - { - throw new ArgumentNullException(nameof(signature)); - } - - return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray()); - } - - public static bool VerifyData(this ECDsa key, ReadOnlyMemory data, byte[] sig) + public static bool VerifyData(this ECDsa key, IMessage data, ByteString sig) { if (key is null) throw new ArgumentNullException(nameof(key)); @@ -71,7 +23,18 @@ public static class Verifier if (sig is null) throw new ArgumentNullException(nameof(sig)); - return key.VerifyHash(data.Sha512(), sig.AsSpan(1).ToArray()); + var signature = sig.Span.Slice(1).ToArray(); + using var sha = SHA512.Create(); + + if (data is null) + { + return key.VerifyHash(DataHasher.Sha512(new Span([])), signature); + } + + using var stream = new HashStream(sha); + data.WriteTo(stream); + + return key.VerifyHash(stream.Hash(), signature); } public static bool VerifyMessagePart(this Signature sig, IMessage data) @@ -80,9 +43,8 @@ public static class Verifier return false; using var key = sig.Key.ToByteArray().LoadPublicKey(); - var data2Verify = data is null ? [] : data.ToByteArray(); - return key.VerifyData(data2Verify, sig.Sign.ToByteArray()); + return key.VerifyData(data, sig.Sign); } internal static bool VerifyMatryoskaLevel(IMessage body, IMetaHeader meta, IVerificationHeader verification) diff --git a/src/FrostFS.SDK.Cryptography/ArrayHelper.cs b/src/FrostFS.SDK.Cryptography/ArrayHelper.cs index 00c356d..c88134d 100644 --- a/src/FrostFS.SDK.Cryptography/ArrayHelper.cs +++ b/src/FrostFS.SDK.Cryptography/ArrayHelper.cs @@ -21,4 +21,18 @@ internal static class ArrayHelper return dst; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void GetRevertedArray(ReadOnlySpan source, byte[] data) + { + if (source.Length != 0) + { + int i = 0; + int j = source.Length - 1; + while (i < source.Length) + { + data[i++] = source[j--]; + } + } + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index c03d604..9f33f25 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Cryptography")] -[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyFileVersion("1.0.4.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] -[assembly: AssemblyVersion("1.0.3.0")] +[assembly: AssemblyVersion("1.0.4.0")] diff --git a/src/FrostFS.SDK.Cryptography/Base58.cs b/src/FrostFS.SDK.Cryptography/Base58.cs index 636eb0a..0026bc3 100644 --- a/src/FrostFS.SDK.Cryptography/Base58.cs +++ b/src/FrostFS.SDK.Cryptography/Base58.cs @@ -19,19 +19,20 @@ public static class Base58 if (buffer.Length < 4) throw new FormatException(); - var check = buffer.AsSpan(0, buffer.Length - 4).ToArray(); - byte[] checksum = check.Sha256().Sha256(); + var check = buffer.AsSpan(0, buffer.Length - 4); + byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(check).AsSpan()); if (!buffer.AsSpan(buffer.Length - 4).SequenceEqual(checksum.AsSpan(0, 4))) throw new FormatException(); + var result = check.ToArray(); Array.Clear(buffer, 0, buffer.Length); - return check; + return result; } - public static string Base58CheckEncode(this ReadOnlySpan data) + public static string Base58CheckEncode(this Span data) { - byte[] checksum = data.ToArray().Sha256().Sha256(); + byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan()); ; Span buffer = stackalloc byte[data.Length + 4]; data.CopyTo(buffer); @@ -59,10 +60,9 @@ public static class Base58 } int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count(); - var leadingZeros = new byte[leadingZeroCount]; if (bi.IsZero) - return leadingZeros; + return new byte[leadingZeroCount]; var bytesBigEndian = bi.ToByteArray().Reverse().ToArray(); @@ -73,12 +73,26 @@ public static class Base58 var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray(); - return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros); + var result = new byte[leadingZeroCount + bytesBigEndian.Length - firstNonZeroIndex]; + + int p = 0; + while (p < leadingZeroCount) + result[p++] = 0; + + for (int j = firstNonZeroIndex; j < bytesBigEndian.Length; j++) + result[p++] = bytesBigEndian[j]; + + return result; } public static string Encode(ReadOnlySpan input) { - var data = input.ToArray().Reverse().Concat(new byte[] { 0 }).ToArray(); + var data = new byte[input.Length + 1]; + + ArrayHelper.GetRevertedArray(input, data); + + data[input.Length] = 0; + BigInteger value = new(data); // Encode BigInteger to Base58 string diff --git a/src/FrostFS.SDK.Cryptography/DataHasher.cs b/src/FrostFS.SDK.Cryptography/DataHasher.cs new file mode 100644 index 0000000..fceeeaa --- /dev/null +++ b/src/FrostFS.SDK.Cryptography/DataHasher.cs @@ -0,0 +1,121 @@ +using System; +using System.Buffers; +using System.Security.Cryptography; + +namespace FrostFS.SDK.Cryptography; + +public static class DataHasher +{ + private const int LargeBlockSize = 1024 * 1024; + private const int SmallBlockSize = 256; + + public static byte[] Hash(this ReadOnlyMemory bytes, HashAlgorithm algorithm) + { + if (algorithm is null) + { + throw new ArgumentNullException(nameof(algorithm)); + } + + if (bytes.Length == 0) + { + return algorithm.ComputeHash([]); + } + + int rest, pos = 0; + + var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize; + + byte[] buffer = ArrayPool.Shared.Rent(blockSize); + + try + { + while ((rest = bytes.Length - pos) > 0) + { + var size = Math.Min(rest, blockSize); + + bytes.Slice(pos, size).CopyTo(buffer); + + algorithm.TransformBlock(buffer, 0, size, buffer, 0); + + pos += size; + } + + algorithm.TransformFinalBlock([], 0, 0); + return algorithm.Hash; + } + finally + { + if (buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + public static byte[] Hash(this ReadOnlySpan bytes, HashAlgorithm algorithm) + { + if (algorithm is null) + { + throw new ArgumentNullException(nameof(algorithm)); + } + + if (bytes.Length == 0) + { + return algorithm.ComputeHash([]); + } + + int rest, pos = 0; + + var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize; + + byte[] buffer = ArrayPool.Shared.Rent(blockSize); + + try + { + while ((rest = bytes.Length - pos) > 0) + { + var size = Math.Min(rest, blockSize); + + bytes.Slice(pos, size).CopyTo(buffer); + + algorithm.TransformBlock(buffer, 0, size, buffer, 0); + + pos += size; + } + + algorithm.TransformFinalBlock([], 0, 0); + return algorithm.Hash; + } + finally + { + if (buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + public static byte[] Sha256(ReadOnlyMemory value) + { + using SHA256 sha = SHA256.Create(); + return Hash(value, sha); + } + + public static byte[] Sha256(ReadOnlySpan value) + { + using SHA256 sha = SHA256.Create(); + return Hash(value, sha); + } + + public static byte[] Sha512(ReadOnlyMemory value) + { + using SHA512 sha = SHA512.Create(); + return Hash(value, sha); + } + + public static byte[] Sha512(ReadOnlySpan value) + { + using SHA512 sha = SHA512.Create(); + return Hash(value, sha); + } +} diff --git a/src/FrostFS.SDK.Cryptography/Extentions.cs b/src/FrostFS.SDK.Cryptography/Extentions.cs index 40ad6f0..5f4fdd9 100644 --- a/src/FrostFS.SDK.Cryptography/Extentions.cs +++ b/src/FrostFS.SDK.Cryptography/Extentions.cs @@ -1,20 +1,9 @@ -using System; -using System.IO; -using System.Security.Cryptography; -using System.Threading; -using CommunityToolkit.HighPerformance; using Org.BouncyCastle.Crypto.Digests; namespace FrostFS.SDK.Cryptography; public static class Extentions { - private static readonly SHA256 _sha256 = SHA256.Create(); - private static SpinLock _spinlockSha256; - - private static readonly SHA512 _sha512 = SHA512.Create(); - private static SpinLock _spinlockSha512; - internal static byte[] RIPEMD160(this byte[] value) { var hash = new byte[20]; @@ -24,72 +13,4 @@ public static class Extentions digest.DoFinal(hash, 0); return hash; } - - public static byte[] Sha256(this byte[] value) - { - bool lockTaken = false; - try - { - _spinlockSha256.Enter(ref lockTaken); - - return _sha256.ComputeHash(value); - } - finally - { - if (lockTaken) - { - _spinlockSha256.Exit(false); - } - } - } - - public static byte[] Sha256(this ReadOnlyMemory value) - { - bool lockTaken = false; - try - { - _spinlockSha256.Enter(ref lockTaken); - - return _sha256.ComputeHash(value.AsStream()); - } - finally - { - if (lockTaken) - { - _spinlockSha256.Exit(false); - } - } - } - - public static byte[] Sha512(this ReadOnlyMemory value) - { - bool lockTaken = false; - try - { - _spinlockSha512.Enter(ref lockTaken); - - return _sha512.ComputeHash(value.AsStream()); - } - finally - { - if (lockTaken) - _spinlockSha512.Exit(false); - } - } - - public static byte[] Sha512(this Stream stream) - { - bool lockTaken = false; - try - { - _spinlockSha512.Enter(ref lockTaken); - - return _sha512.ComputeHash(stream); - } - finally - { - if (lockTaken) - _spinlockSha512.Exit(false); - } - } } diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index e4c7824..9dfd370 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -30,7 +30,6 @@ - all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/FrostFS.SDK.Cryptography/HashStream.cs b/src/FrostFS.SDK.Cryptography/HashStream.cs index d2ede1a..2893f26 100644 --- a/src/FrostFS.SDK.Cryptography/HashStream.cs +++ b/src/FrostFS.SDK.Cryptography/HashStream.cs @@ -3,11 +3,11 @@ using System.Security.Cryptography; namespace FrostFS.SDK.Cryptography; -public sealed class HashStream() : Stream +public sealed class HashStream(HashAlgorithm algorithm) : Stream { private long position; - private readonly SHA512 _hash = SHA512.Create(); + private readonly HashAlgorithm _hash = algorithm; public override bool CanRead => false; diff --git a/src/FrostFS.SDK.Cryptography/Key.cs b/src/FrostFS.SDK.Cryptography/Key.cs index 072f155..5f382fa 100644 --- a/src/FrostFS.SDK.Cryptography/Key.cs +++ b/src/FrostFS.SDK.Cryptography/Key.cs @@ -16,7 +16,7 @@ public static class KeyExtension private const int UncompressedPublicKeyLength = 65; private static readonly uint CheckSigDescriptor = - BinaryPrimitives.ReadUInt32LittleEndian(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").Sha256()); + BinaryPrimitives.ReadUInt32LittleEndian(DataHasher.Sha256(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").AsSpan())); public static byte[] Compress(this byte[] publicKey) { @@ -60,10 +60,15 @@ public static class KeyExtension $"expected length={CompressedPublicKeyLength}, actual={publicKey.Length}" ); - var script = new byte[] { 0x0c, CompressedPublicKeyLength }; //PUSHDATA1 33 - script = ArrayHelper.Concat(script, publicKey); - script = ArrayHelper.Concat(script, [0x41]); //SYSCALL - script = ArrayHelper.Concat(script, BitConverter.GetBytes(CheckSigDescriptor)); //Neo_Crypto_CheckSig + var signDescriptor = BitConverter.GetBytes(CheckSigDescriptor); + + var script = new byte[3 + publicKey.Length + signDescriptor.Length]; + + script[0] = 0x0c; + script[1] = CompressedPublicKeyLength; //PUSHDATA1 33 + Buffer.BlockCopy(publicKey, 0, script, 2, publicKey.Length); + script[publicKey.Length + 2] = 0x41; //SYSCALL + Buffer.BlockCopy(signDescriptor, 0, script, publicKey.Length + 3, signDescriptor.Length); return script; } @@ -74,7 +79,7 @@ public static class KeyExtension throw new ArgumentNullException(nameof(publicKey)); var script = publicKey.CreateSignatureRedeemScript(); - return script.Sha256().RIPEMD160(); + return DataHasher.Sha256(script.AsSpan()).RIPEMD160(); } private static string ToAddress(this byte[] scriptHash, byte version) @@ -102,11 +107,6 @@ public static class KeyExtension return privateKey; } - public static string Address(this ECDsa key) - { - return key.PublicKey().PublicKeyToAddress(); - } - public static string PublicKeyToAddress(this byte[] publicKey) { if (publicKey == null) @@ -132,10 +132,12 @@ public static class KeyExtension var pos = 33 - param.Q.X.Length; param.Q.X.CopyTo(pubkey, pos); - if (new BigInteger(param.Q.Y.Reverse().Concat(new byte[] { 0x0 }).ToArray()).IsEven) - pubkey[0] = 0x2; - else - pubkey[0] = 0x3; + + var y = new byte[33]; + ArrayHelper.GetRevertedArray(param.Q.Y, y); + y[32] = 0; + + pubkey[0] = new BigInteger(y).IsEven ? (byte)0x2 : (byte)0x3; return pubkey; } @@ -152,7 +154,9 @@ public static class KeyExtension { var secp256R1 = SecNamedCurves.GetByName("secp256r1"); var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)) - .GetEncoded(false).Skip(1).ToArray(); + .GetEncoded(false) + .Skip(1) + .ToArray(); var key = ECDsa.Create(new ECParameters { diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index 11ace79..c3c813c 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Protos")] -[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyFileVersion("1.0.4.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] -[assembly: AssemblyVersion("1.0.3.0")] +[assembly: AssemblyVersion("1.0.4.0")] diff --git a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs index 0a3aec6..58a9d40 100644 --- a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs @@ -3,7 +3,6 @@ using System.Security.Cryptography; using FrostFS.Object; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; using FrostFS.Session; using Google.Protobuf; @@ -43,7 +42,7 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader) Signature = new Refs.Signature { Key = Key.PublicKeyProto, - Sign = Key.ECDsaKey. SignData(header.ToByteArray()), + Sign = Key.ECDsaKey.SignData(header.ToByteArray()), } } }, diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs index e6f25c2..eafdda5 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs @@ -1,5 +1,3 @@ -using System.Security.Cryptography; - using FrostFS.Container; using FrostFS.Object; using FrostFS.SDK.Client; diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs index 0ad6ab0..7491950 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs @@ -23,13 +23,16 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) var grpcVersion = Version.ToMessage(); + Span ContainerGuidSpan = stackalloc byte[16]; + ContainerGuid.ToBytes(ContainerGuidSpan); + PutResponse putResponse = new() { Body = new PutResponse.Types.Body { ContainerId = new ContainerID { - Value = ByteString.CopyFrom(ContainerGuid.ToBytes()) + Value = ByteString.CopyFrom(ContainerGuidSpan) } }, MetaHeader = new ResponseMetaHeader @@ -69,7 +72,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) Container = new Container.Container { Version = grpcVersion, - Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()), + Nonce = ByteString.CopyFrom(ContainerGuidSpan), PlacementPolicy = PlacementPolicy.GetPolicy() } }, diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index 49a0f6d..f33b9a1 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -4,7 +4,6 @@ using System.Security.Cryptography; using FrostFS.Object; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; using Google.Protobuf; diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs index 37a8c13..04a8b2c 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs @@ -40,9 +40,9 @@ public class ContainerTests : SmokeTestsBase { var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); await Cleanup(client); - + client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - + FrostFsContainerId containerId = await CreateContainer(client, ctx: default, token: null, @@ -50,22 +50,25 @@ public class ContainerTests : SmokeTestsBase backupFactor: 1, selectors: [], filter: [], - containerAttributes: [new ("testKey1", "testValue1")], + containerAttributes: [new("testKey1", "testValue1")], new FrostFsReplica(3)); Assert.NotNull(containerId); var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default); - + Assert.NotNull(container); Assert.NotNull(container.Attributes); - Assert.Equal(2, container.Attributes.Count); Assert.Equal("testKey1", container.Attributes[0].Key); Assert.Equal("testValue1", container.Attributes[0].Value); - Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key); - Assert.Equal("true", container.Attributes[1].Value); - + + //Assert.Equal("true", container.Attributes[1].Value); + + // for cluster + //Assert.Equal(2, container.Attributes.Count); + //Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key); + Assert.True(container.PlacementPolicy.HasValue); Assert.Equal(1u, container.PlacementPolicy.Value.BackupFactor); @@ -82,7 +85,7 @@ public class ContainerTests : SmokeTestsBase Assert.Equal(OwnerId!.ToString(), container.Owner!.Value); Assert.NotNull(container.Version); - + Assert.Equal(Version!.Major, container.Version.Major); Assert.Equal(Version.Minor, container.Version.Minor); } @@ -101,8 +104,8 @@ public class ContainerTests : SmokeTestsBase new ("filter2", "filterKey2", 2, "testValue2", [new ("subFilter2", "subFilterKey2", 3, "testValue3",[])]) ]; - Collection selectors = [ - new ("selector1") { + Collection selectors = [ + new ("selector1") { Count = 1, Clause = 1, Attribute = "attribute1", @@ -129,19 +132,19 @@ public class ContainerTests : SmokeTestsBase Assert.NotNull(containerId); var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default); - + Assert.NotNull(container); Assert.NotNull(container.Attributes); - Assert.Single(container.Attributes); - Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key); - Assert.Equal("true", container.Attributes[0].Value); + //Assert.Single(container.Attributes); + //Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key); + //Assert.Equal("true", container.Attributes[0].Value); Assert.True(container.PlacementPolicy.HasValue); Assert.Equal(2u, container.PlacementPolicy.Value.BackupFactor); Assert.False(container.PlacementPolicy.Value.Unique); - + Assert.NotEmpty(container.PlacementPolicy.Value.Selectors); Assert.Equal(2, container.PlacementPolicy.Value.Selectors.Count); @@ -196,7 +199,7 @@ public class ContainerTests : SmokeTestsBase Assert.Equal(OwnerId!.ToString(), container.Owner!.Value); Assert.NotNull(container.Version); - + Assert.Equal(Version!.Major, container.Version.Major); Assert.Equal(Version.Minor, container.Version.Minor); } @@ -233,12 +236,12 @@ public class ContainerTests : SmokeTestsBase }); } - #pragma warning disable xUnit1031 // Timeout is used +#pragma warning disable xUnit1031 // Timeout is used if (!Task.WaitAll(tasks, 20000)) { Assert.Fail("cannot create containers"); } - #pragma warning restore xUnit1031 +#pragma warning restore xUnit1031 var containers = client.ListContainersAsync(new PrmContainerGetAll(), default); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs index f1fce3c..a3d25af 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs @@ -32,8 +32,10 @@ public class SmokeClientTests : SmokeTestsBase Assert.Equal(13, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); - Assert.Equal(2, result.Addresses.Count); - Assert.Equal(11, result.Attributes.Count); + Assert.Single(result.Addresses); + Assert.Equal(9, result.Attributes.Count); + //Assert.Equal(2, result.Addresses.Count); + //Assert.Equal(11, result.Attributes.Count); } [Fact] @@ -65,13 +67,13 @@ public class SmokeClientTests : SmokeTestsBase Assert.True(result.HomomorphicHashingDisabled); Assert.True(result.MaintenanceModeAllowed); Assert.True(0u < result.MagicNumber); - + Assert.Equal(0u, result.AuditFee); Assert.Equal(0u, result.BasicIncomeRate); Assert.Equal(0u, result.ContainerAliasFee); Assert.Equal(0u, result.ContainerFee); Assert.Equal(75u, result.EpochDuration); - Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee); + Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee); Assert.Equal(12u, result.MaxECDataCount); Assert.Equal(4u, result.MaxECParityCount); Assert.Equal(5242880u, result.MaxObjectSize); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs index 03a9169..a0f501f 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs @@ -58,7 +58,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase int[] objectSizes = [1, 257, 5 * 1024 * 1024, 20 * 1024 * 1024]; string[] objectTypes = [clientCut, serverCut, singleObject]; - + foreach (var objectSize in objectSizes) { _testOutputHelper.WriteLine($"test set for object size {objectSize}"); @@ -157,7 +157,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.NotNull(x); Assert.True(x.Length > 0); - // Assert.True(expectedHash.SequenceEqual(h.ToArray())); + // Assert.True(expectedHash.SequenceEqual(h.ToArray())); } } @@ -201,7 +201,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId) { - if (bytes.Length < 1024 + 64 || bytes.Length > 5900) + if (bytes.Length < 1024 + 64 || bytes.Length > 5900) return; var patch = new byte[1024]; diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs deleted file mode 100644 index 831f37e..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Smoke; - -public abstract class MultiThreadTestsBase -{ - private TestNodeInfo[] nodes; - - protected CallContext? Ctx { get; } - - protected MultiThreadTestsBase() - { - nodes = new TestNodeInfo[4]; - - nodes[0] = new(new Uri(""), ""); - nodes[1] = new(new Uri(""), ""); - nodes[2] = new(new Uri(""), ""); - nodes[3] = new(new Uri(""), ""); - } -} - -public class TestNodeInfo -{ - internal Uri Uri; - - protected ECDsa? Key { get; } - - protected FrostFsOwner? OwnerId { get; } - - protected FrostFsVersion? Version { get; } - - public TestNodeInfo(Uri uri, string keyString) - { - Uri = uri; - Key = keyString.LoadWif(); - OwnerId = FrostFsOwner.FromKey(Key); - Version = new FrostFsVersion(2, 13); - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs deleted file mode 100644 index 194ef98..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs +++ /dev/null @@ -1,544 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Smoke; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] -public class MultithreadPoolSmokeTests : SmokeTestsBase -{ - private InitParameters GetDefaultParams() - { - return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [new(1, url, 100.0f)], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null - }; - } - - [Fact] - public async void NetworkMapTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNetmapSnapshotAsync(default); - - Assert.True(result.Epoch > 0); - Assert.Single(result.NodeInfoCollection); - - var item = result.NodeInfoCollection[0]; - Assert.Equal(2, item.Version.Major); - Assert.Equal(13, item.Version.Minor); - Assert.Equal(NodeState.Online, item.State); - Assert.True(item.PublicKey.Length > 0); - Assert.Single(item.Addresses); - Assert.Equal(9, item.Attributes.Count); - } - - [Fact] - public async void NodeInfoTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } - - [Fact] - public async void NodeInfoStatisticsTwoNodesTest() - { - var callbackText = string.Empty; - - var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [ - new(1, url, 100.0f), - new(2, url.Replace('0', '1'), 100.0f) - ], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null, - Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds" - }; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - var statistics = pool.Statistic(); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void NodeInfoStatisticsTest() - { - var options = GetDefaultParams(); - - var callbackText = string.Empty; - options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void GetSessionTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var prm = new PrmSessionCreate(100); - - var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true); - - var ownerHash = Base58.Decode(OwnerId!.Value); - - Assert.NotNull(token); - Assert.NotEqual(Guid.Empty, token.Id); - Assert.Equal(33, token.SessionKey.Length); - } - - [Fact] - public async void CreateObjectWithSessionToken() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["key1", "value1"]); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = GetRandomBytes(1024); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - } - - [Fact] - public async void FilterTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = new byte[] { 1, 2, 3 }; - - var ParentHeader = new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular) - { - PayloadLength = 3 - }; - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")], - new FrostFsSplit())); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId, false), default); - - var ecdsaKey = keyString.LoadWif(); - - var networkInfo = await pool.GetNetmapSnapshotAsync(default); - - await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); - - await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); - - await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); - - await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); - - await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); - - await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); - - await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); - - await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); - - var checkSum = CheckSum.CreateCheckSum(bytes); - - await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); - - await CheckFilter(pool, containerId, new FilterByPhysicallyStored()); - } - - private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter) - { - var resultObjectsCount = 0; - - PrmObjectSearch searchParam = new(containerId, null, [], filter); - - await foreach (var objId in pool.SearchObjectsAsync(searchParam, default)) - { - resultObjectsCount++; - var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); - } - - Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioTest(int objectSize) - { - bool callbackInvoked = false; - - var options = GetDefaultParams(); - - options.Callback = new((cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await pool.PutContainerAsync(createContainerParam, default); - - var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - Assert.True(callbackInvoked); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes!); - Assert.Equal("fileName", objHeader.Attributes!.First().Key); - Assert.Equal("test", objHeader.Attributes!.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioWithSessionTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0)); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - await Cleanup(pool); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams); - - var container = await pool.PutContainerAsync(createContainerParam, ctx); - - var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx); - Assert.NotNull(containerInfo); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: container, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB - [InlineData(64 * 1024 * 1024 - 1)] - [InlineData(64 * 1024 * 1024 + 1)] - [InlineData(2 * 64 * 1024 * 1024 + 256)] - [InlineData(200)] - public async void ClientCutScenarioTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var ctx = new CallContext(TimeSpan.FromSeconds(10)); - - // ctx.Interceptors.Add(new CallbackInterceptor()); - - var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx); - - Assert.NotNull(container); - - byte[] bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectClientCutPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - payload: new MemoryStream(bytes)); - - var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes[0].Key); - Assert.Equal("test", objHeader.Attributes[0].Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await CheckFilter(pool, containerId, new FilterByRootObject()); - - await Cleanup(pool); - - await foreach (var cid in pool.ListContainersAsync(default, default)) - { - Assert.Fail($"Container {cid.GetValue()} exist"); - } - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs deleted file mode 100644 index 7de8155..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs +++ /dev/null @@ -1,684 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Client.Interfaces; -using FrostFS.SDK.Cryptography; -using FrostFS.SDK.SmokeTests; - -namespace FrostFS.SDK.Tests.Smoke; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] -public class MultithreadSmokeClientTests : SmokeTestsBase -{ - [Fact] - public async void AccountTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var result = await client.GetBalanceAsync(default); - - Assert.NotNull(result); - Assert.True(result.Value == 0); - } - - [Fact] - public async void NetworkMapTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var result = await client.GetNetmapSnapshotAsync(default); - - Assert.True(result.Epoch > 0); - Assert.Single(result.NodeInfoCollection); - - var item = result.NodeInfoCollection[0]; - Assert.Equal(2, item.Version.Major); - Assert.Equal(13, item.Version.Minor); - Assert.Equal(NodeState.Online, item.State); - Assert.True(item.PublicKey.Length > 0); - Assert.Single(item.Addresses); - Assert.Equal(9, item.Attributes.Count); - } - - - [Fact] - public async void NodeInfoTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var result = await client.GetNodeInfoAsync(default); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } - - [Fact] - public async void NodeInfoStatisticsTest() - { - var options = ClientOptions; - - var callbackContent = string.Empty; - options.Value.Callback = (cs) => callbackContent = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - var result = await client.GetNodeInfoAsync(default); - - Assert.NotEmpty(callbackContent); - } - - [Fact] - public async void GetSessionTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - var token = await client.CreateSessionAsync(new(100), default); - - Assert.NotNull(token); - Assert.NotEqual(Guid.Empty, token.Id); - Assert.Equal(33, token.SessionKey.Length); - } - - [Fact] - public async void CreateObjectWithSessionToken() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["key1", "value1"]); - - var containerId = await client.PutContainerAsync(createContainerParam, default); - - var bytes = GetRandomBytes(1024); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default) - .ConfigureAwait(true); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk().ConfigureAwait(true)) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client).ConfigureAwait(true); - } - - [Fact] - public async void FilterTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await client.PutContainerAsync(createContainerParam, default); - - var bytes = new byte[] { 1, 2, 3 }; - - var ParentHeader = new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular) - { - PayloadLength = 3 - }; - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")], - new FrostFsSplit())); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var head = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var ecdsaKey = keyString.LoadWif(); - - var networkInfo = await client.GetNetmapSnapshotAsync(default); - - await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); - - await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); - - await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); - - await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); - - await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); - - await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); - - await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); - - await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); - - var checkSum = CheckSum.CreateCheckSum(bytes); - - await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); - - await CheckFilter(client, containerId, new FilterByPhysicallyStored()); - } - - private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter) - { - var resultObjectsCount = 0; - - PrmObjectSearch searchParam = new(containerId, null, [], filter); - - await foreach (var objId in client.SearchObjectsAsync(searchParam, default)) - { - resultObjectsCount++; - var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); - } - - Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioTest(int objectSize) - { - bool callbackInvoked = false; - - var options = ClientOptions; - - options.Value.Callback = new((cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }); - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - await Cleanup(client); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, ctx); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - Assert.True(callbackInvoked); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) - { - hasObject = true; - - var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Fact] - public async void PatchTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, default); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - - var bytes = new byte[1024]; - for (int i = 0; i < 1024; i++) - { - bytes[i] = 31; - } - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var patch = new byte[16]; - for (int i = 0; i < 16; i++) - { - patch[i] = 32; - } - - var range = new FrostFsRange(8, (ulong)patch.Length); - - var patchParams = new PrmObjectPatch( - new FrostFsAddress(createdContainer, objectId), - payload: new MemoryStream(patch), - maxChunkLength: 32, - range: range); - - var newIbjId = await client.PatchObjectAsync(patchParams, default); - - var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - for (int i = 0; i < (int)range.Offset; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); - - var rangeEnd = range.Offset + range.Length; - - for (int i = (int)range.Offset; i < (int)rangeEnd; i++) - Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); - - for (int i = (int)rangeEnd; i < bytes.Length; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Fact] - public async void RangeTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, default); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - - var bytes = new byte[256]; - for (int i = 0; i < 256; i++) - { - bytes[i] = (byte)i; - } - - var param = new PrmObjectPut( - new FrostFsObjectHeader(containerId: createdContainer, type: FrostFsObjectType.Regular)); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64)); - - var rangeReader = await client.GetRangeAsync(rangeParam, default); - - var downloadedBytes = new byte[rangeParam.Range.Length]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await rangeReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Fact] - public async void RangeHashTest() - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await client.PutContainerAsync(createContainerParam, default); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - - var bytes = new byte[256]; - for (int i = 0; i < 256; i++) - { - bytes[i] = (byte)i; - } - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular)); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [new FrostFsRange(100, 64)], bytes); - - var hashes = await client.GetRangeHashAsync(rangeParam, default); - - foreach (var hash in hashes) - { - var x = hash[..32].ToArray(); - } - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioWithSessionTest(int objectSize) - { - var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - //Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0)) - var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - await Cleanup(client); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams); - - var container = await client.PutContainerAsync(createContainerParam, ctx); - - var containerInfo = await client.GetContainerAsync(new PrmContainerGet(container), ctx); - Assert.NotNull(containerInfo); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: container, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync( - new PrmObjectSearch(container, token, [], filter), default)) - { - hasObject = true; - - var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await client.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB - [InlineData(64 * 1024 * 1024 - 1)] - [InlineData(64 * 1024 * 1024 + 1)] - [InlineData(2 * 64 * 1024 * 1024 + 256)] - [InlineData(200)] - public async void ClientCutScenarioTest(int objectSize) - { - var options = ClientOptions; - options.Value.Interceptors.Add(new CallbackInterceptor()); - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await client.PutContainerAsync(createContainerParam, default); - - var ctx = new CallContext(TimeSpan.FromSeconds(10)); - - var container = await client.GetContainerAsync(new PrmContainerGet(containerId), ctx); - - Assert.NotNull(container); - - byte[] bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectClientCutPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - payload: new MemoryStream(bytes)); - - var objectId = await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) - { - hasObject = true; - - var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes[0].Key); - Assert.Equal("test", objHeader.Attributes[0].Value); - } - - Assert.True(hasObject); - - var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await CheckFilter(client, containerId, new FilterByRootObject()); - - await Cleanup(client); - - var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5)); - - IAsyncEnumerator? enumerator = null; - do - { - if (deadline <= DateTime.UtcNow) - { - Assert.Fail("Containers exist"); - break; - } - - enumerator = client.ListContainersAsync(default, default).GetAsyncEnumerator(); - await Task.Delay(500); - } - while (await enumerator!.MoveNextAsync()); - } - - [Fact] - public async void NodeInfoCallbackAndInterceptorTest() - { - bool callbackInvoked = false; - bool intercepterInvoked = false; - - var options = ClientOptions; - options.Value.Callback = (cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }; - - options.Value.Interceptors.Add(new CallbackInterceptor(s => intercepterInvoked = true)); - - var client = FrostFSClient.GetInstance(options, GrpcChannel); - - var result = await client.GetNodeInfoAsync(default); - - Assert.True(callbackInvoked); - Assert.True(intercepterInvoked); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs deleted file mode 100644 index 9e7ea80..0000000 --- a/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs +++ /dev/null @@ -1,546 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; - -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Smoke; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] -public class PoolSmokeTests : SmokeTestsBase -{ - private InitParameters GetDefaultParams() - { - return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [new(1, url, 100.0f)], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null - }; - } - - [Fact] - public async void NetworkMapTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNetmapSnapshotAsync(default); - - Assert.True(result.Epoch > 0); - Assert.Single(result.NodeInfoCollection); - - var item = result.NodeInfoCollection[0]; - Assert.Equal(2, item.Version.Major); - Assert.Equal(13, item.Version.Minor); - Assert.Equal(NodeState.Online, item.State); - Assert.True(item.PublicKey.Length > 0); - Assert.Single(item.Addresses); - Assert.Equal(9, item.Attributes.Count); - } - - [Fact] - public async void NodeInfoTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); - Assert.Equal(NodeState.Online, result.State); - Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - } - - [Fact] - public async void NodeInfoStatisticsTwoNodesTest() - { - var callbackText = string.Empty; - - var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) - { - Key = keyString.LoadWif(), - NodeParams = [ - new(1, url, 100.0f), - new(2, url.Replace('0', '1'), 100.0f) - ], - ClientBuilder = null, - GracefulCloseOnSwitchTimeout = 30_000_000, - Logger = null, - Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds" - }; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - var statistics = pool.Statistic(); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void NodeInfoStatisticsTest() - { - var options = GetDefaultParams(); - - var callbackText = string.Empty; - options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; - - var pool = new Pool(options); - - var ctx = new CallContext(TimeSpan.Zero); - - var error = await pool.Dial(ctx).ConfigureAwait(true); - - Assert.Null(error); - - var result = await pool.GetNodeInfoAsync(default); - - Assert.False(string.IsNullOrEmpty(callbackText)); - Assert.Contains(" took ", callbackText, StringComparison.Ordinal); - } - - [Fact] - public async void GetSessionTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var prm = new PrmSessionCreate(100); - - var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true); - - var ownerHash = Base58.Decode(OwnerId!.Value); - - Assert.NotNull(token); - Assert.NotEqual(Guid.Empty, token.Id); - Assert.Equal(33, token.SessionKey.Length); - } - - [Fact] - public async void CreateObjectWithSessionToken() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["key1", "value1"]); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = GetRandomBytes(1024); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - } - - [Fact] - public async void FilterTest() - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var bytes = new byte[] { 1, 2, 3 }; - - var ParentHeader = new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular) - { - PayloadLength = 3 - }; - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")], - new FrostFsSplit())); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var ecdsaKey = keyString.LoadWif(); - - var networkInfo = await pool.GetNetmapSnapshotAsync(default); - - await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); - - await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); - - await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); - - await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); - - await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); - - await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); - - await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); - - await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); - - var checkSum = CheckSum.CreateCheckSum(bytes); - - await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); - - await CheckFilter(pool, containerId, new FilterByPhysicallyStored()); - } - - private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter) - { - var resultObjectsCount = 0; - - PrmObjectSearch searchParam = new(containerId, null, [], filter); - - await foreach (var objId in pool.SearchObjectsAsync(searchParam, default)) - { - resultObjectsCount++; - var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); - } - - Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioTest(int objectSize) - { - bool callbackInvoked = false; - - var options = GetDefaultParams(); - - options.Callback = new((cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var ctx = new CallContext(TimeSpan.Zero); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams, - xheaders: ["testKey", "testValue"]); - - var createdContainer = await pool.PutContainerAsync(createContainerParam, ctx); - - var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default); - Assert.NotNull(container); - Assert.True(callbackInvoked); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")])); - - var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes!); - Assert.Equal("fileName", objHeader.Attributes!.First().Key); - Assert.Equal("test", objHeader.Attributes!.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB - [InlineData(6 * 1024 * 1024 + 100)] - public async void SimpleScenarioWithSessionTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0)); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); - - await Cleanup(pool); - - var ctx = new CallContext(TimeSpan.FromSeconds(20)); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - PrmWait.DefaultParams); - - var container = await pool.PutContainerAsync(createContainerParam, ctx); - - var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx); - Assert.NotNull(containerInfo); - - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut( - new FrostFsObjectHeader( - containerId: container, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - sessionToken: token); - - var stream = await pool.PutObjectAsync(param, new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - await stream.WriteAsync(bytes.AsMemory()); - var objectId = await stream.CompleteAsync(); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - var objs = pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default); - await foreach (var objId in objs) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(pool); - - await foreach (var _ in pool.ListContainersAsync(default, default)) - { - Assert.Fail("Containers exist"); - } - } - - [Theory] - [InlineData(1)] - [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB - [InlineData(64 * 1024 * 1024 - 1)] - [InlineData(64 * 1024 * 1024 + 1)] - [InlineData(2 * 64 * 1024 * 1024 + 256)] - [InlineData(200)] - public async void ClientCutScenarioTest(int objectSize) - { - var options = GetDefaultParams(); - - var pool = new Pool(options); - - var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); - - Assert.Null(error); - - await Cleanup(pool); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), - lightWait); - - var containerId = await pool.PutContainerAsync(createContainerParam, default); - - var ctx = new CallContext(TimeSpan.FromSeconds(10)); - - // ctx.Interceptors.Add(new CallbackInterceptor()); - - var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx); - - Assert.NotNull(container); - - byte[] bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectClientCutPut( - new FrostFsObjectHeader( - containerId: containerId, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - payload: new MemoryStream(bytes)); - - var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true); - - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) - { - hasObject = true; - - var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); - - var objHeader = res.HeaderInfo; - Assert.NotNull(objHeader); Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes[0].Key); - Assert.Equal("test", objHeader.Attributes[0].Value); - } - - Assert.True(hasObject); - - var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await CheckFilter(pool, containerId, new FilterByRootObject()); - - await Cleanup(pool); - - await foreach (var cid in pool.ListContainersAsync(default, default)) - { - Assert.Fail($"Container {cid.GetValue()} exist"); - } - } -} diff --git a/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs b/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs index 59099a9..5e6998b 100644 --- a/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs @@ -16,19 +16,13 @@ namespace FrostFS.SDK.Tests.Smoke; [SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] public abstract class SmokeTestsBase { - // cluster Ori - // internal readonly string url = "http://10.78.128.207:8080"; - // internal readonly string keyString = "L4JWLdedUd4b21sriRHtCPGkjG2Mryz2AWLiVqTBSNyxxyAUcc7s"; - // cluster - internal readonly string url = "http://10.78.128.190:8080"; - internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba"; + // internal readonly string url = "http://10.78.128.190:8080"; + // internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba"; // WSL2 - // internal readonly string url = "http://172.29.238.97:8080"; - // internal readonly string keyString = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; // "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; - - //"KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; + internal readonly string url = "http://172.29.238.97:8080"; // "http://172.20.8.23:8080"; + internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; protected ECDsa? Key { get; } @@ -77,7 +71,7 @@ public abstract class SmokeTestsBase if (networkSettings.HomomorphicHashingDisabled) attributes.Add(new("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", "true")); - + var containerInfo = new FrostFsContainerInfo( new FrostFsPlacementPolicy(unique, backupFactor, selectors, filter, replicas), [.. attributes]); diff --git a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs index db1e193..cf4bdfa 100644 --- a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; - using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; using FrostFS.SDK.Cryptography; @@ -20,13 +19,16 @@ public class ContainerTest : ContainerTestsBase Assert.NotNull(result); Assert.NotNull(result.GetValue()); - Assert.True(Base58.Encode(Mocker.ContainerGuid.ToBytes()) == result.GetValue()); + + var bytes = Mocker.ContainerGuid.ToByteArray(true); + + Assert.True(Base58.Encode(new Span(bytes)) == result.GetValue()); } [Fact] public async void GetContainerTest() { - var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); + var cid = new FrostFsContainerId(Base58.Encode(new Span(Mocker.ContainerGuid.ToByteArray(true)))); var result = await GetClient().GetContainerAsync(new PrmContainerGet(cid), default); @@ -61,7 +63,7 @@ public class ContainerTest : ContainerTestsBase public async void DeleteContainerAsyncTest() { Mocker.ReturnContainerRemoved = true; - var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); + var cid = new FrostFsContainerId(Base58.Encode(new Span(Mocker.ContainerGuid.ToByteArray(true)))); await GetClient().DeleteContainerAsync(new PrmContainerDelete(cid, PrmWait.DefaultParams), default); diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs index 479c6ba..7a595e6 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs @@ -88,7 +88,7 @@ public class ObjectTest : ObjectTestsBase // PART1 Assert.Equal(blockSize, objects[0].Payload.Length); Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload)); - + Assert.NotNull(objects[0].Header.Split.SplitId); Assert.Null(objects[0].Header.Split.Previous); Assert.True(objects[0].Header.Attributes.Count == 0); @@ -104,7 +104,7 @@ public class ObjectTest : ObjectTestsBase // last part Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); - Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload)); + Assert.True(bytes.AsMemory(2 * blockSize).ToArray().SequenceEqual(objects[2].Payload)); Assert.NotNull(objects[3].Header.Split.Parent); Assert.NotNull(objects[3].Header.Split.ParentHeader); diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs index 18e3981..c3681ce 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs @@ -35,7 +35,7 @@ public abstract class ObjectTestsBase ContainerGuid = Guid.NewGuid() }; - ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); + ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToByteArray(true))); Mocker.ObjectHeader = new( ContainerId, diff --git a/src/FrostFS.SDK.Tests/Unit/SessionTests.cs b/src/FrostFS.SDK.Tests/Unit/SessionTests.cs index b2f1b78..be2e072 100644 --- a/src/FrostFS.SDK.Tests/Unit/SessionTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/SessionTests.cs @@ -2,7 +2,6 @@ using System.Diagnostics.CodeAnalysis; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; namespace FrostFS.SDK.Tests.Unit; @@ -38,7 +37,7 @@ public class SessionTest : SessionTestsBase Assert.NotNull(result); Assert.NotEqual(Guid.Empty, result.Id); - Assert.Equal(Mocker.SessionId, result.Id.ToBytes()); + Assert.Equal(Mocker.SessionId, result.Id.ToByteArray(true)); Assert.Equal(Mocker.SessionKey, result.SessionKey.ToArray()); //Assert.Equal(OwnerId.ToMessage(), result.Token.Body.OwnerId); From 45e73a6f8ea45fab645c3bea6b00f444c6e47986 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Mon, 31 Mar 2025 13:51:45 +0300 Subject: [PATCH 02/12] [#43] Client: Set nuget version Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj | 2 +- src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj | 2 +- src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index e94d3f8..1d16492 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -6,7 +6,7 @@ enable AllEnabledByDefault FrostFS.SDK.Client - 1.0.3 + 1.0.4 C# SDK for FrostFS gRPC native protocol diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 9dfd370..04077e8 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Cryptography - 1.0.3 + 1.0.4 Cryptography tools for C# SDK diff --git a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj index 5207bbc..72bfe4c 100644 --- a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj +++ b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Protos - 1.0.3 + 1.0.4 Protobuf client for C# SDK From 30af6145584a40781695f5084978b78f81ba3460 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 9 Apr 2025 17:09:12 +0300 Subject: [PATCH 03/12] [#57] Add helpers for signing Nuget packages Discussion: OBJECT-16744 Signed-off-by: Vitaliy Potyarkin --- .gitignore | 5 ++- Makefile | 58 ++++++++++++++++++++++++++++++++ release/README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++ release/ca.cert | 34 +++++++++++++++++++ release/codesign.mk | 74 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 release/README.md create mode 100644 release/ca.cert create mode 100644 release/codesign.mk diff --git a/.gitignore b/.gitignore index ae0b81f..449284e 100644 --- a/.gitignore +++ b/.gitignore @@ -32,5 +32,8 @@ antlr-*.jar # binary bin/ -release/ obj/ + +# Repository signing keys +release/maintainer.* +release/ca.* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f02478e --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ +DOTNET?=dotnet +DOCKER?=docker + +NUGET_REGISTRY?=TrueCloudLab +NUGET_REGISTRY_URL?=https://git.frostfs.info/api/packages/TrueCloudLab/nuget/index.json +NUGET_REGISTRY_USER?= +NUGET_REGISTRY_PASSWORD?= + +NUPKG=find -iname '*.nupkg' | grep . | xargs -d'\n' -t -r -n1 +RFC3161_TSA?=http://timestamp.digicert.com + + +.PHONY: build +build: + $(DOTNET) build + + +.PHONY: sign +sign: export NUGET_CERT_REVOCATION_MODE=offline +sign: release/maintainer.pfx + $(NUPKG) $(DOTNET) nuget sign --overwrite --certificate-path $< --timestamper "$(RFC3161_TSA)" + @rm -v "$<" # maintainer.pfx is not password protected and must be ephemeral + $(NUPKG) $(DOTNET) nuget verify + + +.PHONY: publish +publish: + $(NUPKG) $(DOTNET) nuget verify + $(NUPKG) $(DOTNET) nuget push --source "$(NUGET_REGISTRY)" + + +.PHONY: nuget-registry +nuget-registry: +ifeq (,$(NUGET_REGISTRY_USER)) + $(error NUGET_REGISTRY_USER not set) +endif +ifeq (,$(NUGET_REGISTRY_PASSWORD)) + $(error NUGET_REGISTRY_PASSWORD not set) +endif + $(DOTNET) nuget add source \ + --name "$(NUGET_REGISTRY)" \ + --username "$(NUGET_REGISTRY_USER)" \ + --password "$(NUGET_REGISTRY_PASSWORD)" \ + --store-password-in-clear-text \ + "$(NUGET_REGISTRY_URL)" + + +.PHONY: clean +clean: + -$(NUPKG) rm -v + + +.PHONY: container +container: + $(DOCKER) run --pull=always --rm -it -v "$$PWD:/src" -w /src git.frostfs.info/truecloudlab/env:dotnet-8.0 + + +include release/codesign.mk diff --git a/release/README.md b/release/README.md new file mode 100644 index 0000000..96f8c25 --- /dev/null +++ b/release/README.md @@ -0,0 +1,82 @@ +# Release process + +## Preparing release + +_TBD_ + +## Trusting TrueCloudLab code signing CA certificate + +Verifying signatures (and signing) TrueCloudLab packages requires adding +[TrueCloudLab Code Signing CA](ca.cert) to the list of trusted roots. + +On Linux this can be done by appending [release/ca.cert](ca.cert) to one of: + +- `/etc/pki/ca-trust/extracted/pem/objsign-ca-bundle.pem`: compatible with + [update-ca-trust] and originally proposed in [.NET design docs] +- `…/dotnet/sdk/X.Y.ZZZ/trustedroots/codesignctl.pem`: [fallback] codesigning certificate trust list for .NET + +[update-ca-trust]: https://www.linux.org/docs/man8/update-ca-trust.html +[.NET design docs]: https://github.com/dotnet/designs/blob/main/accepted/2021/signed-package-verification/re-enable-signed-package-verification-technical.md#linux +[fallback]: https://github.com/dotnet/sdk/blob/11150c0ec9020625308edeec555a8b78dbfb2aa5/src/Layout/redist/trustedroots/README.md + +## Signing Nuget packages + +Repository maintainer places `maintainer.cert` and `maintainer.key` (see below +regarding obtaining these files) into `release/` directory and then +executes: + +```console +$ make build sign +``` + +## Uploading packages to Nuget registry + +**IMPORTANT: the following steps upload all `*.nupkg` files located under +`src/`. Maintainer MUST make sure that no unnecessary package versions will be +uploaded to the registry.** + +Configure registry credentials (once per machine): + +```console +$ make nuget-registry NUGET_REGISTRY_USER=username NUGET_REGISTRY_PASSWORD=token +``` + +Publish all locally built packages (implicitly clear existing `*.nupkg` and +rebuild current version only): + +```console +$ make clean build sign publish +``` + + +## Obtaining release signing certificate + +Repository maintainer owns and keeps safe the release signing key +(`maintainer.key`). Private key should never leave maintainer's machine and +should be considered a highly sensitive secret. + +- Generating new maintainer key and the corresponding CSR: + + ```console + $ make maintainer.csr + ...lines skipped... + Enter PEM pass phrase: + Verifying - Enter PEM pass phrase: + ----- + IMPORTANT: Keep maintainer.key private! + + Certificate signing request is ready. + Send maintainer.csr to CA administrator to obtain the certificate. + ``` + + Resulting CSR (`maintainer.csr`) does not contain any sensitive + cryptographic material and may be passed to CA administrator through regular + communication channels. + +- CA administrator then issues the certificate (`make maintainer.cert`) and + sends it back to the maintainer to be used in combination with + `maintainer.key` + +This procedure should be repeated once per machine per `maintainer.cert` +lifetime (1 year) - typically just once per year since we expect the +maintainer to use only a single computer to sign releases. diff --git a/release/ca.cert b/release/ca.cert new file mode 100644 index 0000000..4f4c2f4 --- /dev/null +++ b/release/ca.cert @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF2zCCA8OgAwIBAgIUa9xC/RgvFtUG/xeR016nn0B4K0YwDQYJKoZIhvcNAQEL +BQAwdTELMAkGA1UEBhMCUlUxFTATBgNVBAoMDFRydWVDbG91ZExhYjEVMBMGA1UE +CwwMVHJ1ZUNsb3VkTGFiMTgwNgYDVQQDDC9UcnVlQ2xvdWRMYWIgQ29kZSBTaWdu +aW5nIENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yNTA0MTAxNTI2MTFaFw0zNTA0 +MDgxNTI2MTFaMHUxCzAJBgNVBAYTAlJVMRUwEwYDVQQKDAxUcnVlQ2xvdWRMYWIx +FTATBgNVBAsMDFRydWVDbG91ZExhYjE4MDYGA1UEAwwvVHJ1ZUNsb3VkTGFiIENv +ZGUgU2lnbmluZyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCyANB4cjf+ZEAFx9RiUYXCAOPMV+jyqgcVbhzh2YKc +9SlvGKRlc00Ar1RlFcrycdkIrTKmhhobiWsFp7UgphwLqRTCb5NB6qfUoWhnfiD9 +m0OBgeVX5wivVaibRI9PSTbFDcIhYUiNvwFJ6GduH/9zOxf1BvuL7LMaoyhIDcg/ +XVLuekE2lnX83zsedv0v/2jyyMY9Ct6N2BXzyHSAzSdYYg0F9Qu9fIMAPjoKhWPc +PnotqaACjb1DScLUr3E/o2W1FfprTT2Pip/0AXxO4wixl4QWh9HeOKV22KRcCHo6 +3wNdg5q1ZVGTNBW0+yoB4jsSG8/JM+2Ujhc1ZnYH10armvGq/0Oc2YQE00960Wy8 +t0drCFWJUO1XHNeBxkkupmj7N1TAPbixtfiGZJhECOWOJwyMpcKixlt5P0cNH4N/ +p3vjyrGQxGLBIkgV/QgjfGkpTHKT1/H40YK6DliWJc01KfNTqn0K+/TIyF0n26kD +BWYVlvDh5P1+V9DGuD2zeXB3PstoifD/Pd7D8wuqpm17noFE19MLp94xv03q9nEa +jRMEd2J2buTLvMh5BBVH0Sm38QAHpSIZ9O3dSLvvjlALbVtwmcsNE9fgxiue3vTB +iXNW8wWs+/DMYwbWyBoCwORxVOdOyc1JLn7qAAEUBweilPVXpWuzMLdUsifPiqrV +dQIDAQABo2MwYTAdBgNVHQ4EFgQUEz4y/RvQMmbUFvf5JbGe/0PZR90wHwYDVR0j +BBgwFoAUEz4y/RvQMmbUFvf5JbGe/0PZR90wDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAF79W9hMGnrKUWLDOcoeXDEn ++gZxd5rjeF0tNEQGbWKGTJAGQdprOkzWQ47PewpFeS+ePpjOglBAt7cV2ZnOugAT +Brx31vYpNAvYnUCYn+/IUqD8S/U4uErVS9zG9QjirZWo56eJP62vnScKuApCQCbA +pp0zrIyJ+2lQKzlMOENRqzYYA/UTOqYTtnW6x2D8goVqCmDXpgpxEp5XliFjJSr6 +dOjiopNWMvaV3R/Bnd4i41taM7M6HpIV+gbXmEKEFS0ejVfzT8z1pTigN7GBqbxf +nXD03eLUIsbMDv4ZQPrheN7nKnjRUn8kxz0SSK1m2YDrXW51m8fOs6aTvwC/cNe+ +FJMqQMF32i4IXVfUbyUJi+JMvawqm2wEY46vrh7siprY6rXsAzCKJo07i6jvUwnT +TXMldSCPgTEqzT2JBzzr0tRfuPKsv0/NqflHvwfuMRCpcZ7jJZ700iN92xXkiQHP +DmCZOILXcNclAth3nAnyY4XE5a8myv8bwYaPdJdIFlV+BoU/8mClDeA8ck4rDy12 +T5YChKew2oiL4j4B6v9/yrDjD1IT0gv4BWyPhb/n390BCEXt8g9auNcT0s6O8kEc +VUDVc1519ocMCuWVuqUK9o2w0zu50/pBn4hVLfT3QyW8sqtlRKghOWtqZzigvCWF +VjATeO5F/Z7OSDebHUGv +-----END CERTIFICATE----- diff --git a/release/codesign.mk b/release/codesign.mk new file mode 100644 index 0000000..1c39361 --- /dev/null +++ b/release/codesign.mk @@ -0,0 +1,74 @@ +PKI_ROLE?=maintainer +PKI_DIR?=release + +# Note: Only RSA signatures are supported (NU3013) +# https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3013) + + +ifeq ($(PKI_ROLE),maintainer) +.PHONY: maintainer.csr +maintainer.csr: $(PKI_DIR)/maintainer.csr +$(PKI_DIR)/maintainer.csr: KEY=$(patsubst %.csr,%.key,$@) +$(PKI_DIR)/maintainer.csr: + openssl req \ + -new \ + -newkey rsa:4096 \ + -keyout $(KEY) \ + -out $@ \ + -sha256 \ + -addext keyUsage=critical,digitalSignature \ + -addext extendedKeyUsage=critical,codeSigning,msCodeCom \ + -subj "/C=RU/O=TrueCloudLab/OU=TrueCloudLab/CN=frostfs-sdk-csharp Release Team" + @echo "IMPORTANT: Keep $(KEY) private!\n" + @echo "Certificate signing request is ready.\nSend $@ to CA administrator to obtain the certificate." + +$(PKI_DIR)/maintainer.pfx: $(PKI_DIR)/maintainer.cert $(PKI_DIR)/maintainer.key $(PKI_DIR)/ca.cert + openssl verify \ + -CAfile $(PKI_DIR)/ca.cert \ + $(PKI_DIR)/maintainer.cert + openssl pkcs12 \ + -export \ + -out $@ \ + -inkey $(PKI_DIR)/maintainer.key \ + -in $(PKI_DIR)/maintainer.cert \ + -CAfile $(PKI_DIR)/ca.cert \ + -chain \ + -passout pass: +endif + + +ifeq ($(PKI_ROLE),ca) +.PHONY: maintainer.cert +maintainer.cert: $(PKI_DIR)/maintainer.cert +$(PKI_DIR)/maintainer.cert: CSR=$(patsubst %.cert,%.csr,$@) +$(PKI_DIR)/maintainer.cert: $(PKI_DIR)/ca.key $(PKI_DIR)/ca.cert + openssl req -noout -text -in $(CSR) + @read -p "Review the CSR above. Press Enter to continue, Ctrl+C to cancel " -r null + openssl x509 \ + -req \ + -days 365 \ + -in $(CSR) \ + -copy_extensions copy \ + -ext keyUsage,extendedKeyUsage \ + -CA $(PKI_DIR)/ca.cert \ + -CAkey $(PKI_DIR)/ca.key \ + -CAcreateserial \ + -out $@ + echo >> $@ + cat $(PKI_DIR)/ca.cert >> $@ + openssl x509 -noout -text -in $@ -fingerprint -sha256 + @echo "Certificate is ready.\nSend $@ back to maintainer." + +$(PKI_DIR)/ca.key: CERT=$(patsubst %.key,%.cert,$@) +$(PKI_DIR)/ca.key: + openssl req \ + -x509 \ + -newkey rsa:4096 \ + -keyout $@ \ + -out $(CERT) \ + -sha256 \ + -days 3650 \ + -addext keyUsage=critical,keyCertSign \ + -subj "/C=RU/O=TrueCloudLab/OU=TrueCloudLab/CN=TrueCloudLab Code Signing Certificate Authority" + @echo "IMPORTANT: Keep $@ private!\n" +endif From b3907782012285b2b0c58da01829def467315e2a Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 11 Apr 2025 10:33:19 +0300 Subject: [PATCH 04/12] [#57] ci: Disable automatic publishing of unsigned nugets We're switching to non-automatic process for publishing signed nugets, unsigned workflow will still be available as an escape hatch but it won't ever be triggered automatically. Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.forgejo/workflows/publish.yml b/.forgejo/workflows/publish.yml index ca66daa..fd79900 100644 --- a/.forgejo/workflows/publish.yml +++ b/.forgejo/workflows/publish.yml @@ -1,5 +1,4 @@ on: - push: workflow_dispatch: jobs: @@ -16,7 +15,7 @@ jobs: # `dotnet build` implies and replaces `dotnet pack` thanks to `GeneratePackageOnBuild` run: dotnet build - - name: Publish NuGet packages + - name: Publish unsigned NuGet packages run: |- dotnet nuget add source \ --name "$NUGET_REGISTRY" \ From 42f45076384f40558f4bebdc23b38d08e3aeab08 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Thu, 3 Apr 2025 17:41:14 +0300 Subject: [PATCH 05/12] [#55] Fix Replica mapper Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs index cd7a0b7..00ad4a3 100644 --- a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs +++ b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs @@ -12,7 +12,9 @@ public static class PolicyMapper return new Replica { Count = (uint)replica.Count, - Selector = replica.Selector + Selector = replica.Selector, + EcDataCount = replica.EcDataCount, + EcParityCount = replica.EcParityCount }; } @@ -23,7 +25,11 @@ public static class PolicyMapper throw new ArgumentNullException(nameof(replica)); } - return new FrostFsReplica((int)replica.Count, replica.Selector); + return new FrostFsReplica((int)replica.Count, replica.Selector) + { + EcDataCount = replica.EcDataCount, + EcParityCount = replica.EcParityCount + }; } public static Selector ToMessage(this FrostFsSelector selector) From 5f451c881881a9297a184a4f8ad9798eeb59615b Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Fri, 11 Apr 2025 15:52:31 +0300 Subject: [PATCH 06/12] [#60] Wallet tools in SDK Signed-off-by: Pavel Gross --- .../FrostFS.SDK.Client.csproj | 1 + src/FrostFS.SDK.Client/Tools/WalletTools.cs | 31 ++++++ src/FrostFS.SDK.Client/Wallets/Account.cs | 24 +++++ src/FrostFS.SDK.Client/Wallets/Contract.cs | 15 +++ src/FrostFS.SDK.Client/Wallets/Extra.cs | 6 ++ src/FrostFS.SDK.Client/Wallets/Parameter.cs | 12 +++ src/FrostFS.SDK.Client/Wallets/ScryptValue.cs | 15 +++ src/FrostFS.SDK.Client/Wallets/Wallet.cs | 18 ++++ src/FrostFS.SDK.Cryptography/Base58.cs | 2 +- .../FrostFS.SDK.Cryptography.csproj | 1 + src/FrostFS.SDK.Cryptography/Key.cs | 19 ++++ .../WalletExtractor.cs | 98 +++++++++++++++++++ src/FrostFS.SDK.Tests/TestData/wallet.json | 30 ++++++ src/FrostFS.SDK.Tests/Unit/WalletTests.cs | 25 +++++ 14 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 src/FrostFS.SDK.Client/Tools/WalletTools.cs create mode 100644 src/FrostFS.SDK.Client/Wallets/Account.cs create mode 100644 src/FrostFS.SDK.Client/Wallets/Contract.cs create mode 100644 src/FrostFS.SDK.Client/Wallets/Extra.cs create mode 100644 src/FrostFS.SDK.Client/Wallets/Parameter.cs create mode 100644 src/FrostFS.SDK.Client/Wallets/ScryptValue.cs create mode 100644 src/FrostFS.SDK.Client/Wallets/Wallet.cs create mode 100644 src/FrostFS.SDK.Cryptography/WalletExtractor.cs create mode 100644 src/FrostFS.SDK.Tests/TestData/wallet.json create mode 100644 src/FrostFS.SDK.Tests/Unit/WalletTests.cs diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index 1d16492..ed050a2 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -45,6 +45,7 @@ + diff --git a/src/FrostFS.SDK.Client/Tools/WalletTools.cs b/src/FrostFS.SDK.Client/Tools/WalletTools.cs new file mode 100644 index 0000000..45532a7 --- /dev/null +++ b/src/FrostFS.SDK.Client/Tools/WalletTools.cs @@ -0,0 +1,31 @@ +using System.Text.Json; +using System; +using FrostFS.SDK.Client.Wallets; +using FrostFS.SDK.Cryptography; + +namespace FrostFS.SDK.Client; + +public static class WalletTools +{ + public static string GetWifFromWallet(string walletJsonText, byte[] password, int accountIndex = 0) + { + var wallet = JsonSerializer.Deserialize(walletJsonText) ?? throw new ArgumentException("Wrong wallet format"); + + if (wallet.Accounts == null || wallet.Accounts.Length < accountIndex + 1) + { + throw new ArgumentException("Wrong wallet content"); + } + + var encryptedKey = wallet.Accounts[accountIndex].Key; + + if (string.IsNullOrEmpty(encryptedKey)) + { + throw new ArgumentException("Cannot get encrypted WIF"); + } + + var privateKey = CryptoWallet.GetKeyFromEncodedWif(password, encryptedKey!); + var wif = privateKey.GetWIFFromPrivateKey(); + + return wif; + } +} diff --git a/src/FrostFS.SDK.Client/Wallets/Account.cs b/src/FrostFS.SDK.Client/Wallets/Account.cs new file mode 100644 index 0000000..1d5b3eb --- /dev/null +++ b/src/FrostFS.SDK.Client/Wallets/Account.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace FrostFS.SDK.Client.Wallets; + +public class Account +{ + [JsonPropertyName("address")] + public string? Address { get; set; } + + [JsonPropertyName("key")] + public string? Key { get; set; } + + [JsonPropertyName("label")] + public string? Label { get; set; } + + [JsonPropertyName("contract")] + public Contract? Contract { get; set; } + + [JsonPropertyName("lock")] + public bool Lock { get; set; } + + [JsonPropertyName("isDefault")] + public bool IsDefault { get; set; } +} diff --git a/src/FrostFS.SDK.Client/Wallets/Contract.cs b/src/FrostFS.SDK.Client/Wallets/Contract.cs new file mode 100644 index 0000000..b15d21d --- /dev/null +++ b/src/FrostFS.SDK.Client/Wallets/Contract.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace FrostFS.SDK.Client.Wallets; + +public class Contract +{ + [JsonPropertyName("script")] + public string? Script { get; set; } + + [JsonPropertyName("parameters")] + public Parameter[]? Parameters { get; set; } + + [JsonPropertyName("deployed")] + public bool Deployed { get; set; } +} diff --git a/src/FrostFS.SDK.Client/Wallets/Extra.cs b/src/FrostFS.SDK.Client/Wallets/Extra.cs new file mode 100644 index 0000000..bff8b8d --- /dev/null +++ b/src/FrostFS.SDK.Client/Wallets/Extra.cs @@ -0,0 +1,6 @@ +namespace FrostFS.SDK.Client.Wallets; + +public class Extra +{ + public string? Tokens { get; set; } +} diff --git a/src/FrostFS.SDK.Client/Wallets/Parameter.cs b/src/FrostFS.SDK.Client/Wallets/Parameter.cs new file mode 100644 index 0000000..c831e36 --- /dev/null +++ b/src/FrostFS.SDK.Client/Wallets/Parameter.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace FrostFS.SDK.Client.Wallets; + +public class Parameter +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("type")] + public string? Type { get; set; } +} diff --git a/src/FrostFS.SDK.Client/Wallets/ScryptValue.cs b/src/FrostFS.SDK.Client/Wallets/ScryptValue.cs new file mode 100644 index 0000000..2548b32 --- /dev/null +++ b/src/FrostFS.SDK.Client/Wallets/ScryptValue.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace FrostFS.SDK.Client.Wallets; + +public class ScryptValue +{ + [JsonPropertyName("n")] + public int N { get; set; } + + [JsonPropertyName("r")] + public int R { get; set; } + + [JsonPropertyName("p")] + public int P { get; set; } +} diff --git a/src/FrostFS.SDK.Client/Wallets/Wallet.cs b/src/FrostFS.SDK.Client/Wallets/Wallet.cs new file mode 100644 index 0000000..e1439e0 --- /dev/null +++ b/src/FrostFS.SDK.Client/Wallets/Wallet.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace FrostFS.SDK.Client.Wallets; + +public class Wallet +{ + [JsonPropertyName("version")] + public string? Version { get; set; } + + [JsonPropertyName("accounts")] + public Account[]? Accounts { get; set; } + + [JsonPropertyName("scrypt")] + public ScryptValue? Scrypt { get; set; } + + [JsonPropertyName("extra")] + public Extra? Extra { get; set; } +} diff --git a/src/FrostFS.SDK.Cryptography/Base58.cs b/src/FrostFS.SDK.Cryptography/Base58.cs index 0026bc3..4dc6187 100644 --- a/src/FrostFS.SDK.Cryptography/Base58.cs +++ b/src/FrostFS.SDK.Cryptography/Base58.cs @@ -32,7 +32,7 @@ public static class Base58 public static string Base58CheckEncode(this Span data) { - byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan()); ; + byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan()); Span buffer = stackalloc byte[data.Length + 4]; data.CopyTo(buffer); diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 04077e8..43f7187 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -35,6 +35,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/FrostFS.SDK.Cryptography/Key.cs b/src/FrostFS.SDK.Cryptography/Key.cs index 5f382fa..9eca104 100644 --- a/src/FrostFS.SDK.Cryptography/Key.cs +++ b/src/FrostFS.SDK.Cryptography/Key.cs @@ -82,6 +82,25 @@ public static class KeyExtension return DataHasher.Sha256(script.AsSpan()).RIPEMD160(); } + public static string GetWIFFromPrivateKey(this byte[] privateKey) + { + if (privateKey == null || privateKey.Length != 32) + { + throw new ArgumentNullException(nameof(privateKey)); + } + + Span wifSpan = stackalloc byte[34]; + + wifSpan[0] = 0x80; + wifSpan[33] = 0x01; + + privateKey.AsSpan().CopyTo(wifSpan.Slice(1)); + + var wif = Base58.Base58CheckEncode(wifSpan); + + return wif; + } + private static string ToAddress(this byte[] scriptHash, byte version) { Span data = stackalloc byte[21]; diff --git a/src/FrostFS.SDK.Cryptography/WalletExtractor.cs b/src/FrostFS.SDK.Cryptography/WalletExtractor.cs new file mode 100644 index 0000000..3e940f5 --- /dev/null +++ b/src/FrostFS.SDK.Cryptography/WalletExtractor.cs @@ -0,0 +1,98 @@ +using System; +using System.Text; +using System.Linq; +using System.Security.Cryptography; +using Org.BouncyCastle.Crypto.Generators; + +namespace FrostFS.SDK.Cryptography; + +public static class CryptoWallet +{ + private static int N = 16384; + private static int R = 8; + private static int P = 8; + + private enum CipherAction + { + Encrypt, + Decrypt + } + + public static byte[] GetKeyFromEncodedWif(byte[] password, string encryptedWIF) + { + var nep2Data = Base58.Base58CheckDecode(encryptedWIF); + + if (nep2Data.Length == 39 && nep2Data[0] == 1 && nep2Data[1] == 66 && nep2Data[2] == 224) + { + var addressHash = nep2Data.AsSpan(3, 4).ToArray(); + var derivedKey = SCrypt.Generate(password, addressHash, N, R, P, 64); + var derivedKeyHalf1 = derivedKey.Take(32).ToArray(); + var derivedKeyHalf2 = derivedKey.Skip(32).ToArray(); + var encrypted = nep2Data.AsSpan(7, 32).ToArray(); + var decrypted = Aes(encrypted, derivedKeyHalf2, CipherAction.Decrypt); + var plainPrivateKey = XorRange(decrypted, derivedKeyHalf1, 0, decrypted.Length); + + return plainPrivateKey; + } + else + { + throw new ArgumentException("Not valid NEP2 prefix."); + } + } + + public static string EncryptWif(string plainText, ECDsa key) + { + var addressHash = GetAddressHash(key); + var derivedKey = SCrypt.Generate(Encoding.UTF8.GetBytes(plainText), addressHash, N, R, P, 64); + var derivedHalf1 = derivedKey.Take(32).ToArray(); + var derivedHalf2 = derivedKey.Skip(32).ToArray(); + var encryptedHalf1 = Aes(XorRange(key.PrivateKey(), derivedHalf1, 0, 16), derivedHalf2, CipherAction.Encrypt); + var encryptedHalf2 = Aes(XorRange(key.PrivateKey(), derivedHalf1, 16, 32), derivedHalf2, CipherAction.Encrypt); + var prefixes = new byte[] { 1, 66, 224 }; + var concatenation = ArrayHelper.Concat([prefixes, addressHash, encryptedHalf1, encryptedHalf2]); + + return Base58.Base58CheckEncode(concatenation); + } + + public static byte[] GetAddressHash(ECDsa key) + { + string address = key.PublicKey().PublicKeyToAddress(); + + using SHA256 sha256 = SHA256.Create(); + byte[] addressHashed = sha256.ComputeHash(Encoding.UTF8.GetBytes(address)); + return [.. addressHashed.Take(4)]; + } + + private static byte[] XorRange(byte[] arr1, byte[] arr2, int from, int length) + { + byte[] result = new byte[length]; + + var j = 0; + for (var i = from; i < length; ++i) + { + result[j++] = (byte)(arr1[i] ^ arr2[i]); + } + + return result; + } + + private static byte[] Aes(byte[] data, byte[] key, CipherAction action) + { + using var aes = System.Security.Cryptography.Aes.Create(); + aes.Key = key; + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + + if (action == CipherAction.Encrypt) + { + return aes.CreateEncryptor().TransformFinalBlock(data, 0, data.Length); + } + + if (action == CipherAction.Decrypt) + { + return aes.CreateDecryptor().TransformFinalBlock(data, 0, data.Length); + } + + throw new ArgumentException("Wrong cippher action", nameof(action)); + } +} diff --git a/src/FrostFS.SDK.Tests/TestData/wallet.json b/src/FrostFS.SDK.Tests/TestData/wallet.json new file mode 100644 index 0000000..7a4bdd4 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/wallet.json @@ -0,0 +1,30 @@ +{ + "version": "1.0", + "accounts": [ + { + "address": "NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw", + "key": "6PYVCcS2yp89JpcfR61FGhdhhzyYjSErNedmpZErnybNTxUZMRdhzJLrek", + "label": "", + "contract": { + "script": "DCEDJOdiiPy5ABANAYAqFO+XfMpFrQc1YSMERt8Us0TIWLZBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isDefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/WalletTests.cs b/src/FrostFS.SDK.Tests/Unit/WalletTests.cs new file mode 100644 index 0000000..2acd77e --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/WalletTests.cs @@ -0,0 +1,25 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text; +using FrostFS.SDK.Client; + +namespace FrostFS.SDK.Tests.Unit; + +[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] +public class WalletTest : SessionTestsBase +{ + [Fact] + public void TestWallet() + { + var password = Encoding.UTF8.GetBytes(""); + + var d = Directory.GetCurrentDirectory(); + var path = ".\\..\\..\\..\\TestData\\wallet.json"; + Assert.True(File.Exists(path)); + + var content = File.ReadAllText(path); + + var wif = WalletTools.GetWifFromWallet(content, password); + + Assert.Equal("KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK", wif); + } +} \ No newline at end of file From c88eea1f8234bf980bfc1cb362c0f019bec3485f Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Fri, 11 Apr 2025 15:18:18 +0300 Subject: [PATCH 07/12] [#58] Observable client cut Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/FrostFSClient.cs | 3 +- .../Mappers/Object/ObjectHeaderMapper.cs | 5 + .../Models/Object/PartUploadedEventArgs.cs | 8 + .../Models/Object/UploadInfo.cs | 38 +++ .../Models/Object/UploadProgressInfo.cs | 48 +++ .../Parameters/PrmObjectClientCutPut.cs | 5 +- .../Services/ObjectServiceProvider.cs | 277 ++++++++++++++---- 7 files changed, 328 insertions(+), 56 deletions(-) create mode 100644 src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs create mode 100644 src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs create mode 100644 src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs diff --git a/src/FrostFS.SDK.Client/FrostFSClient.cs b/src/FrostFS.SDK.Client/FrostFSClient.cs index 860d346..bce9104 100644 --- a/src/FrostFS.SDK.Client/FrostFSClient.cs +++ b/src/FrostFS.SDK.Client/FrostFSClient.cs @@ -254,7 +254,8 @@ public class FrostFSClient : IFrostFSClient public Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { - return GetObjectService().PutClientCutObjectAsync(args, ctx); + return GetObjectService().PutClientCutSingleObjectAsync(args, ctx); + // return GetObjectService().PutClientCutObjectAsync(args, ctx); } public Task PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx) diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs index 115f9e2..fde13ff 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs @@ -44,6 +44,11 @@ public static class ObjectHeaderMapper public static FrostFsSplit ToModel(this Header.Types.Split split) { + if (split is null) + { + throw new ArgumentNullException(nameof(split)); + } + var children = split!.Children.Count != 0 ? new ReadOnlyCollection([.. split.Children.Select(x => x.ToModel())]) : null; diff --git a/src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs b/src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs new file mode 100644 index 0000000..16ad326 --- /dev/null +++ b/src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs @@ -0,0 +1,8 @@ +using System; + +namespace FrostFS.SDK.Client; + +public class PartUploadedEventArgs(ObjectPartInfo part) : EventArgs +{ + public ObjectPartInfo Part { get; } = part; +} diff --git a/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs b/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs new file mode 100644 index 0000000..748fdfe --- /dev/null +++ b/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs @@ -0,0 +1,38 @@ +using FrostFS.SDK; + +namespace FrostFS.SDK.Client; + +public readonly struct ObjectPartInfo(long offset, int length, FrostFsObjectId objectId) : System.IEquatable +{ + public long Offset { get; } = offset; + public int Length { get; } = length; + public FrostFsObjectId ObjectId { get; } = objectId; + + public override bool Equals(object obj) + { + if (obj == null || obj is not ObjectPartInfo) + return false; + + return Equals((ObjectPartInfo)obj); + } + + public override int GetHashCode() + { + return ((int)(Offset >> 32)) ^ (int)Offset ^ Length ^ ObjectId.Value.GetHashCode(); + } + + public static bool operator ==(ObjectPartInfo left, ObjectPartInfo right) + { + return left.Equals(right); + } + + public static bool operator !=(ObjectPartInfo left, ObjectPartInfo right) + { + return !(left == right); + } + + public bool Equals(ObjectPartInfo other) + { + return GetHashCode() == other.GetHashCode(); + } +} diff --git a/src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs b/src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs new file mode 100644 index 0000000..a6fa905 --- /dev/null +++ b/src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; + +namespace FrostFS.SDK.Client; + +public class UploadProgressInfo +{ + private List _parts; + + public UploadProgressInfo(Guid splitId, int capacity = 8) + { + _parts = new List(capacity); + SplitId = new SplitId(splitId); + } + + public UploadProgressInfo(Guid splitId, Collection parts) + { + _parts = [.. parts]; + SplitId = new SplitId(splitId); + } + + public event EventHandler? Notify; + + public SplitId SplitId { get; } + + internal void AddPart(ObjectPartInfo part) + { + _parts.Add(part); + Notify?.Invoke(this, new PartUploadedEventArgs(part)); + } + + public ObjectPartInfo GetPart(int index) + { + return _parts[index]; + } + + public ObjectPartInfo GetLast() + { + return _parts.LastOrDefault(); + } + + public ReadOnlyCollection GetParts() + { + return new ReadOnlyCollection(_parts); + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs b/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs index 706e367..46f25e0 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs @@ -8,7 +8,8 @@ public readonly struct PrmObjectClientCutPut( int bufferMaxSize = 0, FrostFsSessionToken? sessionToken = null, byte[]? customBuffer = null, - string[]? xheaders = null) : PrmObjectPutBase, System.IEquatable + string[]? xheaders = null, + UploadProgressInfo? progress = null) : PrmObjectPutBase, System.IEquatable { /// /// Need to provide values like ContainerId and ObjectType to create and object. @@ -41,6 +42,8 @@ public readonly struct PrmObjectClientCutPut( /// public string[] XHeaders { get; } = xheaders ?? []; + public UploadProgressInfo? Progress { get; } = progress; + internal PutObjectContext PutObjectContext { get; } = new(); public override readonly bool Equals(object obj) diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index ea3b727..ed9e9bc 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -385,30 +385,202 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return response.Body.ObjectId.ToModel(); } - internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) + internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { - var stream = args.Payload!; - var header = args.Header!; + if (args.Payload == null) + throw new ArgumentException(nameof(args.Payload)); - if (header.PayloadLength > 0) - args.PutObjectContext.FullLength = header.PayloadLength; - else if (stream.CanSeek) - args.PutObjectContext.FullLength = (ulong)stream.Length; - else - throw new ArgumentException("The stream does not have a length and payload length is not defined"); - - if (args.PutObjectContext.FullLength == 0) - throw new ArgumentException("The stream has zero length"); + if (args.Header == null) + throw new ArgumentException(nameof(args.Header)); var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - var partSize = (int)networkSettings.MaxObjectSize; + int partSize = (int)networkSettings.MaxObjectSize; - var restBytes = args.PutObjectContext.FullLength; + int chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize; - var objectSize = (int)Math.Min((ulong)partSize, restBytes); + ulong fullLength; - // define collection capacity - var objectsCount = (int)(restBytes / (ulong)objectSize) + ((restBytes % (ulong)objectSize) > 0 ? 1 : 0); + // Information about the uploaded parts. + var progressInfo = args.Progress; // + var offset = 0L; + + if (progressInfo != null && progressInfo.GetParts().Count > 0) + { + if (!args.Payload.CanSeek) + { + throw new FrostFsException("Cannot resume client cut upload for this stream. Seek must be supported."); + } + + var lastPart = progressInfo.GetLast(); + args.Payload.Position = lastPart.Offset + lastPart.Length; + fullLength = (ulong)(args.Payload.Length - args.Payload.Position); + offset = args.Payload.Position; + } + else + { + if (args.Header.PayloadLength > 0) + fullLength = args.Header.PayloadLength; + else if (args.Payload.CanSeek) + fullLength = (ulong)args.Payload.Length; + else + throw new ArgumentException("The stream does not have a length and payload length is not defined"); + } + + //define collection capacity + var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0; + var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; + + progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); + + var remain = fullLength; + + byte[]? buffer = null; + bool isRentBuffer = false; + + try + { + if (args.CustomBuffer != null) + { + if (args.CustomBuffer.Length < chunkSize) + throw new ArgumentException($"Buffer size is too small. At least {chunkSize} required"); + + buffer = args.CustomBuffer; + } + else + { + buffer = ArrayPool.Shared.Rent(chunkSize); + isRentBuffer = true; + } + + FrostFsObjectId? resultObjectId = null; + FrostFsObjectHeader? parentHeader = null; + + while (remain > 0) + { + var bytesToWrite = Math.Min((ulong)partSize, remain); + var isLastPart = remain <= (ulong)partSize; + + // When the last part of the object is uploaded, all metadata for the object must be added + if (isLastPart && objectsCount > 1) + { + parentHeader = new FrostFsObjectHeader(args.Header.ContainerId, FrostFsObjectType.Regular) + { + Attributes = args.Header.Attributes, + PayloadLength = fullLength + }; + } + + // Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter + var header = objectsCount == 1 ? args.Header : new FrostFsObjectHeader( + args.Header.ContainerId, + FrostFsObjectType.Regular, + [], + new FrostFsSplit(progressInfo.SplitId, progressInfo.GetLast().ObjectId, parentHeader: parentHeader)); + + var prm = new PrmObjectPut(header); + using var stream = await PutStreamObjectAsync(prm, ctx).ConfigureAwait(false); + var uploaded = 0; + + // If an error occurs while uploading a part of the object, there is no need to re-upload the parts + // that were successfully uploaded before. It is sufficient to re-upload only the failed part + + var thisPartRest = (int)Math.Min((ulong)partSize, remain); + while (thisPartRest > 0) + { + var nextChunkSize = Math.Min(thisPartRest, chunkSize); + var size = await args.Payload.ReadAsync(buffer, 0, nextChunkSize).ConfigureAwait(false); + + if (size == 0) + break; + + await stream.WriteAsync(buffer.AsMemory(0, size)).ConfigureAwait(false); + uploaded += size; + thisPartRest -= size; + } + + var objectId = await stream.CompleteAsync().ConfigureAwait(false); + var part = new ObjectPartInfo(offset, uploaded, objectId); + offset += uploaded; + progressInfo.AddPart(part); + + remain -= bytesToWrite; + + if (isLastPart) + { + if (objectsCount == 1) + { + return progressInfo.GetPart(0).ObjectId; + } + + if (parentHeader == null) continue; + + // Once all parts of the object are uploaded, they must be linked into a single entity + var linkObject = new FrostFsLinkObject(header.ContainerId, progressInfo.SplitId, parentHeader, + [.. progressInfo.GetParts().Select(p => p.ObjectId)]); + + await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false); + + // Retrieve the ID of the linked object + resultObjectId = FrostFsObjectId.FromHash(prm.Header!.GetHeader().Split!.Parent.Value.Span); + return resultObjectId; + } + } + + throw new FrostFsException("Unexpected error: cannot send object"); + } + finally + { + if (isRentBuffer && buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + internal async Task PutClientCutSingleObjectAsync(PrmObjectClientCutPut args, CallContext ctx) + { + if (args.Payload == null) + throw new ArgumentException(nameof(args.Payload)); + + if (args.Header == null) + throw new ArgumentException(nameof(args.Header)); + + var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); + int partSize = (int)networkSettings.MaxObjectSize; + + int chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize; + + ulong fullLength; + + // Information about the uploaded parts. + var progressInfo = args.Progress; // + var offset = 0L; + + if (progressInfo != null && progressInfo.GetParts().Count > 0) + { + if (!args.Payload.CanSeek) + { + throw new FrostFsException("Cannot resume client cut upload for this stream. Seek must be supported."); + } + + var lastPart = progressInfo.GetLast(); + args.Payload.Position = lastPart.Offset + lastPart.Length; + fullLength = (ulong)(args.Payload.Length - args.Payload.Position); + offset = args.Payload.Position; + } + else + { + if (args.Header.PayloadLength > 0) + fullLength = args.Header.PayloadLength; + else if (args.Payload.CanSeek) + fullLength = (ulong)args.Payload.Length; + else + throw new ArgumentException("The stream does not have a length and payload length is not defined"); + } + + //define collection capacity + var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0; + var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; // if the object fits one part, it can be loaded as non-complex object if (objectsCount == 1) @@ -418,60 +590,54 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return singlePartResult.ObjectId; } - List parts = new(objectsCount); + progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); - SplitId splitId = new(); + var remain = fullLength; - // keep attributes for the large object - var attributes = args.Header!.Attributes.ToArray(); - header.Attributes = null; - - var remain = args.PutObjectContext.FullLength; - - FrostFsObjectHeader? parentHeader = null; - - var lastIndex = objectsCount - 1; - - bool rentBuffer = false; byte[]? buffer = null; + bool isRentBuffer = false; try { - for (int i = 0; i < objectsCount; i++) + if (args.CustomBuffer != null) { - if (args.CustomBuffer != null) + if (args.CustomBuffer.Length < partSize) { - if (args.CustomBuffer.Length < partSize) - { - throw new ArgumentException($"Buffer size is too small. A buffer with capacity {partSize} is required"); - } - - buffer = args.CustomBuffer; - } - else - { - buffer = ArrayPool.Shared.Rent(partSize); - rentBuffer = true; + throw new ArgumentException($"Buffer size is too small. A buffer with capacity {partSize} is required"); } + buffer = args.CustomBuffer; + } + else + { + buffer = ArrayPool.Shared.Rent(partSize); + isRentBuffer = true; + } + + FrostFsObjectHeader? parentHeader = null; + + for (int i = 0; i < objectsCount;) + { + i++; var bytesToWrite = Math.Min((ulong)partSize, remain); - var size = await stream.ReadAsync(buffer, 0, (int)bytesToWrite).ConfigureAwait(false); + var size = await args.Payload.ReadAsync(buffer, 0, (int)bytesToWrite).ConfigureAwait(false); - if (i == lastIndex) + if (i == objectsCount) { - parentHeader = new FrostFsObjectHeader(header.ContainerId, FrostFsObjectType.Regular, attributes) + parentHeader = new FrostFsObjectHeader(args.Header.ContainerId, FrostFsObjectType.Regular) { - PayloadLength = args.PutObjectContext.FullLength + PayloadLength = args.PutObjectContext.FullLength, + Attributes = args.Header.Attributes }; } // Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter var partHeader = new FrostFsObjectHeader( - header.ContainerId, + args.Header.ContainerId, FrostFsObjectType.Regular, [], - new FrostFsSplit(splitId, parts.LastOrDefault(), + new FrostFsSplit(progressInfo.SplitId, progressInfo.GetLast().ObjectId, parentHeader: parentHeader)) { PayloadLength = (ulong)size @@ -484,15 +650,18 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var prm = new PrmSingleObjectPut(obj); - var objId = await PutSingleObjectAsync(prm, ctx).ConfigureAwait(false); + var objectId = await PutSingleObjectAsync(prm, ctx).ConfigureAwait(false); - parts.Add(objId); + var part = new ObjectPartInfo(offset, size, objectId); + progressInfo.AddPart(part); - if (i < lastIndex) + offset += size; + + if (i < objectsCount) continue; // Once all parts of the object are uploaded, they must be linked into a single entity - var linkObject = new FrostFsLinkObject(header.ContainerId, splitId, parentHeader!, parts); + var linkObject = new FrostFsLinkObject(args.Header.ContainerId, progressInfo.SplitId, parentHeader!, [.. progressInfo.GetParts().Select(p => p.ObjectId)]); _ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false); @@ -504,7 +673,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } finally { - if (rentBuffer && buffer != null) + if (isRentBuffer && buffer != null) { ArrayPool.Shared.Return(buffer); } From 764b669295bbf1ddf55c65693029180cf8a23bac Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Wed, 16 Apr 2025 13:50:18 +0300 Subject: [PATCH 08/12] [#63] Set System.Text.Json 7.0.1 as a reference Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj | 4 ++-- src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj | 4 ++-- src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index ed050a2..f8aefb7 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -6,7 +6,7 @@ enable AllEnabledByDefault FrostFS.SDK.Client - 1.0.4 + 1.0.5 C# SDK for FrostFS gRPC native protocol @@ -45,7 +45,7 @@ - + diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 43f7187..6a400c2 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Cryptography - 1.0.4 + 1.0.5 Cryptography tools for C# SDK @@ -35,7 +35,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj index 72bfe4c..81a8490 100644 --- a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj +++ b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Protos - 1.0.4 + 1.0.5 Protobuf client for C# SDK From f099edb17b9089099d87d8733a5164c715d13a17 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Thu, 17 Apr 2025 10:07:32 +0300 Subject: [PATCH 09/12] [#64] Fix for client cut logic Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/AssemblyInfo.cs | 4 ++-- src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj | 2 +- src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs | 7 ++++--- src/FrostFS.SDK.Cryptography/AssemblyInfo.cs | 4 ++-- .../FrostFS.SDK.Cryptography.csproj | 2 +- src/FrostFS.SDK.Protos/AssemblyInfo.cs | 4 ++-- src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj | 2 +- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/FrostFS.SDK.Client/AssemblyInfo.cs b/src/FrostFS.SDK.Client/AssemblyInfo.cs index 8703376..2c89c60 100644 --- a/src/FrostFS.SDK.Client/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Client/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Client")] -[assembly: AssemblyFileVersion("1.0.4.0")] +[assembly: AssemblyFileVersion("1.0.6.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Client")] [assembly: AssemblyTitle("FrostFS.SDK.Client")] -[assembly: AssemblyVersion("1.0.4")] +[assembly: AssemblyVersion("1.0.6")] diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index f8aefb7..6fb9ae4 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -6,7 +6,7 @@ enable AllEnabledByDefault FrostFS.SDK.Client - 1.0.5 + 1.0.6 C# SDK for FrostFS gRPC native protocol diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index ed9e9bc..a8b748f 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -582,14 +582,15 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0; var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; - // if the object fits one part, it can be loaded as non-complex object - if (objectsCount == 1) + // if the object fits one part, it can be loaded as non-complex object, but if it is not upload resuming + if (objectsCount == 1 && progressInfo != null && progressInfo.GetLast().Length == 0) { args.PutObjectContext.MaxObjectSizeCache = partSize; + args.PutObjectContext.FullLength = fullLength; var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false); return singlePartResult.ObjectId; } - + progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); var remain = fullLength; diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index 9f33f25..ba08d91 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Cryptography")] -[assembly: AssemblyFileVersion("1.0.4.0")] +[assembly: AssemblyFileVersion("1.0.6.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] -[assembly: AssemblyVersion("1.0.4.0")] +[assembly: AssemblyVersion("1.0.6.0")] diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 6a400c2..0bb72c5 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Cryptography - 1.0.5 + 1.0.6 Cryptography tools for C# SDK diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index c3c813c..27da7e7 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("FrostFS.SDK.Protos")] -[assembly: AssemblyFileVersion("1.0.4.0")] +[assembly: AssemblyFileVersion("1.0.6.0")] [assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] -[assembly: AssemblyVersion("1.0.4.0")] +[assembly: AssemblyVersion("1.0.6.0")] diff --git a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj index 81a8490..e165524 100644 --- a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj +++ b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Protos - 1.0.5 + 1.0.6 Protobuf client for C# SDK From 20586d8adaa44810ca6478d3538477b33af38e6f Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Thu, 17 Apr 2025 13:41:41 +0300 Subject: [PATCH 10/12] [#64] Add string naming Signed-off-by: Pavel Gross --- keyfile.snk | Bin 0 -> 596 bytes src/FrostFS.SDK.Client/ApeRules/RuleSerializer.cs | 3 --- src/FrostFS.SDK.Client/AssemblyInfo.cs | 10 ++++++++-- src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj | 2 ++ src/FrostFS.SDK.Cryptography/AssemblyInfo.cs | 3 +-- .../FrostFS.SDK.Cryptography.csproj | 2 ++ src/FrostFS.SDK.Protos/AssemblyInfo.cs | 3 +-- src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj | 8 +++++--- src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj | 10 +++++++--- 9 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 keyfile.snk diff --git a/keyfile.snk b/keyfile.snk new file mode 100644 index 0000000000000000000000000000000000000000..a14537bebfd9a433f5892391ede8ca82350481cb GIT binary patch literal 596 zcmV-a0;~N80ssI2Bme+XQ$aES1ONa50097rxsv-7>(5Jtw6UYF7?&vwQvZda1_NA2 z+*Sn3s#vZufc43^Qa|op(Yv0h&*56Khvp-knAP8@78SjBNL^AoIzMHk|>H~JiY-Y@%qoT^WXX&U3Z>e*Iks*+~|Km98<14q{T*PUo+~c7N}64uBG@1KxX*&RkS|O@aG)X+MAv2 z4LklzkL$6|$Qyhp)nCZFHM$b$kHGjJ+#ckXJU0Mh?E;0C)K`IkNKj~u3_#`y#$MHy zV6+UvVgNo#&vw!SS4*ZmrR2GW{Dcl2I_(tMA zRB$RGHap$PL5K8Id4UNFX{eLSZKZ!(T9`AP!&j7!ebndcojP;XT*h5-& iO%&N(%fUyp&bxLGnhVRV%2K@Dw@=W$$KvKDWo>MpA|Ues literal 0 HcmV?d00001 diff --git a/src/FrostFS.SDK.Client/ApeRules/RuleSerializer.cs b/src/FrostFS.SDK.Client/ApeRules/RuleSerializer.cs index 738eb54..24f784e 100644 --- a/src/FrostFS.SDK.Client/ApeRules/RuleSerializer.cs +++ b/src/FrostFS.SDK.Client/ApeRules/RuleSerializer.cs @@ -1,7 +1,4 @@ using System; -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("FrostFS.SDK.Tests")] namespace FrostFS.SDK.Client; diff --git a/src/FrostFS.SDK.Client/AssemblyInfo.cs b/src/FrostFS.SDK.Client/AssemblyInfo.cs index 2c89c60..4d87f8c 100644 --- a/src/FrostFS.SDK.Client/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Client/AssemblyInfo.cs @@ -1,8 +1,14 @@ using System.Reflection; +using System.Runtime.CompilerServices; -[assembly: AssemblyCompany("FrostFS.SDK.Client")] +[assembly: AssemblyCompany("TrueCloudLab")] +[assembly: InternalsVisibleTo("FrostFS.SDK.Tests, PublicKey=" + + "002400000480000094000000060200000024000052534131000400000100010089b992fb14ebcf"+ + "4b85b4b1a3af1897290c52ff85a106035c47dc5604cbaa58ae3180f5c9b8523fee5dd1bb9ea9cf"+ + "e15ab287e6239c98d5dfa91615bd77485d523a3a3f65a4e5028454cedd5ac4d9eca6da18b81985"+ + "ac6905d33cc64b5a2587050c16f67b71ef8889dbd3c90ef7cc0b06bbbe09886601d195f5db179a"+ + "3c2a25b1")] [assembly: AssemblyFileVersion("1.0.6.0")] -[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Client")] [assembly: AssemblyTitle("FrostFS.SDK.Client")] [assembly: AssemblyVersion("1.0.6")] diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index 6fb9ae4..037a2f4 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -32,6 +32,8 @@ false True + True + .\\..\\..\\keyfile.snk diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index ba08d91..cff5cc1 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,8 +1,7 @@ using System.Reflection; -[assembly: AssemblyCompany("FrostFS.SDK.Cryptography")] +[assembly: AssemblyCompany("TrueCloudLab")] [assembly: AssemblyFileVersion("1.0.6.0")] -[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] [assembly: AssemblyVersion("1.0.6.0")] diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 0bb72c5..20e0a83 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -26,6 +26,8 @@ false + True + .\\..\\..\\keyfile.snk diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index 27da7e7..f16be42 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,8 +1,7 @@ using System.Reflection; -[assembly: AssemblyCompany("FrostFS.SDK.Protos")] +[assembly: AssemblyCompany("TrueCloudLab")] [assembly: AssemblyFileVersion("1.0.6.0")] -[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] [assembly: AssemblyVersion("1.0.6.0")] diff --git a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj index e165524..b6bc876 100644 --- a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj +++ b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj @@ -13,19 +13,21 @@ - true + true - <_SkipUpgradeNetAnalyzersNuGetWarning>true + <_SkipUpgradeNetAnalyzersNuGetWarning>true - true + true false + True + .\\..\\..\\keyfile.snk diff --git a/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj b/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj index 30c0564..37bbace 100644 --- a/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj +++ b/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj @@ -10,11 +10,16 @@ - true + true - true + true + + + + True + .\\..\\..\\keyfile.snk @@ -42,5 +47,4 @@ PreserveNewest - From eebba7665b77a059a593ba98a403242d111c2cbc Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Wed, 23 Apr 2025 00:30:34 +0300 Subject: [PATCH 11/12] [#67] Add unit tests Signed-off-by: Pavel Gross --- .../Models/Object/UploadInfo.cs | 2 - .../Models/Response/FrostFsResponseStatus.cs | 1 + src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs | 9 +- src/FrostFS.SDK.Tests/Unit/ContainerTest.cs | 22 ++ src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs | 29 ++ .../Unit/NetmapSnapshotTests.cs | 101 +++++ .../Unit/NetworkSettingsTests.cs | 68 ++++ src/FrostFS.SDK.Tests/Unit/NetworkTest.cs | 225 ----------- src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs | 69 ++++ src/FrostFS.SDK.Tests/Unit/ObjectTest.cs | 356 +++++++++++++++++- .../Unit/ObjectToolsTests.cs | 99 +++++ .../Unit/PlacementPolicyTests.cs | 307 +++++++++++++++ src/FrostFS.SDK.Tests/Unit/SignatureTests.cs | 39 ++ 13 files changed, 1094 insertions(+), 233 deletions(-) create mode 100644 src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs create mode 100644 src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs create mode 100644 src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs delete mode 100644 src/FrostFS.SDK.Tests/Unit/NetworkTest.cs create mode 100644 src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs create mode 100644 src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs create mode 100644 src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs create mode 100644 src/FrostFS.SDK.Tests/Unit/SignatureTests.cs diff --git a/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs b/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs index 748fdfe..ca96c7d 100644 --- a/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs +++ b/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs @@ -1,5 +1,3 @@ -using FrostFS.SDK; - namespace FrostFS.SDK.Client; public readonly struct ObjectPartInfo(long offset, int length, FrostFsObjectId objectId) : System.IEquatable diff --git a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs index 9e1a846..7b003e7 100644 --- a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs +++ b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs @@ -3,6 +3,7 @@ namespace FrostFS.SDK; public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = null, string? details = null) { public FrostFsStatusCode Code { get; set; } = code; + public string Message { get; set; } = message ?? string.Empty; public string Details { get; set; } = details ?? string.Empty; diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index f33b9a1..772f92e 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -41,6 +41,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) public Collection RangeHashResponses { get; } = []; + public Action? Callback; + public override Mock GetMock() { var mock = new Mock(); @@ -165,9 +167,14 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) It.IsAny())) .Returns((PutSingleRequest r, Metadata m, DateTime? dt, CancellationToken ct) => { + Callback?.Invoke(); Verifier.CheckRequest(r); - PutSingleRequests.Add(r); + var req = r.Clone(); + + // Clone method does not clone the payload but keeps a reference + req.Body.Object.Payload = ByteString.CopyFrom(r.Body.Object.Payload.ToByteArray()); + PutSingleRequests.Add(req); return new AsyncUnaryCall( Task.FromResult(putSingleResponse), diff --git a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs index cf4bdfa..e713b4a 100644 --- a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using FrostFS.Netmap; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; using FrostFS.SDK.Cryptography; @@ -10,6 +11,27 @@ namespace FrostFS.SDK.Tests.Unit; [SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] public class ContainerTest : ContainerTestsBase { + [Theory] + [InlineData(1, "test", 0, 0)] + + public void ReplicaToMessagelTest(int count, string selector, uint ecDataCount, uint ecParityCount) + { + FrostFsReplica replica = new() + { + Count = count, + Selector = selector, + EcDataCount = ecDataCount, + EcParityCount = ecParityCount + }; + + Replica message = replica.ToMessage(); + + Assert.Equal((uint)count, message.Count); + Assert.Equal(selector, message.Selector); + Assert.Equal(ecDataCount, message.EcDataCount); + Assert.Equal(ecParityCount, message.EcParityCount); + } + [Fact] public async void CreateContainerTest() { diff --git a/src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs b/src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs new file mode 100644 index 0000000..3a71d78 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs @@ -0,0 +1,29 @@ +using FrostFS.SDK.Client; +using FrostFS.SDK.Client.Mappers.GRPC; + +namespace FrostFS.SDK.Tests.Unit; + +public class MetaheaderTests +{ + [Theory] + [InlineData(2, 13, 1, 1)] + [InlineData(200, 0, 1000000, 8)] + public void MetaheaderTest(int major, int minor, int epoch, int ttl) + { + MetaHeader metaHeader = new MetaHeader( + new FrostFsVersion( + major: 2, + minor: 13 + ), + epoch: 0, + ttl: 2 + ); + + var result = metaHeader.ToMessage(); + + Assert.Equal((ulong)metaHeader.Epoch, result.Epoch); + Assert.Equal((ulong)metaHeader.Ttl, result.Ttl); + Assert.Equal((ulong)metaHeader.Version.Major, result.Version.Major); + Assert.Equal((ulong)metaHeader.Version.Minor, result.Version.Minor); + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs b/src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs new file mode 100644 index 0000000..233e12e --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs @@ -0,0 +1,101 @@ +using FrostFS.Netmap; +using FrostFS.SDK.Client; + +using Google.Protobuf; + +namespace FrostFS.SDK.Tests.Unit; + +public class NetmapSnapshotTests : NetworkTestsBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + + public async void NetmapSnapshotTest(bool useContext) + { + var body = new NetmapSnapshotResponse.Types.Body + { + Netmap = new Netmap.Netmap { Epoch = 99 } + }; + + var nodeInfo1 = new NodeInfo + { + State = NodeInfo.Types.State.Online, + PublicKey = ByteString.CopyFrom([1, 2, 3]) + }; + + nodeInfo1.Addresses.Add("address1"); + nodeInfo1.Addresses.Add("address2"); + nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); + nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); + + var nodeInfo2 = new NodeInfo + { + State = NodeInfo.Types.State.Offline, + PublicKey = ByteString.CopyFrom([3, 4, 5]) + }; + + nodeInfo2.Addresses.Add("address3"); + nodeInfo2.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key3", Value = "value3" }); + + body.Netmap.Nodes.Add(nodeInfo1); + body.Netmap.Nodes.Add(nodeInfo2); + + Mocker.NetmapSnapshotResponse = new NetmapSnapshotResponse { Body = body }; + + var ctx = useContext + ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) + : default; + + var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); + + var result = await GetClient(DefaultSettings).GetNetmapSnapshotAsync(ctx); + + var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); + + Assert.NotNull(result); + + Assert.Equal(99u, result.Epoch); + Assert.Equal(2, result.NodeInfoCollection.Count); + + var node1 = result.NodeInfoCollection[0]; + Assert.Equal(NodeState.Online, node1.State); + Assert.Equal(2, node1.Addresses.Count); + Assert.Equal("address1", node1.Addresses.ElementAt(0)); + Assert.Equal("address2", node1.Addresses.ElementAt(1)); + + Assert.Equal(2, node1.Attributes.Count); + + Assert.Equal("key1", node1.Attributes.ElementAt(0).Key); + Assert.Equal("value1", node1.Attributes.ElementAt(0).Value); + Assert.Equal("key2", node1.Attributes.ElementAt(1).Key); + Assert.Equal("value2", node1.Attributes.ElementAt(1).Value); + + var node2 = result.NodeInfoCollection[1]; + Assert.Equal(NodeState.Offline, node2.State); + Assert.Single(node2.Addresses); + Assert.Equal("address3", node2.Addresses.ElementAt(0)); + + Assert.Single(node2.Attributes); + + Assert.Equal("key3", node2.Attributes.ElementAt(0).Key); + Assert.Equal("value3", node2.Attributes.ElementAt(0).Value); + + if (useContext) + { + Assert.NotNull(Mocker.NetmapSnapshotRequest); + Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); + + Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); + Assert.NotNull(Mocker.DateTime); + Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); + Assert.True(Mocker.DateTime.Value <= validTimeoutTo); + } + else + { + Assert.NotNull(Mocker.NetmapSnapshotRequest); + Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); + Assert.Null(Mocker.DateTime); + } + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs b/src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs new file mode 100644 index 0000000..3f3fb2f --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs @@ -0,0 +1,68 @@ +using System.Diagnostics.CodeAnalysis; +using FrostFS.SDK.Client; + +namespace FrostFS.SDK.Tests.Unit; + +[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] +public class NetworkSettingsTests : NetworkTestsBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async void NetworkSettingsTest(bool useContext) + { + Mocker.Parameters.Add("AuditFee", [1]); + Mocker.Parameters.Add("BasicIncomeRate", [2]); + Mocker.Parameters.Add("ContainerFee", [3]); + Mocker.Parameters.Add("ContainerAliasFee", [4]); + Mocker.Parameters.Add("EpochDuration", [5]); + Mocker.Parameters.Add("InnerRingCandidateFee", [6]); + Mocker.Parameters.Add("MaxECDataCount", [7]); + Mocker.Parameters.Add("MaxECParityCount", [8]); + Mocker.Parameters.Add("MaxObjectSize", [9]); + Mocker.Parameters.Add("WithdrawFee", [10]); + Mocker.Parameters.Add("HomomorphicHashingDisabled", [1]); + Mocker.Parameters.Add("MaintenanceModeAllowed", [1]); + + var ctx = useContext + ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) + : default; + + var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); + + var result = await GetClient(DefaultSettings).GetNetworkSettingsAsync(ctx); + + var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); + + Assert.NotNull(result); + + Assert.Equal(Mocker.Parameters["AuditFee"], [(byte)result.AuditFee]); + Assert.Equal(Mocker.Parameters["BasicIncomeRate"], [(byte)result.BasicIncomeRate]); + Assert.Equal(Mocker.Parameters["ContainerFee"], [(byte)result.ContainerFee]); + Assert.Equal(Mocker.Parameters["ContainerAliasFee"], [(byte)result.ContainerAliasFee]); + Assert.Equal(Mocker.Parameters["EpochDuration"], [(byte)result.EpochDuration]); + Assert.Equal(Mocker.Parameters["InnerRingCandidateFee"], [(byte)result.InnerRingCandidateFee]); + Assert.Equal(Mocker.Parameters["MaxECDataCount"], [(byte)result.MaxECDataCount]); + Assert.Equal(Mocker.Parameters["MaxECParityCount"], [(byte)result.MaxECParityCount]); + Assert.Equal(Mocker.Parameters["MaxObjectSize"], [(byte)result.MaxObjectSize]); + Assert.Equal(Mocker.Parameters["WithdrawFee"], [(byte)result.WithdrawFee]); + + Assert.True(result.HomomorphicHashingDisabled); + Assert.True(result.MaintenanceModeAllowed); + + if (useContext) + { + Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); + Assert.NotNull(Mocker.DateTime); + + Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); + Assert.True(Mocker.DateTime.Value <= validTimeoutTo); + } + else + { + Assert.NotNull(Mocker.NetworkInfoRequest); + Assert.Empty(Mocker.NetworkInfoRequest.MetaHeader.XHeaders); + Assert.Null(Mocker.DateTime); + } + } +} diff --git a/src/FrostFS.SDK.Tests/Unit/NetworkTest.cs b/src/FrostFS.SDK.Tests/Unit/NetworkTest.cs deleted file mode 100644 index b1146f7..0000000 --- a/src/FrostFS.SDK.Tests/Unit/NetworkTest.cs +++ /dev/null @@ -1,225 +0,0 @@ -using System.Diagnostics.CodeAnalysis; - -using FrostFS.Netmap; -using FrostFS.SDK.Client; - -using Google.Protobuf; - -namespace FrostFS.SDK.Tests.Unit; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -public class NetworkTest : NetworkTestsBase -{ - [Theory] - [InlineData(false)] - [InlineData(true)] - public async void NetworkSettingsTest(bool useContext) - { - Mocker.Parameters.Add("AuditFee", [1]); - Mocker.Parameters.Add("BasicIncomeRate", [2]); - Mocker.Parameters.Add("ContainerFee", [3]); - Mocker.Parameters.Add("ContainerAliasFee", [4]); - Mocker.Parameters.Add("EpochDuration", [5]); - Mocker.Parameters.Add("InnerRingCandidateFee", [6]); - Mocker.Parameters.Add("MaxECDataCount", [7]); - Mocker.Parameters.Add("MaxECParityCount", [8]); - Mocker.Parameters.Add("MaxObjectSize", [9]); - Mocker.Parameters.Add("WithdrawFee", [10]); - Mocker.Parameters.Add("HomomorphicHashingDisabled", [1]); - Mocker.Parameters.Add("MaintenanceModeAllowed", [1]); - - var ctx = useContext - ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) - : default; - - var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); - - var result = await GetClient(DefaultSettings).GetNetworkSettingsAsync(ctx); - - var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); - - Assert.NotNull(result); - - Assert.Equal(Mocker.Parameters["AuditFee"], [(byte)result.AuditFee]); - Assert.Equal(Mocker.Parameters["BasicIncomeRate"], [(byte)result.BasicIncomeRate]); - Assert.Equal(Mocker.Parameters["ContainerFee"], [(byte)result.ContainerFee]); - Assert.Equal(Mocker.Parameters["ContainerAliasFee"], [(byte)result.ContainerAliasFee]); - Assert.Equal(Mocker.Parameters["EpochDuration"], [(byte)result.EpochDuration]); - Assert.Equal(Mocker.Parameters["InnerRingCandidateFee"], [(byte)result.InnerRingCandidateFee]); - Assert.Equal(Mocker.Parameters["MaxECDataCount"], [(byte)result.MaxECDataCount]); - Assert.Equal(Mocker.Parameters["MaxECParityCount"], [(byte)result.MaxECParityCount]); - Assert.Equal(Mocker.Parameters["MaxObjectSize"], [(byte)result.MaxObjectSize]); - Assert.Equal(Mocker.Parameters["WithdrawFee"], [(byte)result.WithdrawFee]); - - Assert.True(result.HomomorphicHashingDisabled); - Assert.True(result.MaintenanceModeAllowed); - - if (useContext) - { - Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); - Assert.NotNull(Mocker.DateTime); - - Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); - Assert.True(Mocker.DateTime.Value <= validTimeoutTo); - } - else - { - Assert.NotNull(Mocker.NetworkInfoRequest); - Assert.Empty(Mocker.NetworkInfoRequest.MetaHeader.XHeaders); - Assert.Null(Mocker.DateTime); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - - public async void NetmapSnapshotTest(bool useContext) - { - var body = new NetmapSnapshotResponse.Types.Body - { - Netmap = new Netmap.Netmap { Epoch = 99 } - }; - - var nodeInfo1 = new NodeInfo - { - State = NodeInfo.Types.State.Online, - PublicKey = ByteString.CopyFrom([1, 2, 3]) - }; - - nodeInfo1.Addresses.Add("address1"); - nodeInfo1.Addresses.Add("address2"); - nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); - nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); - - var nodeInfo2 = new NodeInfo - { - State = NodeInfo.Types.State.Offline, - PublicKey = ByteString.CopyFrom([3, 4, 5]) - }; - - nodeInfo2.Addresses.Add("address3"); - nodeInfo2.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key3", Value = "value3" }); - - body.Netmap.Nodes.Add(nodeInfo1); - body.Netmap.Nodes.Add(nodeInfo2); - - Mocker.NetmapSnapshotResponse = new NetmapSnapshotResponse { Body = body }; - - var ctx = useContext - ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) - : default; - - var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); - - var result = await GetClient(DefaultSettings).GetNetmapSnapshotAsync(ctx); - - var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); - - Assert.NotNull(result); - - Assert.Equal(99u, result.Epoch); - Assert.Equal(2, result.NodeInfoCollection.Count); - - var node1 = result.NodeInfoCollection[0]; - Assert.Equal(NodeState.Online, node1.State); - Assert.Equal(2, node1.Addresses.Count); - Assert.Equal("address1", node1.Addresses.ElementAt(0)); - Assert.Equal("address2", node1.Addresses.ElementAt(1)); - - Assert.Equal(2, node1.Attributes.Count); - - Assert.Equal("key1", node1.Attributes.ElementAt(0).Key); - Assert.Equal("value1", node1.Attributes.ElementAt(0).Value); - Assert.Equal("key2", node1.Attributes.ElementAt(1).Key); - Assert.Equal("value2", node1.Attributes.ElementAt(1).Value); - - var node2 = result.NodeInfoCollection[1]; - Assert.Equal(NodeState.Offline, node2.State); - Assert.Single(node2.Addresses); - Assert.Equal("address3", node2.Addresses.ElementAt(0)); - - Assert.Single(node2.Attributes); - - Assert.Equal("key3", node2.Attributes.ElementAt(0).Key); - Assert.Equal("value3", node2.Attributes.ElementAt(0).Value); - - if (useContext) - { - Assert.NotNull(Mocker.NetmapSnapshotRequest); - Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); - - Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); - Assert.NotNull(Mocker.DateTime); - Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); - Assert.True(Mocker.DateTime.Value <= validTimeoutTo); - } - else - { - Assert.NotNull(Mocker.NetmapSnapshotRequest); - Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); - Assert.Null(Mocker.DateTime); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async void NodeInfoTest(bool useContext) - { - var body = new LocalNodeInfoResponse.Types.Body - { - NodeInfo = new NodeInfo() - { - State = NodeInfo.Types.State.Online, - PublicKey = ByteString.CopyFrom([1, 2, 3]) - }, - Version = new Refs.Version { Major = 2, Minor = 12 } - }; - - body.NodeInfo.Addresses.Add("address1"); - body.NodeInfo.Addresses.Add("address2"); - body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); - body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); - - Mocker.NodeInfoResponse = new LocalNodeInfoResponse { Body = body }; - - var ctx = useContext - ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) - : default; - - var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); - - var result = await GetClient(DefaultSettings).GetNodeInfoAsync(ctx); - - var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); - - Assert.NotNull(result); - - Assert.Equal(NodeState.Online, result.State); - - Assert.Equal(2, result.Addresses.Count); - Assert.Equal("address1", result.Addresses.ElementAt(0)); - Assert.Equal("address2", result.Addresses.ElementAt(1)); - - Assert.Equal(2, result.Attributes.Count); - Assert.Equal("value1", result.Attributes["key1"]); - Assert.Equal("value2", result.Attributes["key2"]); - - Assert.NotNull(Mocker.LocalNodeInfoRequest); - if (useContext) - { - Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); - Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); - Assert.NotNull(Mocker.DateTime); - - Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); - Assert.True(Mocker.DateTime.Value <= validTimeoutTo); - } - else - { - Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); - Assert.Null(Mocker.DateTime); - } - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs b/src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs new file mode 100644 index 0000000..6449beb --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs @@ -0,0 +1,69 @@ +using FrostFS.Netmap; +using FrostFS.SDK.Client; +using Google.Protobuf; + +namespace FrostFS.SDK.Tests.Unit; + +public class NodeInfoTests : NetworkTestsBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async void NodeInfoTest(bool useContext) + { + var body = new LocalNodeInfoResponse.Types.Body + { + NodeInfo = new NodeInfo() + { + State = NodeInfo.Types.State.Online, + PublicKey = ByteString.CopyFrom([1, 2, 3]) + }, + Version = new Refs.Version { Major = 2, Minor = 12 } + }; + + body.NodeInfo.Addresses.Add("address1"); + body.NodeInfo.Addresses.Add("address2"); + body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); + body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); + + Mocker.NodeInfoResponse = new LocalNodeInfoResponse { Body = body }; + + var ctx = useContext + ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) + : default; + + var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); + + var result = await GetClient(DefaultSettings).GetNodeInfoAsync(ctx); + + var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); + + Assert.NotNull(result); + + Assert.Equal(NodeState.Online, result.State); + + Assert.Equal(2, result.Addresses.Count); + Assert.Equal("address1", result.Addresses.ElementAt(0)); + Assert.Equal("address2", result.Addresses.ElementAt(1)); + + Assert.Equal(2, result.Attributes.Count); + Assert.Equal("value1", result.Attributes["key1"]); + Assert.Equal("value2", result.Attributes["key2"]); + + Assert.NotNull(Mocker.LocalNodeInfoRequest); + if (useContext) + { + Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); + Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); + Assert.NotNull(Mocker.DateTime); + + Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); + Assert.True(Mocker.DateTime.Value <= validTimeoutTo); + } + else + { + Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); + Assert.Null(Mocker.DateTime); + } + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs index 7a595e6..985f02e 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs @@ -6,8 +6,9 @@ using System.Text; using FrostFS.Refs; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; - +using FrostFS.SDK.Cryptography; using Google.Protobuf; +using Org.BouncyCastle.Utilities; namespace FrostFS.SDK.Tests.Unit; @@ -60,7 +61,7 @@ public class ObjectTest : ObjectTestsBase var param = new PrmObjectClientCutPut( Mocker.ObjectHeader, payload: new MemoryStream(bytes), - bufferMaxSize: 1024); + bufferMaxSize: blockSize); Random rnd = new(); @@ -87,7 +88,7 @@ public class ObjectTest : ObjectTestsBase // PART1 Assert.Equal(blockSize, objects[0].Payload.Length); - Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload)); + Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); Assert.NotNull(objects[0].Header.Split.SplitId); Assert.Null(objects[0].Header.Split.Previous); @@ -96,7 +97,7 @@ public class ObjectTest : ObjectTestsBase // PART2 Assert.Equal(blockSize, objects[1].Payload.Length); - Assert.True(bytes.AsMemory(blockSize, blockSize).ToArray().SequenceEqual(objects[1].Payload)); + Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); Assert.True(objects[1].Header.Attributes.Count == 0); @@ -104,7 +105,7 @@ public class ObjectTest : ObjectTestsBase // last part Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); - Assert.True(bytes.AsMemory(2 * blockSize).ToArray().SequenceEqual(objects[2].Payload)); + Assert.Equal(bytes.AsMemory(2 * blockSize).ToArray(), objects[2].Payload); Assert.NotNull(objects[3].Header.Split.Parent); Assert.NotNull(objects[3].Header.Split.ParentHeader); @@ -128,6 +129,351 @@ public class ObjectTest : ObjectTestsBase Assert.Equal(result.Value, modelObjId.ToString()); } + [Fact] + public async void ClientCutWithInterruptionOnFirstPartTest() + { + NetworkMocker.Parameters.Add("MaxObjectSize", [0x0, 0xa]); + + var blockSize = 2560; + byte[] bytes = File.ReadAllBytes(@".\..\..\..\cat.jpg"); + var fileLength = bytes.Length; + + var splitId = Guid.NewGuid(); + var progress = new UploadProgressInfo(splitId); + + var param = new PrmObjectClientCutPut( + Mocker.ObjectHeader, + payload: new MemoryStream(bytes), + bufferMaxSize: blockSize, + progress: progress); + + Random rnd = new(); + + Collection objIds = new([new byte[32], new byte[32], new byte[32]]); + rnd.NextBytes(objIds.ElementAt(0)); + rnd.NextBytes(objIds.ElementAt(1)); + rnd.NextBytes(objIds.ElementAt(2)); + + foreach (var objId in objIds) + Mocker.ResultObjectIds!.Add(objId); + + int sentBlockCount = 0; + Mocker.Callback = () => + { + if (++sentBlockCount == 1) + throw new FrostFsException("some error"); + }; + + bool gotException = false; + try + { + var result = await GetClient().PutClientCutObjectAsync(param, default); + } + catch (FrostFsException ex) + { + if (ex.Message == "some error") + gotException = true; + } + + Assert.True(gotException); + + var singleObjects = Mocker.PutSingleRequests.ToArray(); + + Assert.Empty(singleObjects); + } + + [Fact] + public async void ClientCutWithInterruptionOnMiddlePartTest() + { + NetworkMocker.Parameters.Add("MaxObjectSize", [0x0, 0xa]); + + var blockSize = 2560; + byte[] bytes = File.ReadAllBytes(@".\..\..\..\cat.jpg"); + var fileLength = bytes.Length; + + var splitId = Guid.Parse("67e4bbe9-86ca-474d-9385-6569ce89db61"); + var progress = new UploadProgressInfo(splitId); + + var param = new PrmObjectClientCutPut( + Mocker.ObjectHeader, + payload: new MemoryStream(bytes), + bufferMaxSize: blockSize, + progress: progress); + + Random rnd = new(); + + Collection objIds = new([new byte[32], new byte[32], new byte[32]]); + rnd.NextBytes(objIds.ElementAt(0)); + rnd.NextBytes(objIds.ElementAt(1)); + rnd.NextBytes(objIds.ElementAt(2)); + + foreach (var objId in objIds) + Mocker.ResultObjectIds!.Add(objId); + + int sentBlockCount = 0; + Mocker.Callback = () => + { + if (++sentBlockCount == 2) + throw new FrostFsException("some error"); + }; + + bool gotException = false; + try + { + _ = await GetClient().PutClientCutObjectAsync(param, default); + } + catch (FrostFsException ex) + { + if (ex.Message == "some error") + gotException = true; + } + + Assert.True(gotException); + + var singleObjects = Mocker.PutSingleRequests.ToArray(); + + Assert.NotNull(Mocker.ClientStreamWriter?.Messages); + + var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); + + Assert.Single(objects); + + Assert.Single(progress.GetParts()); + + var part = progress.GetPart(0); + Assert.Equal(0, part.Offset); + Assert.Equal(2560, part.Length); + + // PART1 + Assert.Equal(blockSize, objects[0].Payload.Length); + Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); + + Assert.NotNull(objects[0].Header.Split.SplitId); + Assert.Null(objects[0].Header.Split.Previous); + Assert.True(objects[0].Header.Attributes.Count == 0); + Assert.Null(objects[0].Header.Split.Parent); + + // resume uploading + sentBlockCount = 10; + + var result = await GetClient().PutClientCutObjectAsync(param, default); + + singleObjects = Mocker.PutSingleRequests.ToArray(); + + Assert.NotNull(Mocker.ClientStreamWriter?.Messages); + + objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); + + Assert.Equal(4, objects.Length); + + // PART1 + Assert.Equal(blockSize, objects[0].Payload.Length); + Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); + + Assert.NotNull(objects[0].Header.Split.SplitId); + Assert.Null(objects[0].Header.Split.Previous); + Assert.True(objects[0].Header.Attributes.Count == 0); + Assert.Null(objects[0].Header.Split.Parent); + + // PART2 + Assert.Equal(blockSize, objects[1].Payload.Length); + Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); + + Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); + Assert.True(objects[1].Header.Attributes.Count == 0); + Assert.Null(objects[1].Header.Split.Parent); + + // last part + Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); + Assert.Equal(bytes.AsMemory(2 * blockSize).ToArray(), objects[2].Payload); + + Assert.NotNull(objects[3].Header.Split.Parent); + Assert.NotNull(objects[3].Header.Split.ParentHeader); + Assert.NotNull(objects[3].Header.Split.ParentSignature); + Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); + Assert.True(objects[2].Header.Attributes.Count == 0); + + // link object + Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent); + Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader); + Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); + Assert.Equal(0, (int)objects[3].Header.PayloadLength); + Assert.True(objects[3].Header.Attributes.Count == 0); + + Assert.Single(objects[3].Header.Split.ParentHeader.Attributes); + Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key); + Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value); + + var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray()); + + Assert.Equal(result.Value, modelObjId.ToString()); + + Assert.Equal(3, progress.GetParts().Count); + part = progress.GetPart(0); + Assert.Equal(0, part.Offset); + Assert.Equal(2560, part.Length); + + part = progress.GetPart(1); + Assert.Equal(2560, part.Offset); + Assert.Equal(2560, part.Length); + + part = progress.GetPart(2); + Assert.Equal(2 * 2560, part.Offset); + Assert.Equal(1620, part.Length); + } + + [Fact] + public async void ClientCutWithInterruptionOnLastPartTest() + { + NetworkMocker.Parameters.Add("MaxObjectSize", [0x0, 0xa]); + + var blockSize = 2560; + byte[] bytes = File.ReadAllBytes(@".\..\..\..\cat.jpg"); + var fileLength = bytes.Length; + + var splitId = Guid.Parse("67e4bbe9-86ca-474d-9385-6569ce89db61"); + var progress = new UploadProgressInfo(splitId); + + var param = new PrmObjectClientCutPut( + Mocker.ObjectHeader, + payload: new MemoryStream(bytes), + bufferMaxSize: blockSize, + progress: progress); + + Random rnd = new(); + + Collection objIds = new([new byte[32], new byte[32], new byte[32]]); + rnd.NextBytes(objIds.ElementAt(0)); + rnd.NextBytes(objIds.ElementAt(1)); + rnd.NextBytes(objIds.ElementAt(2)); + + foreach (var objId in objIds) + Mocker.ResultObjectIds!.Add(objId); + + int sentBlockCount = 0; + Mocker.Callback = () => + { + if (++sentBlockCount == 3) + throw new FrostFsException("some error"); + }; + + bool gotException = false; + try + { + _ = await GetClient().PutClientCutObjectAsync(param, default); + } + catch (FrostFsException ex) + { + if (ex.Message == "some error") + gotException = true; + } + + Assert.True(gotException); + + var singleObjects = Mocker.PutSingleRequests.ToArray(); + + Assert.NotNull(Mocker.ClientStreamWriter?.Messages); + + var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); + + Assert.Equal(2, objects.Length); + + Assert.Equal(2, progress.GetParts().Count); + + var part = progress.GetPart(0); + Assert.Equal(0, part.Offset); + Assert.Equal(2560, part.Length); + + part = progress.GetPart(1); + Assert.Equal(2560, part.Offset); + Assert.Equal(2560, part.Length); + + // PART1 + Assert.Equal(blockSize, objects[0].Payload.Length); + Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); + + Assert.NotNull(objects[0].Header.Split.SplitId); + Assert.Null(objects[0].Header.Split.Previous); + Assert.True(objects[0].Header.Attributes.Count == 0); + Assert.Null(objects[0].Header.Split.Parent); + + // PART2 + Assert.Equal(blockSize, objects[1].Payload.Length); + Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); + + Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); + Assert.True(objects[1].Header.Attributes.Count == 0); + Assert.Null(objects[1].Header.Split.Parent); + + // resume uploading + sentBlockCount = 10; + + var result = await GetClient().PutClientCutObjectAsync(param, default); + + singleObjects = Mocker.PutSingleRequests.ToArray(); + + Assert.NotNull(Mocker.ClientStreamWriter?.Messages); + + objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); + + Assert.Equal(4, objects.Length); + + // PART1 + Assert.Equal(blockSize, objects[0].Payload.Length); + Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); + + Assert.NotNull(objects[0].Header.Split.SplitId); + Assert.Null(objects[0].Header.Split.Previous); + Assert.True(objects[0].Header.Attributes.Count == 0); + Assert.Null(objects[0].Header.Split.Parent); + + // PART2 + Assert.Equal(blockSize, objects[1].Payload.Length); + Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); + + Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); + Assert.True(objects[1].Header.Attributes.Count == 0); + Assert.Null(objects[1].Header.Split.Parent); + + // last part + Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); + Assert.Equal(bytes.AsMemory(2 * blockSize).ToArray(), objects[2].Payload); + + Assert.NotNull(objects[3].Header.Split.Parent); + Assert.NotNull(objects[3].Header.Split.ParentHeader); + Assert.NotNull(objects[3].Header.Split.ParentSignature); + Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); + Assert.True(objects[2].Header.Attributes.Count == 0); + + // link object + Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent); + Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader); + Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); + Assert.Equal(0, (int)objects[3].Header.PayloadLength); + Assert.True(objects[3].Header.Attributes.Count == 0); + + Assert.Single(objects[3].Header.Split.ParentHeader.Attributes); + Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key); + Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value); + + var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray()); + + Assert.Equal(result.Value, modelObjId.ToString()); + + Assert.Equal(3, progress.GetParts().Count); + part = progress.GetPart(0); + Assert.Equal(0, part.Offset); + Assert.Equal(2560, part.Length); + + part = progress.GetPart(1); + Assert.Equal(2560, part.Offset); + Assert.Equal(2560, part.Length); + + part = progress.GetPart(2); + Assert.Equal(2 * 2560, part.Offset); + Assert.Equal(1620, part.Length); + } + [Fact] public async void DeleteObject() { diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs b/src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs new file mode 100644 index 0000000..3c0245b --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs @@ -0,0 +1,99 @@ +using System.Security.Cryptography; +using System.Text; +using FrostFS.SDK.Client; +using FrostFS.SDK.Cryptography; + +namespace FrostFS.SDK.Tests.Unit; + +public class ObjectToolsTests +{ + internal readonly string key = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; + + [Fact] + public void CalculateObjectIdTest() + { + var payload = Encoding.UTF8.GetBytes("testPayload"); + + var payloadHash = SHA256.HashData(payload); + + FrostFsContainerId containerId = new("test"); + FrostFsObjectHeader header = new(containerId); + + var ecdsaKey = key.LoadWif(); + var owner = FrostFsOwner.FromKey(ecdsaKey); + + var clientKey = new ClientKey(ecdsaKey); + + var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); + + Assert.NotNull(objId.Value); + Assert.Equal("HuAojwCYi62iUKr1FtSCCkMLLWv1uAnznF8iSb1bRV1N", objId.Value); + } + + [Fact] + public void CalculateObjectIdTest1() + { + var payload = Encoding.UTF8.GetBytes("testPayload"); + + var payloadHash = SHA256.HashData(payload); + var ecdsaKey = key.LoadWif(); + var owner = FrostFsOwner.FromKey(ecdsaKey); + + var clientKey = new ClientKey(ecdsaKey); + FrostFsContainerId containerId = new("test"); + FrostFsObjectHeader header = new(containerId, FrostFsObjectType.Regular, null, null, owner, new FrostFsVersion(2, 13)); + + var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); + + Assert.NotNull(objId.Value); + Assert.Equal("HuAojwCYi62iUKr1FtSCCkMLLWv1uAnznF8iSb1bRV1N", objId.Value); + } + + [Fact] + public void CalculateObjectIdWithAttrTest() + { + var payload = Encoding.UTF8.GetBytes("testPayload"); + + var payloadHash = SHA256.HashData(payload); + var ecdsaKey = key.LoadWif(); + var owner = FrostFsOwner.FromKey(ecdsaKey); + + var clientKey = new ClientKey(ecdsaKey); + FrostFsContainerId containerId = new("test"); + + FrostFsAttributePair[] attribs = [new("key", "val")]; + + FrostFsObjectHeader header = new(containerId, FrostFsObjectType.Regular, attribs, null, owner, new FrostFsVersion(2, 13)); + + var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); + + Assert.NotNull(objId.Value); + Assert.Equal("4zq5NYEbzkrfmdKne3GnpavE24gU2PnuV17ZExb9hcn3", objId.Value); + } + + [Fact] + public void CalculateObjectIdWithSplitIdTest() + { + var payload = Encoding.UTF8.GetBytes("testPayload"); + + var payloadHash = SHA256.HashData(payload); + var ecdsaKey = key.LoadWif(); + var owner = FrostFsOwner.FromKey(ecdsaKey); + + var clientKey = new ClientKey(ecdsaKey); + FrostFsContainerId containerId = new("test"); + + FrostFsAttributePair[] attribs = [new("key", "val")]; + + var guid = Guid.Parse("790a8d04-f5c3-4cd6-b46f-a78ee7e325f2"); + SplitId splitId = new(guid); + FrostFsSplit split = new (splitId); + + FrostFsObjectHeader header = new(containerId, FrostFsObjectType.Regular, attribs, split, owner, new FrostFsVersion(2, 13)); + + var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); + + Assert.NotNull(objId.Value); + Assert.Equal("HCYzsuXyfe5LmQzi58hPQxExGPAFv7dU5TzEACLxM1os", objId.Value); + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs b/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs new file mode 100644 index 0000000..9957c63 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs @@ -0,0 +1,307 @@ +using System.Xml.Linq; +using FrostFS.Netmap; +using FrostFS.SDK.Client; +using Google.Protobuf.WellKnownTypes; + +namespace FrostFS.SDK.Tests.Unit; + +public class PlacementPolicyTests : NetworkTestsBase +{ + [Theory] + [InlineData(true, 1)] + [InlineData(true, 3)] + [InlineData(true, 5)] + [InlineData(false, 1)] + [InlineData(false, 3)] + [InlineData(false, 5)] + public void PlacementPolicySimpleFullTest(bool unique, uint backupFactor) + { + PlacementPolicy policy = new() + { + ContainerBackupFactor = backupFactor, + Unique = unique + }; + + var result = policy.ToModel(); + + Assert.Equal(backupFactor, result.BackupFactor); + Assert.Equal(unique, result.Unique); + Assert.Empty(result.Filters); + Assert.Empty(result.Replicas); + Assert.Empty(result.Selectors); + } + + [Fact] + public void PlacementPolicyFullTest() + { + PlacementPolicy policy = new() + { + ContainerBackupFactor = 3, + Unique = true + }; + + policy.Filters.AddRange( + [ + new () { Name = "filter1", Key = "filterKey1", Op =Operation.Eq, Value = "testValue1" }, + new () { Name = "filter2", Key = "filterKey2", Op =Operation.And, Value = "testValue2" } + ]); + + policy.Selectors.AddRange( + [ + new () { Name = "name1", Attribute = "attrib1", Clause = Clause.Same, Count = 5, Filter = "filter1" }, + new () { Name = "name2", Attribute = "attrib2", Clause = Clause.Distinct, Count = 4, Filter = "filter2" } + ]); + + policy.Replicas.AddRange( + [ + new () { EcDataCount = 2, EcParityCount = 3, Count = 4, Selector = "selector1"}, + new () { EcDataCount = 5, EcParityCount = 6, Count = 7, Selector = "selector2"}, + ]); + + var result = policy.ToModel(); + + Assert.Equal(3L, result.BackupFactor); + Assert.True(result.Unique); + Assert.Equal(2, result.Filters.Count); + Assert.Equal(2, result.Replicas.Length); + Assert.Equal(2, result.Selectors.Count); + + var rep0 = result.Replicas[0]; + Assert.Equal(2u, rep0.EcDataCount); + Assert.Equal(3u, rep0.EcParityCount); + Assert.Equal(4, rep0.Count); + Assert.Equal("selector1", rep0.Selector); + + var rep1 = result.Replicas[1]; + Assert.Equal(5u, rep1.EcDataCount); + Assert.Equal(6u, rep1.EcParityCount); + Assert.Equal(7, rep1.Count); + Assert.Equal("selector2", rep1.Selector); + + var f0 = result.Filters[0]; + Assert.Equal("filterKey1", f0.Key); + Assert.Equal("filter1", f0.Name); + Assert.Equal(1, f0.Operation); + Assert.Equal("testValue1", f0.Value); + + var f1 = result.Filters[1]; + Assert.Equal("filterKey2", f1.Key); + Assert.Equal("filter2", f1.Name); + Assert.Equal(8, f1.Operation); + Assert.Equal("testValue2", f1.Value); + + var s0 = result.Selectors[0]; + Assert.Equal("name1", s0.Name); + Assert.Equal("attrib1", s0.Attribute); + Assert.Equal(1, s0.Clause); + Assert.Equal(5L, s0.Count); + Assert.Equal("filter1", s0.Filter); + + var s1 = result.Selectors[1]; + Assert.Equal("name2", s1.Name); + Assert.Equal("attrib2", s1.Attribute); + Assert.Equal(2, s1.Clause); + Assert.Equal(4L, s1.Count); + Assert.Equal("filter2", s1.Filter); + } + + + [Theory] + [InlineData(1, "test" , 0, 0)] + [InlineData(1, "", 1000, 9999)] + [InlineData(1, "some long text to test reasonable length of the selector name", 100000000, 100000001)] + [InlineData(100, "test2", 1, 1)] + [InlineData(1, " ", 2, 3)] + [InlineData(10, "!", 0, 0)] + [InlineData(1, "123", 0, 0)] + public void ReplicaToModelTest(uint count, string selector, uint ecDataCount, uint ecParityCount) + { + Replica replica = new () + { + Count = count, + Selector = selector, + EcDataCount = ecDataCount, + EcParityCount = ecParityCount + }; + + FrostFsReplica model = replica.ToModel(); + + Assert.Equal(count, (uint)model.Count); + Assert.Equal(selector, model.Selector); + Assert.Equal(ecDataCount, model.EcDataCount); + Assert.Equal(ecParityCount, model.EcParityCount); + } + + [Theory] + [InlineData(1, "test", 0, 0)] + [InlineData(1, "", 1000, 9999)] + [InlineData(1, "some long text to test reasonable length of the selector name", 100000000, 100000001)] + [InlineData(100, "test2", 1, 1)] + [InlineData(1, " ", 2, 3)] + [InlineData(10, "!", 0, 0)] + [InlineData(1, "123", 0, 0)] + public void ReplicaToMessagelTest(int count, string selector, uint ecDataCount, uint ecParityCount) + { + FrostFsReplica replica = new () + { + Count = count, + Selector = selector, + EcDataCount = ecDataCount, + EcParityCount = ecParityCount + }; + + Replica message = replica.ToMessage(); + + Assert.Equal((uint)count, message.Count); + Assert.Equal(selector, message.Selector); + Assert.Equal(ecDataCount, message.EcDataCount); + Assert.Equal(ecParityCount, message.EcParityCount); + } + + [Theory] + [InlineData("test", 1, 2, "attribute", "filter")] + [InlineData("test", 0, 0, "longlonglonglonglonglonglonglonglonglonglonglonglong attribute", "longlonglonglonglonglonglonglonglonglonglonglonglong filter")] + [InlineData("test", 0, 1, "attribute", "filter")] + public void SelectorToMessageTest(string name, uint count, int clause, string attr, string filter) + { + FrostFsSelector selector = new (name) + { + Count = count, + Clause = clause, + Attribute = attr, + Filter = filter, + }; + + var message = selector.ToMessage(); + + Assert.Equal(name, message.Name); + Assert.Equal(count, message.Count); + Assert.Equal(clause, (int)message.Clause); + Assert.Equal(attr, message.Attribute); + Assert.Equal(filter, message.Filter); + + } + + [Theory] + [InlineData("test", 1, Clause.Same, "attribute", "filter")] + [InlineData("test", 0, Clause.Distinct, "longlonglonglonglonglonglonglonglonglonglonglonglong attribute", "longlonglonglonglonglonglonglonglonglonglonglonglong filter")] + public void SelectorToModelTest(string name, uint count, Clause clause, string attr, string filter) + { + Selector selector = new () + { + Name = name, + Count = count, + Clause = clause, + Attribute = attr, + Filter = filter + }; + + var model = selector.ToModel(); + + Assert.Equal(name, model.Name); + Assert.Equal(count, model.Count); + Assert.Equal((int)clause, model.Clause); + Assert.Equal(attr, model.Attribute); + Assert.Equal(filter, model.Filter); + } + + [Theory] + [InlineData("", "", 1, "")] + [InlineData("name", "key", 1, "val")] + [InlineData("longlonglonglonglonglonglonglonglonglonglonglonglong name", "longlonglonglonglonglonglonglonglonglonglonglonglong key", 10, "longlonglonglonglonglonglonglonglonglonglonglonglong val")] + public void FilterToMessageTest(string name, string key, int operation, string value) + { + FrostFsFilter filter = new (name, key, operation, value, []); + + var message = filter.ToMessage(); + + Assert.Equal(name, message.Name); + Assert.Equal(key, message.Key); + Assert.Equal(operation, (int)message.Op); + Assert.Equal(value, message.Value); + } + + [Theory] + [InlineData("", "", 1, "")] + [InlineData("name", "key", 2, "val")] + [InlineData("longlonglonglonglonglonglonglonglonglonglonglonglong name", "longlonglonglonglonglonglonglonglonglonglonglonglong key", 10, "longlonglonglonglonglonglonglonglonglonglonglonglong val")] + public void SubFilterToMessageTest(string name, string key, int operation, string value) + { + FrostFsFilter subFilter = new(name, key, operation, value, []); + + FrostFsFilter filter = new("name", "key", 1, "value", [subFilter]); + + var message = filter.ToMessage(); + + Assert.Single(message.Filters); + + var subfilter = message.Filters[0]; + Assert.Equal(name, subfilter.Name); + Assert.Equal(key, subfilter.Key); + Assert.Equal(operation, (int)subfilter.Op); + Assert.Equal(value, subfilter.Value); + } + + [Fact] + public void SubFiltersToMessageTest() + { + string[] names = ["", "name1", "some pretty long name for name test"]; + string[] keys = ["", "key1", "some pretty long key for name test"]; + int[] operations = [1, 2, 10]; + string[] values = ["", "val1", "some pretty long value for name test"]; + + var subFilter = new FrostFsFilter[3]; + + for (int i = 0; i < 3; i++) + { + subFilter[i] = new FrostFsFilter(names[i], keys[i], operations[i], values[i], []); + } + + FrostFsFilter filter = new("name", "key", 1, "value", subFilter); + + var message = filter.ToMessage(); + + Assert.Equal(3, message.Filters.Count); + + for (int i = 0; i < 3; i++) + { + var subfilter = message.Filters[i]; + Assert.Equal(names[i], subfilter.Name); + Assert.Equal(keys[i], subfilter.Key); + Assert.Equal(operations[i], (int)subfilter.Op); + Assert.Equal(values[i], subfilter.Value); + } + } + + [Theory] + [InlineData("", "", Operation.Unspecified, "")] + [InlineData("name", "key", Operation.Unspecified, "val")] + [InlineData("name", "key", Operation.And, "val")] + [InlineData("name", "key", Operation.Eq, "val")] + [InlineData("name", "key", Operation.Le, "val")] + [InlineData("name", "key", Operation.Like, "val")] + [InlineData("name", "key", Operation.Ge, "val")] + [InlineData("name", "key", Operation.Gt, "val")] + [InlineData("name", "key", Operation.Lt, "val")] + [InlineData("name", "key", Operation.Ne, "val")] + [InlineData("name", "key", Operation.Not, "val")] + [InlineData("name", "key", Operation.Or, "val")] + [InlineData("longlonglonglonglonglonglonglonglonglonglonglonglong name", "longlonglonglonglonglonglonglonglonglonglonglonglong key", Operation.Like, "longlonglonglonglonglonglonglonglonglonglonglonglong val")] + public void FrostFsFilterToModelTest(string name, string key, Operation operation, string value) + { + Filter filter = new() + { + Name = name, + Key = key, + Op = operation, + Value = value + }; + + var model = filter.ToModel(); + + Assert.Equal(name, model.Name); + Assert.Equal(key, model.Key); + Assert.Equal((int)operation, model.Operation); + Assert.Equal(value, model.Value); + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/SignatureTests.cs b/src/FrostFS.SDK.Tests/Unit/SignatureTests.cs new file mode 100644 index 0000000..5062319 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/SignatureTests.cs @@ -0,0 +1,39 @@ +using System.Text; +using FrostFS.SDK.Client; +using FrostFS.SDK.Client.Mappers.GRPC; + +namespace FrostFS.SDK.Tests.Unit; + +public class SignatureTests +{ + [Theory] + [InlineData(Refs.SignatureScheme.EcdsaSha512)] + [InlineData(Refs.SignatureScheme.EcdsaRfc6979Sha256)] + [InlineData(Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect)] + + public void SignatureToMessageTest(Refs.SignatureScheme scheme) + { + var key = Encoding.UTF8.GetBytes("datafortest"); + var sign = Encoding.UTF8.GetBytes("signdatafortest"); + + var frostFsScheme = scheme switch + { + Refs.SignatureScheme.EcdsaRfc6979Sha256 => SignatureScheme.EcdsaRfc6979Sha256, + Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect => SignatureScheme.EcdsaRfc6979Sha256WalletConnect, + Refs.SignatureScheme.EcdsaSha512 => SignatureScheme.EcdsaSha512 + }; + + FrostFsSignature signature = new() + { + Key = key, + Scheme = frostFsScheme, + Sign = sign + }; + + var result = signature.ToMessage(); + + Assert.Equal(scheme, result.Scheme); + Assert.Equal(sign, result.Sign.ToByteArray()); + Assert.Equal(key, result.Key.ToByteArray()); + } +} \ No newline at end of file From 0816be732a85d0b2509d842f550fb89d6aa1edf3 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Wed, 30 Apr 2025 15:19:20 +0300 Subject: [PATCH 12/12] [#69] Fix for Patch and uint types Signed-off-by: Pavel Gross --- src/FrostFS.SDK.Client/AssemblyInfo.cs | 4 +- .../FrostFS.SDK.Client.csproj | 2 +- src/FrostFS.SDK.Client/Mappers/MetaHeader.cs | 4 +- .../Mappers/Netmap/Replica.cs | 4 +- src/FrostFS.SDK.Client/Mappers/Version.cs | 8 +- .../Models/Netmap/FrostFsReplica.cs | 8 +- .../Models/Netmap/FrostFsVersion.cs | 6 +- .../Models/Response/MetaHeader.cs | 6 +- .../Services/ObjectServiceProvider.cs | 163 +++++++++++------- src/FrostFS.SDK.Cryptography/AssemblyInfo.cs | 4 +- .../FrostFS.SDK.Cryptography.csproj | 2 +- src/FrostFS.SDK.Protos/AssemblyInfo.cs | 4 +- .../FrostFS.SDK.Protos.csproj | 2 +- .../Client/ContainerTests/ContainerTests.cs | 4 +- .../Smoke/Client/MiscTests/InterceptorTest.cs | 4 +- .../Smoke/Client/NetworkTests/NetworkTests.cs | 8 +- .../Smoke/Client/ObjectTests/ObjectTests.cs | 94 ++++++---- src/FrostFS.SDK.Tests/Unit/ContainerTest.cs | 4 +- .../Unit/PlacementPolicyTests.cs | 30 ++-- .../Unit/PlacementVectorTests.cs | 4 +- 20 files changed, 212 insertions(+), 153 deletions(-) diff --git a/src/FrostFS.SDK.Client/AssemblyInfo.cs b/src/FrostFS.SDK.Client/AssemblyInfo.cs index 4d87f8c..4fa6cd9 100644 --- a/src/FrostFS.SDK.Client/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Client/AssemblyInfo.cs @@ -8,7 +8,7 @@ using System.Runtime.CompilerServices; "e15ab287e6239c98d5dfa91615bd77485d523a3a3f65a4e5028454cedd5ac4d9eca6da18b81985"+ "ac6905d33cc64b5a2587050c16f67b71ef8889dbd3c90ef7cc0b06bbbe09886601d195f5db179a"+ "3c2a25b1")] -[assembly: AssemblyFileVersion("1.0.6.0")] +[assembly: AssemblyFileVersion("1.0.7.0")] [assembly: AssemblyProduct("FrostFS.SDK.Client")] [assembly: AssemblyTitle("FrostFS.SDK.Client")] -[assembly: AssemblyVersion("1.0.6")] +[assembly: AssemblyVersion("1.0.7")] diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index 037a2f4..88e20c6 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -6,7 +6,7 @@ enable AllEnabledByDefault FrostFS.SDK.Client - 1.0.6 + 1.0.7 C# SDK for FrostFS gRPC native protocol diff --git a/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs b/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs index e4ab8a2..b3fa5e3 100644 --- a/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs +++ b/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs @@ -16,8 +16,8 @@ public static class MetaHeaderMapper return new RequestMetaHeader { Version = metaHeader.Version.ToMessage(), - Epoch = (uint)metaHeader.Epoch, - Ttl = (uint)metaHeader.Ttl + Epoch = metaHeader.Epoch, + Ttl = metaHeader.Ttl }; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs index 00ad4a3..4fa2edd 100644 --- a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs +++ b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs @@ -11,7 +11,7 @@ public static class PolicyMapper { return new Replica { - Count = (uint)replica.Count, + Count = replica.Count, Selector = replica.Selector, EcDataCount = replica.EcDataCount, EcParityCount = replica.EcParityCount @@ -25,7 +25,7 @@ public static class PolicyMapper throw new ArgumentNullException(nameof(replica)); } - return new FrostFsReplica((int)replica.Count, replica.Selector) + return new FrostFsReplica(replica.Count, replica.Selector) { EcDataCount = replica.EcDataCount, EcParityCount = replica.EcParityCount diff --git a/src/FrostFS.SDK.Client/Mappers/Version.cs b/src/FrostFS.SDK.Client/Mappers/Version.cs index a66df6e..af8738d 100644 --- a/src/FrostFS.SDK.Client/Mappers/Version.cs +++ b/src/FrostFS.SDK.Client/Mappers/Version.cs @@ -18,7 +18,7 @@ public static class VersionMapper throw new System.ArgumentNullException(nameof(model)); } - var key = model.Major << 16 + model.Minor; + var key = (int)model.Major << 16 + (int)model.Minor; if (!_cacheMessages.ContainsKey(key)) { @@ -28,8 +28,8 @@ public static class VersionMapper _spinlock.Enter(ref lockTaken); var message = new Version { - Major = (uint)model.Major, - Minor = (uint)model.Minor + Major = model.Major, + Minor = model.Minor }; _cacheMessages.Add(key, message); @@ -64,7 +64,7 @@ public static class VersionMapper try { _spinlock.Enter(ref lockTaken); - var model = new FrostFsVersion((int)message.Major, (int)message.Minor); + var model = new FrostFsVersion(message.Major, message.Minor); _cacheModels.Add(key, model); return model; diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs index 5ace048..82e2240 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs @@ -4,12 +4,12 @@ namespace FrostFS.SDK; public struct FrostFsReplica : IEquatable { - public int Count { get; set; } + public uint Count { get; set; } public string Selector { get; set; } public uint EcDataCount { get; set; } public uint EcParityCount { get; set; } - public FrostFsReplica(int count, string? selector = null) + public FrostFsReplica(uint count, string? selector = null) { selector ??= string.Empty; @@ -31,12 +31,12 @@ public struct FrostFsReplica : IEquatable public readonly uint CountNodes() { - return Count != 0 ? (uint)Count : EcDataCount + EcParityCount; + return Count != 0 ? Count : EcDataCount + EcParityCount; } public override readonly int GetHashCode() { - return (Count + Selector.GetHashCode()) ^ (int)EcDataCount ^ (int)EcParityCount; + return Count.GetHashCode() ^ Selector.GetHashCode() ^ (int)EcDataCount ^ (int)EcParityCount; } public static bool operator ==(FrostFsReplica left, FrostFsReplica right) diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs index 81e642d..9ed9522 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs @@ -3,12 +3,12 @@ using FrostFS.SDK.Client.Mappers.GRPC; namespace FrostFS.SDK; -public class FrostFsVersion(int major, int minor) +public class FrostFsVersion(uint major, uint minor) { private Version? version; - public int Major { get; set; } = major; - public int Minor { get; set; } = minor; + public uint Major { get; set; } = major; + public uint Minor { get; set; } = minor; internal Version VersionID { diff --git a/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs b/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs index 36dad09..547b2b5 100644 --- a/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs +++ b/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs @@ -1,10 +1,10 @@ namespace FrostFS.SDK; -public class MetaHeader(FrostFsVersion version, int epoch, int ttl) +public class MetaHeader(FrostFsVersion version, ulong epoch, uint ttl) { public FrostFsVersion Version { get; set; } = version; - public int Epoch { get; set; } = epoch; - public int Ttl { get; set; } = ttl; + public ulong Epoch { get; set; } = epoch; + public uint Ttl { get; set; } = ttl; public static MetaHeader Default() { diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index a8b748f..b3c1f50 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -295,86 +295,88 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl internal async Task PatchObjectAsync(PrmObjectPatch args, CallContext ctx) { var chunkSize = args.MaxChunkLength; - Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null"); - + var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken); - byte[]? chunkBuffer = null; - try + var address = new Address { - chunkBuffer = ArrayPool.Shared.Rent(chunkSize); + ObjectId = args.Address.ObjectId, + ContainerId = args.Address.ContainerId + }; - bool isFirstChunk = true; - ulong currentPos = args.Range.Offset; - - var address = new Address + if (args.Payload != null && args.Payload.Length > 0) + { + byte[]? chunkBuffer = null; + try { - ObjectId = args.Address.ObjectId, - ContainerId = args.Address.ContainerId - }; + chunkBuffer = ArrayPool.Shared.Rent(chunkSize); - while (true) - { - var bytesCount = await payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false); + bool isFirstChunk = true; + ulong currentPos = args.Range.Offset; - if (bytesCount == 0) + while (true) { - break; - } + var bytesCount = await args.Payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false); - var request = new PatchRequest() - { - Body = new() + if (bytesCount == 0) { - Address = address, - Patch = new PatchRequest.Types.Body.Types.Patch + break; + } + + PatchRequest request; + + if (isFirstChunk) + { + request = await CreateFirstRequest(args, ctx, address).ConfigureAwait(false); + + request.Body.Patch = new PatchRequest.Types.Body.Types.Patch { Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount } - } + }; + + isFirstChunk = false; } - }; - - if (isFirstChunk) - { - var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false); - - var protoToken = sessionToken.CreateObjectTokenContext( - address, - ObjectSessionContext.Types.Verb.Patch, - ClientContext.Key); - - request.AddMetaHeader(args.XHeaders, protoToken); - - if (args.NewAttributes != null && args.NewAttributes.Length > 0) + else { - foreach (var attr in args.NewAttributes) + request = new PatchRequest() { - request.Body.NewAttributes.Add(attr.ToMessage()); - request.Body.ReplaceAttributes = args.ReplaceAttributes; - } + Body = new() + { + Address = address, + Patch = new PatchRequest.Types.Body.Types.Patch + { + Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), + SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount } + } + } + }; + + request.AddMetaHeader(args.XHeaders); } - isFirstChunk = false; + request.Sign(ClientContext.Key); + + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); + + currentPos += (ulong)bytesCount; } - else + } + finally + { + if (chunkBuffer != null) { - request.AddMetaHeader(args.XHeaders); + ArrayPool.Shared.Return(chunkBuffer); } - - request.Sign(ClientContext.Key); - - await call.RequestStream.WriteAsync(request).ConfigureAwait(false); - - currentPos += (ulong)bytesCount; } } - finally + else if (args.NewAttributes != null && args.NewAttributes.Length > 0) { - if (chunkBuffer != null) - { - ArrayPool.Shared.Return(chunkBuffer); - } + PatchRequest request = await CreateFirstRequest(args, ctx, address).ConfigureAwait(false); + + request.Sign(ClientContext.Key); + + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); } await call.RequestStream.CompleteAsync().ConfigureAwait(false); @@ -383,9 +385,36 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl Verifier.CheckResponse(response); return response.Body.ObjectId.ToModel(); + + async Task CreateFirstRequest(PrmObjectPatch args, CallContext ctx, Address address) + { + var body = new PatchRequest.Types.Body() { Address = address }; + + if (args.NewAttributes != null) + { + body.ReplaceAttributes = args.ReplaceAttributes; + + foreach (var attr in args.NewAttributes!) + { + body.NewAttributes.Add(attr.ToMessage()); + } + } + + var request = new PatchRequest() { Body = body }; + + var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false); + + var protoToken = sessionToken.CreateObjectTokenContext( + address, + ObjectSessionContext.Types.Verb.Patch, + ClientContext.Key); + + request.AddMetaHeader(args.XHeaders, protoToken); + return request; + } } - internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) + internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { if (args.Payload == null) throw new ArgumentException(nameof(args.Payload)); @@ -431,9 +460,9 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); - + var remain = fullLength; - + byte[]? buffer = null; bool isRentBuffer = false; @@ -443,7 +472,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { if (args.CustomBuffer.Length < chunkSize) throw new ArgumentException($"Buffer size is too small. At least {chunkSize} required"); - + buffer = args.CustomBuffer; } else @@ -457,7 +486,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl while (remain > 0) { - var bytesToWrite = Math.Min((ulong)partSize, remain); + var bytesToWrite = Math.Min((ulong)partSize, remain); var isLastPart = remain <= (ulong)partSize; // When the last part of the object is uploaded, all metadata for the object must be added @@ -502,7 +531,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var part = new ObjectPartInfo(offset, uploaded, objectId); offset += uploaded; progressInfo.AddPart(part); - + remain -= bytesToWrite; if (isLastPart) @@ -582,16 +611,16 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0; var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; + progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); + // if the object fits one part, it can be loaded as non-complex object, but if it is not upload resuming - if (objectsCount == 1 && progressInfo != null && progressInfo.GetLast().Length == 0) + if (objectsCount == 1 && progressInfo.GetLast().Length == 0) { args.PutObjectContext.MaxObjectSizeCache = partSize; args.PutObjectContext.FullLength = fullLength; var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false); return singlePartResult.ObjectId; } - - progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); var remain = fullLength; @@ -659,8 +688,10 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl offset += size; if (i < objectsCount) + { continue; - + } + // Once all parts of the object are uploaded, they must be linked into a single entity var linkObject = new FrostFsLinkObject(args.Header.ContainerId, progressInfo.SplitId, parentHeader!, [.. progressInfo.GetParts().Select(p => p.ObjectId)]); diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index cff5cc1..64bc8e9 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,7 +1,7 @@ using System.Reflection; [assembly: AssemblyCompany("TrueCloudLab")] -[assembly: AssemblyFileVersion("1.0.6.0")] +[assembly: AssemblyFileVersion("1.0.7.0")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] -[assembly: AssemblyVersion("1.0.6.0")] +[assembly: AssemblyVersion("1.0.7.0")] diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index 20e0a83..faeabb1 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Cryptography - 1.0.6 + 1.0.7 Cryptography tools for C# SDK diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index f16be42..d7fab05 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,7 +1,7 @@ using System.Reflection; [assembly: AssemblyCompany("TrueCloudLab")] -[assembly: AssemblyFileVersion("1.0.6.0")] +[assembly: AssemblyFileVersion("1.0.7.0")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] -[assembly: AssemblyVersion("1.0.6.0")] +[assembly: AssemblyVersion("1.0.7.0")] diff --git a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj index b6bc876..222acd1 100644 --- a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj +++ b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Protos - 1.0.6 + 1.0.7 Protobuf client for C# SDK diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs index 04a8b2c..f1032db 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs @@ -77,7 +77,7 @@ public class ContainerTests : SmokeTestsBase Assert.Empty(container.PlacementPolicy.Value.Filters); Assert.Single(container.PlacementPolicy.Value.Replicas); - Assert.Equal(3, container.PlacementPolicy.Value.Replicas[0].Count); + Assert.Equal(3u, container.PlacementPolicy.Value.Replicas[0].Count); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount); Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector); @@ -191,7 +191,7 @@ public class ContainerTests : SmokeTestsBase Assert.Empty(subFilter.Filters); Assert.Single(container.PlacementPolicy.Value.Replicas); - Assert.Equal(1, container.PlacementPolicy.Value.Replicas[0].Count); + Assert.Equal(1u, container.PlacementPolicy.Value.Replicas[0].Count); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount); Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs b/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs index 664d522..9d2fd63 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs @@ -29,8 +29,8 @@ public class InterceptorTests() : SmokeTestsBase Assert.True(callbackInvoked); Assert.True(interceptorInvoked); - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); + Assert.Equal(2u, result.Version.Major); + Assert.Equal(13u, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); Assert.NotNull(result.Addresses); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs index a3d25af..229531a 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs @@ -28,8 +28,8 @@ public class SmokeClientTests : SmokeTestsBase var result = await client.GetNodeInfoAsync(default); - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); + Assert.Equal(2u, result.Version.Major); + Assert.Equal(13u, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); Assert.Single(result.Addresses); @@ -103,8 +103,8 @@ public class SmokeClientTests : SmokeTestsBase Assert.True(callbackInvoked); Assert.True(interceptorInvoked); - Assert.Equal(2, result.Version.Major); - Assert.Equal(13, result.Version.Minor); + Assert.Equal(2u, result.Version.Major); + Assert.Equal(13u, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); Assert.NotNull(result.Addresses); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs index a0f501f..473b076 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs @@ -26,7 +26,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase [InlineData(false, 2, 3)] [InlineData(true, 2, 1)] [InlineData(false, 2, 1)] - public async void FullScenario(bool unique, uint backupFactor, int replicas) + public async void FullScenario(bool unique, uint backupFactor, uint replicas) { var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); _testOutputHelper.WriteLine("client created"); @@ -121,9 +121,16 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase await ValidateFilters(client, containerId, objectId, null, (ulong)bytes.Length); _testOutputHelper.WriteLine($"\tfilters validated"); - if (type != clientCut) + if (type != clientCut && bytes.Length > 1024 + 64 && bytes.Length < 20 * 1024 * 1024) { - await ValidatePatch(client, containerId, bytes, objectId); + // patch payload only + await ValidatePatch(client, containerId, bytes, true, objectId, [], false); + + // patch attributes only + await ValidatePatch(client, containerId, bytes, false, objectId, [new("a1", "v1"), new("a2", "v2")], false); + + // patch payload and attributes + await ValidatePatch(client, containerId, bytes, true, objectId, [new("a3", "v3"), new("a4", "v4")], true); _testOutputHelper.WriteLine($"\tpatch validated"); } @@ -199,51 +206,73 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase _testOutputHelper.WriteLine($"\t\trange {range.Offset};{range.Length} validated"); } - private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId) + private static async Task ValidatePatch( + IFrostFSClient client, + FrostFsContainerId containerId, + byte[] bytes, + bool patchPayload, + FrostFsObjectId objectId, + FrostFsAttributePair[] attributes, + bool replaceAttributes) { - if (bytes.Length < 1024 + 64 || bytes.Length > 5900) - return; - - var patch = new byte[1024]; - for (int i = 0; i < patch.Length; i++) + byte[]? patch = null; + FrostFsRange range = new(); + if (patchPayload) { - patch[i] = 32; - } + patch = new byte[1024]; + for (int i = 0; i < patch.Length; i++) + { + patch[i] = 32; + } - var range = new FrostFsRange(64, (ulong)patch.Length); + range = new FrostFsRange(64, (ulong)patch.Length); + } var patchParams = new PrmObjectPatch( new FrostFsAddress(containerId, objectId), - payload: new MemoryStream(patch), + payload: new MemoryStream(patch ?? []), maxChunkLength: 1024, - range: range); + range: range, + replaceAttributes: replaceAttributes, + newAttributes: attributes); var newIbjId = await client.PatchObjectAsync(patchParams, default); var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, newIbjId), default); - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + if (patchPayload) { - ms.Write(chunk.Value.Span); + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + for (int i = 0; i < (int)range.Offset; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + var rangeEnd = range.Offset + range.Length; + + for (int i = (int)range.Offset; i < (int)rangeEnd; i++) + Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); + + for (int i = (int)rangeEnd; i < bytes.Length; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); } - for (int i = 0; i < (int)range.Offset; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); - - var rangeEnd = range.Offset + range.Length; - - for (int i = (int)range.Offset; i < (int)rangeEnd; i++) - Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); - - for (int i = (int)rangeEnd; i < bytes.Length; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); + if (attributes != null && attributes.Length > 0) + { + foreach (var newAttr in attributes) + { + var i = @object!.Header!.Attributes!.Count(p => p.Key == newAttr.Key && p.Value == newAttr.Value); + Assert.Equal(1, i); + } + } } - private async Task ValidateFilters(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId, SplitId? splitId, ulong length) { var ecdsaKey = keyString.LoadWif(); @@ -318,7 +347,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.NotNull(objHeader); Assert.Equal(containerId.GetValue(), objHeader.ContainerId.GetValue()); - + Assert.Equal(expected.HeaderInfo!.OwnerId!.Value, objHeader.OwnerId!.Value); Assert.Equal(expected.HeaderInfo.Version!.Major, objHeader.Version!.Major); Assert.Equal(expected.HeaderInfo.Version!.Minor, objHeader.Version!.Minor); @@ -338,7 +367,6 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.Null(objHeader.Split); } - private static async Task CreateObjectServerCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes) { var header = new FrostFsObjectHeader( diff --git a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs index e713b4a..ce1d06c 100644 --- a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs @@ -14,7 +14,7 @@ public class ContainerTest : ContainerTestsBase [Theory] [InlineData(1, "test", 0, 0)] - public void ReplicaToMessagelTest(int count, string selector, uint ecDataCount, uint ecParityCount) + public void ReplicaToMessagelTest(uint count, string selector, uint ecDataCount, uint ecParityCount) { FrostFsReplica replica = new() { @@ -26,7 +26,7 @@ public class ContainerTest : ContainerTestsBase Replica message = replica.ToMessage(); - Assert.Equal((uint)count, message.Count); + Assert.Equal(count, message.Count); Assert.Equal(selector, message.Selector); Assert.Equal(ecDataCount, message.EcDataCount); Assert.Equal(ecParityCount, message.EcParityCount); diff --git a/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs b/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs index 9957c63..46f7a3d 100644 --- a/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs @@ -69,13 +69,13 @@ public class PlacementPolicyTests : NetworkTestsBase var rep0 = result.Replicas[0]; Assert.Equal(2u, rep0.EcDataCount); Assert.Equal(3u, rep0.EcParityCount); - Assert.Equal(4, rep0.Count); + Assert.Equal(4u, rep0.Count); Assert.Equal("selector1", rep0.Selector); var rep1 = result.Replicas[1]; Assert.Equal(5u, rep1.EcDataCount); Assert.Equal(6u, rep1.EcParityCount); - Assert.Equal(7, rep1.Count); + Assert.Equal(7u, rep1.Count); Assert.Equal("selector2", rep1.Selector); var f0 = result.Filters[0]; @@ -126,7 +126,7 @@ public class PlacementPolicyTests : NetworkTestsBase FrostFsReplica model = replica.ToModel(); - Assert.Equal(count, (uint)model.Count); + Assert.Equal(count, model.Count); Assert.Equal(selector, model.Selector); Assert.Equal(ecDataCount, model.EcDataCount); Assert.Equal(ecParityCount, model.EcParityCount); @@ -140,7 +140,7 @@ public class PlacementPolicyTests : NetworkTestsBase [InlineData(1, " ", 2, 3)] [InlineData(10, "!", 0, 0)] [InlineData(1, "123", 0, 0)] - public void ReplicaToMessagelTest(int count, string selector, uint ecDataCount, uint ecParityCount) + public void ReplicaToMessagelTest(uint count, string selector, uint ecDataCount, uint ecParityCount) { FrostFsReplica replica = new () { @@ -152,7 +152,7 @@ public class PlacementPolicyTests : NetworkTestsBase Replica message = replica.ToMessage(); - Assert.Equal((uint)count, message.Count); + Assert.Equal(count, message.Count); Assert.Equal(selector, message.Selector); Assert.Equal(ecDataCount, message.EcDataCount); Assert.Equal(ecParityCount, message.EcParityCount); @@ -235,11 +235,11 @@ public class PlacementPolicyTests : NetworkTestsBase Assert.Single(message.Filters); - var subfilter = message.Filters[0]; - Assert.Equal(name, subfilter.Name); - Assert.Equal(key, subfilter.Key); - Assert.Equal(operation, (int)subfilter.Op); - Assert.Equal(value, subfilter.Value); + var grpcFilter = message.Filters[0]; + Assert.Equal(name, grpcFilter.Name); + Assert.Equal(key, grpcFilter.Key); + Assert.Equal(operation, (int)grpcFilter.Op); + Assert.Equal(value, grpcFilter.Value); } [Fact] @@ -265,11 +265,11 @@ public class PlacementPolicyTests : NetworkTestsBase for (int i = 0; i < 3; i++) { - var subfilter = message.Filters[i]; - Assert.Equal(names[i], subfilter.Name); - Assert.Equal(keys[i], subfilter.Key); - Assert.Equal(operations[i], (int)subfilter.Op); - Assert.Equal(values[i], subfilter.Value); + var grpcFilter = message.Filters[i]; + Assert.Equal(names[i], grpcFilter.Name); + Assert.Equal(keys[i], grpcFilter.Key); + Assert.Equal(operations[i], (int)grpcFilter.Op); + Assert.Equal(values[i], grpcFilter.Value); } } diff --git a/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs b/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs index fdf7ffe..9833ae1 100644 --- a/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs @@ -248,12 +248,12 @@ public class FilterDto Key ?? string.Empty, (int)Op, Value ?? string.Empty, - Filters != null ? Filters.Select(f => f.Filter).ToArray() : []); + Filters != null ? [.. Filters.Select(f => f.Filter)] : []); } public class ReplicaDto { - public int Count { get; set; } + public uint Count { get; set; } public string? Selector { get; set; } }