diff --git a/.forgejo/workflows/publish.yml b/.forgejo/workflows/publish.yml index fd79900..ca66daa 100644 --- a/.forgejo/workflows/publish.yml +++ b/.forgejo/workflows/publish.yml @@ -1,4 +1,5 @@ on: + push: workflow_dispatch: jobs: @@ -15,7 +16,7 @@ jobs: # `dotnet build` implies and replaces `dotnet pack` thanks to `GeneratePackageOnBuild` run: dotnet build - - name: Publish unsigned NuGet packages + - name: Publish NuGet packages run: |- dotnet nuget add source \ --name "$NUGET_REGISTRY" \ diff --git a/.gitignore b/.gitignore index 449284e..ae0b81f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,8 +32,5 @@ antlr-*.jar # binary bin/ +release/ obj/ - -# Repository signing keys -release/maintainer.* -release/ca.* diff --git a/Makefile b/Makefile deleted file mode 100644 index f02478e..0000000 --- a/Makefile +++ /dev/null @@ -1,58 +0,0 @@ -DOTNET?=dotnet -DOCKER?=docker - -NUGET_REGISTRY?=TrueCloudLab -NUGET_REGISTRY_URL?=https://git.frostfs.info/api/packages/TrueCloudLab/nuget/index.json -NUGET_REGISTRY_USER?= -NUGET_REGISTRY_PASSWORD?= - -NUPKG=find -iname '*.nupkg' | grep . | xargs -d'\n' -t -r -n1 -RFC3161_TSA?=http://timestamp.digicert.com - - -.PHONY: build -build: - $(DOTNET) build - - -.PHONY: sign -sign: export NUGET_CERT_REVOCATION_MODE=offline -sign: release/maintainer.pfx - $(NUPKG) $(DOTNET) nuget sign --overwrite --certificate-path $< --timestamper "$(RFC3161_TSA)" - @rm -v "$<" # maintainer.pfx is not password protected and must be ephemeral - $(NUPKG) $(DOTNET) nuget verify - - -.PHONY: publish -publish: - $(NUPKG) $(DOTNET) nuget verify - $(NUPKG) $(DOTNET) nuget push --source "$(NUGET_REGISTRY)" - - -.PHONY: nuget-registry -nuget-registry: -ifeq (,$(NUGET_REGISTRY_USER)) - $(error NUGET_REGISTRY_USER not set) -endif -ifeq (,$(NUGET_REGISTRY_PASSWORD)) - $(error NUGET_REGISTRY_PASSWORD not set) -endif - $(DOTNET) nuget add source \ - --name "$(NUGET_REGISTRY)" \ - --username "$(NUGET_REGISTRY_USER)" \ - --password "$(NUGET_REGISTRY_PASSWORD)" \ - --store-password-in-clear-text \ - "$(NUGET_REGISTRY_URL)" - - -.PHONY: clean -clean: - -$(NUPKG) rm -v - - -.PHONY: container -container: - $(DOCKER) run --pull=always --rm -it -v "$$PWD:/src" -w /src git.frostfs.info/truecloudlab/env:dotnet-8.0 - - -include release/codesign.mk diff --git a/keyfile.snk b/keyfile.snk deleted file mode 100644 index a14537b..0000000 Binary files a/keyfile.snk and /dev/null differ diff --git a/release/README.md b/release/README.md deleted file mode 100644 index 96f8c25..0000000 --- a/release/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Release process - -## Preparing release - -_TBD_ - -## Trusting TrueCloudLab code signing CA certificate - -Verifying signatures (and signing) TrueCloudLab packages requires adding -[TrueCloudLab Code Signing CA](ca.cert) to the list of trusted roots. - -On Linux this can be done by appending [release/ca.cert](ca.cert) to one of: - -- `/etc/pki/ca-trust/extracted/pem/objsign-ca-bundle.pem`: compatible with - [update-ca-trust] and originally proposed in [.NET design docs] -- `…/dotnet/sdk/X.Y.ZZZ/trustedroots/codesignctl.pem`: [fallback] codesigning certificate trust list for .NET - -[update-ca-trust]: https://www.linux.org/docs/man8/update-ca-trust.html -[.NET design docs]: https://github.com/dotnet/designs/blob/main/accepted/2021/signed-package-verification/re-enable-signed-package-verification-technical.md#linux -[fallback]: https://github.com/dotnet/sdk/blob/11150c0ec9020625308edeec555a8b78dbfb2aa5/src/Layout/redist/trustedroots/README.md - -## Signing Nuget packages - -Repository maintainer places `maintainer.cert` and `maintainer.key` (see below -regarding obtaining these files) into `release/` directory and then -executes: - -```console -$ make build sign -``` - -## Uploading packages to Nuget registry - -**IMPORTANT: the following steps upload all `*.nupkg` files located under -`src/`. Maintainer MUST make sure that no unnecessary package versions will be -uploaded to the registry.** - -Configure registry credentials (once per machine): - -```console -$ make nuget-registry NUGET_REGISTRY_USER=username NUGET_REGISTRY_PASSWORD=token -``` - -Publish all locally built packages (implicitly clear existing `*.nupkg` and -rebuild current version only): - -```console -$ make clean build sign publish -``` - - -## Obtaining release signing certificate - -Repository maintainer owns and keeps safe the release signing key -(`maintainer.key`). Private key should never leave maintainer's machine and -should be considered a highly sensitive secret. - -- Generating new maintainer key and the corresponding CSR: - - ```console - $ make maintainer.csr - ...lines skipped... - Enter PEM pass phrase: - Verifying - Enter PEM pass phrase: - ----- - IMPORTANT: Keep maintainer.key private! - - Certificate signing request is ready. - Send maintainer.csr to CA administrator to obtain the certificate. - ``` - - Resulting CSR (`maintainer.csr`) does not contain any sensitive - cryptographic material and may be passed to CA administrator through regular - communication channels. - -- CA administrator then issues the certificate (`make maintainer.cert`) and - sends it back to the maintainer to be used in combination with - `maintainer.key` - -This procedure should be repeated once per machine per `maintainer.cert` -lifetime (1 year) - typically just once per year since we expect the -maintainer to use only a single computer to sign releases. diff --git a/release/ca.cert b/release/ca.cert deleted file mode 100644 index 4f4c2f4..0000000 --- a/release/ca.cert +++ /dev/null @@ -1,34 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIF2zCCA8OgAwIBAgIUa9xC/RgvFtUG/xeR016nn0B4K0YwDQYJKoZIhvcNAQEL -BQAwdTELMAkGA1UEBhMCUlUxFTATBgNVBAoMDFRydWVDbG91ZExhYjEVMBMGA1UE -CwwMVHJ1ZUNsb3VkTGFiMTgwNgYDVQQDDC9UcnVlQ2xvdWRMYWIgQ29kZSBTaWdu -aW5nIENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yNTA0MTAxNTI2MTFaFw0zNTA0 -MDgxNTI2MTFaMHUxCzAJBgNVBAYTAlJVMRUwEwYDVQQKDAxUcnVlQ2xvdWRMYWIx -FTATBgNVBAsMDFRydWVDbG91ZExhYjE4MDYGA1UEAwwvVHJ1ZUNsb3VkTGFiIENv -ZGUgU2lnbmluZyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQCyANB4cjf+ZEAFx9RiUYXCAOPMV+jyqgcVbhzh2YKc -9SlvGKRlc00Ar1RlFcrycdkIrTKmhhobiWsFp7UgphwLqRTCb5NB6qfUoWhnfiD9 -m0OBgeVX5wivVaibRI9PSTbFDcIhYUiNvwFJ6GduH/9zOxf1BvuL7LMaoyhIDcg/ -XVLuekE2lnX83zsedv0v/2jyyMY9Ct6N2BXzyHSAzSdYYg0F9Qu9fIMAPjoKhWPc -PnotqaACjb1DScLUr3E/o2W1FfprTT2Pip/0AXxO4wixl4QWh9HeOKV22KRcCHo6 -3wNdg5q1ZVGTNBW0+yoB4jsSG8/JM+2Ujhc1ZnYH10armvGq/0Oc2YQE00960Wy8 -t0drCFWJUO1XHNeBxkkupmj7N1TAPbixtfiGZJhECOWOJwyMpcKixlt5P0cNH4N/ -p3vjyrGQxGLBIkgV/QgjfGkpTHKT1/H40YK6DliWJc01KfNTqn0K+/TIyF0n26kD -BWYVlvDh5P1+V9DGuD2zeXB3PstoifD/Pd7D8wuqpm17noFE19MLp94xv03q9nEa -jRMEd2J2buTLvMh5BBVH0Sm38QAHpSIZ9O3dSLvvjlALbVtwmcsNE9fgxiue3vTB -iXNW8wWs+/DMYwbWyBoCwORxVOdOyc1JLn7qAAEUBweilPVXpWuzMLdUsifPiqrV -dQIDAQABo2MwYTAdBgNVHQ4EFgQUEz4y/RvQMmbUFvf5JbGe/0PZR90wHwYDVR0j -BBgwFoAUEz4y/RvQMmbUFvf5JbGe/0PZR90wDwYDVR0TAQH/BAUwAwEB/zAOBgNV -HQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAF79W9hMGnrKUWLDOcoeXDEn -+gZxd5rjeF0tNEQGbWKGTJAGQdprOkzWQ47PewpFeS+ePpjOglBAt7cV2ZnOugAT -Brx31vYpNAvYnUCYn+/IUqD8S/U4uErVS9zG9QjirZWo56eJP62vnScKuApCQCbA -pp0zrIyJ+2lQKzlMOENRqzYYA/UTOqYTtnW6x2D8goVqCmDXpgpxEp5XliFjJSr6 -dOjiopNWMvaV3R/Bnd4i41taM7M6HpIV+gbXmEKEFS0ejVfzT8z1pTigN7GBqbxf -nXD03eLUIsbMDv4ZQPrheN7nKnjRUn8kxz0SSK1m2YDrXW51m8fOs6aTvwC/cNe+ -FJMqQMF32i4IXVfUbyUJi+JMvawqm2wEY46vrh7siprY6rXsAzCKJo07i6jvUwnT -TXMldSCPgTEqzT2JBzzr0tRfuPKsv0/NqflHvwfuMRCpcZ7jJZ700iN92xXkiQHP -DmCZOILXcNclAth3nAnyY4XE5a8myv8bwYaPdJdIFlV+BoU/8mClDeA8ck4rDy12 -T5YChKew2oiL4j4B6v9/yrDjD1IT0gv4BWyPhb/n390BCEXt8g9auNcT0s6O8kEc -VUDVc1519ocMCuWVuqUK9o2w0zu50/pBn4hVLfT3QyW8sqtlRKghOWtqZzigvCWF -VjATeO5F/Z7OSDebHUGv ------END CERTIFICATE----- diff --git a/release/codesign.mk b/release/codesign.mk deleted file mode 100644 index 1c39361..0000000 --- a/release/codesign.mk +++ /dev/null @@ -1,74 +0,0 @@ -PKI_ROLE?=maintainer -PKI_DIR?=release - -# Note: Only RSA signatures are supported (NU3013) -# https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3013) - - -ifeq ($(PKI_ROLE),maintainer) -.PHONY: maintainer.csr -maintainer.csr: $(PKI_DIR)/maintainer.csr -$(PKI_DIR)/maintainer.csr: KEY=$(patsubst %.csr,%.key,$@) -$(PKI_DIR)/maintainer.csr: - openssl req \ - -new \ - -newkey rsa:4096 \ - -keyout $(KEY) \ - -out $@ \ - -sha256 \ - -addext keyUsage=critical,digitalSignature \ - -addext extendedKeyUsage=critical,codeSigning,msCodeCom \ - -subj "/C=RU/O=TrueCloudLab/OU=TrueCloudLab/CN=frostfs-sdk-csharp Release Team" - @echo "IMPORTANT: Keep $(KEY) private!\n" - @echo "Certificate signing request is ready.\nSend $@ to CA administrator to obtain the certificate." - -$(PKI_DIR)/maintainer.pfx: $(PKI_DIR)/maintainer.cert $(PKI_DIR)/maintainer.key $(PKI_DIR)/ca.cert - openssl verify \ - -CAfile $(PKI_DIR)/ca.cert \ - $(PKI_DIR)/maintainer.cert - openssl pkcs12 \ - -export \ - -out $@ \ - -inkey $(PKI_DIR)/maintainer.key \ - -in $(PKI_DIR)/maintainer.cert \ - -CAfile $(PKI_DIR)/ca.cert \ - -chain \ - -passout pass: -endif - - -ifeq ($(PKI_ROLE),ca) -.PHONY: maintainer.cert -maintainer.cert: $(PKI_DIR)/maintainer.cert -$(PKI_DIR)/maintainer.cert: CSR=$(patsubst %.cert,%.csr,$@) -$(PKI_DIR)/maintainer.cert: $(PKI_DIR)/ca.key $(PKI_DIR)/ca.cert - openssl req -noout -text -in $(CSR) - @read -p "Review the CSR above. Press Enter to continue, Ctrl+C to cancel " -r null - openssl x509 \ - -req \ - -days 365 \ - -in $(CSR) \ - -copy_extensions copy \ - -ext keyUsage,extendedKeyUsage \ - -CA $(PKI_DIR)/ca.cert \ - -CAkey $(PKI_DIR)/ca.key \ - -CAcreateserial \ - -out $@ - echo >> $@ - cat $(PKI_DIR)/ca.cert >> $@ - openssl x509 -noout -text -in $@ -fingerprint -sha256 - @echo "Certificate is ready.\nSend $@ back to maintainer." - -$(PKI_DIR)/ca.key: CERT=$(patsubst %.key,%.cert,$@) -$(PKI_DIR)/ca.key: - openssl req \ - -x509 \ - -newkey rsa:4096 \ - -keyout $@ \ - -out $(CERT) \ - -sha256 \ - -days 3650 \ - -addext keyUsage=critical,keyCertSign \ - -subj "/C=RU/O=TrueCloudLab/OU=TrueCloudLab/CN=TrueCloudLab Code Signing Certificate Authority" - @echo "IMPORTANT: Keep $@ private!\n" -endif diff --git a/src/FrostFS.SDK.Client/ApeRules/Actions.cs b/src/FrostFS.SDK.Client/ApeRules/Actions.cs index 8bae303..71888b9 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Actions.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Actions.cs @@ -8,7 +8,7 @@ public struct Actions(bool inverted, string[] names) : System.IEquatable public override bool Equals(object obj) { - if (obj == null || obj is not Condition) + if (obj == null || obj is not Condition) return false; return Equals((Condition)obj); diff --git a/src/FrostFS.SDK.Client/ApeRules/Resources.cs b/src/FrostFS.SDK.Client/ApeRules/Resources.cs index 47ff3e1..ef06b4b 100644 --- a/src/FrostFS.SDK.Client/ApeRules/Resources.cs +++ b/src/FrostFS.SDK.Client/ApeRules/Resources.cs @@ -8,7 +8,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable _ownersCache; + + internal static IMemoryCache Containers => _containersCache; } diff --git a/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs b/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs index 87799e2..0e64db5 100644 --- a/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs +++ b/src/FrostFS.SDK.Client/Exceptions/FrostFsResponseException.cs @@ -10,8 +10,8 @@ public class FrostFsResponseException : FrostFsException { } - public FrostFsResponseException(FrostFsResponseStatus status) - : base(status != null ? status.Message != null ? "" : "" : "") + public FrostFsResponseException(FrostFsResponseStatus status) + : base(status != null ? status.Message != null ? "" : "" : "") { Status = status; } diff --git a/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs b/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs index 87728d5..a986a87 100644 --- a/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs +++ b/src/FrostFS.SDK.Client/Extensions/FrostFsExtensions.cs @@ -1,17 +1,14 @@ using System; -using System.Security.Cryptography; + using Google.Protobuf; namespace FrostFS.SDK.Cryptography; public static class FrostFsExtensions { - public static byte[] Sha256(this IMessage data) + public static ByteString Sha256(this IMessage data) { - using var sha256 = SHA256.Create(); - using HashStream stream = new(sha256); - data.WriteTo(stream); - return stream.Hash(); + return ByteString.CopyFrom(data.ToByteArray().Sha256()); } public static Guid ToUuid(this ByteString id) @@ -19,18 +16,9 @@ public static class FrostFsExtensions if (id == null) throw new ArgumentNullException(nameof(id)); - return new Guid( - (id[0] << 24) + (id[1] << 16) + (id[2] << 8) + id[3], - (short)((id[4] << 8) + id[5]), - (short)((id[6] << 8) + id[7]), - id[8], - id[9], - id[10], - id[11], - id[12], - id[13], - id[14], - id[15]); + var orderedBytes = GetGuidBytesDirectOrder(id.Span); + + return new Guid(orderedBytes); } /// @@ -38,25 +26,37 @@ public static class FrostFsExtensions /// /// /// - public unsafe static void ToBytes(this Guid id, Span span) + public static byte[] ToBytes(this Guid id) { - var pGuid = (byte*)&id; + var bytes = id.ToByteArray(); - span[0] = pGuid[3]; - span[1] = pGuid[2]; - span[2] = pGuid[1]; - span[3] = pGuid[0]; - span[4] = pGuid[5]; - span[5] = pGuid[4]; - span[6] = pGuid[7]; - span[7] = pGuid[6]; - span[8] = pGuid[8]; - span[9] = pGuid[9]; - span[10] = pGuid[10]; - span[11] = pGuid[11]; - span[12] = pGuid[12]; - span[13] = pGuid[13]; - span[14] = pGuid[14]; - span[15] = pGuid[15]; + var orderedBytes = GetGuidBytesDirectOrder(bytes); + + return orderedBytes; + } + + private static byte[] GetGuidBytesDirectOrder(ReadOnlySpan source) + { + if (source.Length != 16) + throw new ArgumentException("Wrong uuid binary format"); + + return [ + source[3], + source[2], + source[1], + source[0], + source[5], + source[4], + source[7], + source[6], + source[8], + source[9], + source[10], + source[11], + source[12], + source[13], + source[14], + source[15] + ]; } } diff --git a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj index 88e20c6..2205b7e 100644 --- a/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj +++ b/src/FrostFS.SDK.Client/FrostFS.SDK.Client.csproj @@ -6,7 +6,7 @@ enable AllEnabledByDefault FrostFS.SDK.Client - 1.0.7 + 1.0.3 C# SDK for FrostFS gRPC native protocol @@ -31,12 +31,10 @@ false - True - True - .\\..\\..\\keyfile.snk + all @@ -47,7 +45,6 @@ - diff --git a/src/FrostFS.SDK.Client/FrostFSClient.cs b/src/FrostFS.SDK.Client/FrostFSClient.cs index bce9104..f72265c 100644 --- a/src/FrostFS.SDK.Client/FrostFSClient.cs +++ b/src/FrostFS.SDK.Client/FrostFSClient.cs @@ -162,7 +162,25 @@ public class FrostFSClient : IFrostFSClient Callback = settings.Value.Callback, Interceptors = settings.Value.Interceptors }; - } + + // TODO: define timeout logic + // CheckFrostFsVersionSupport(new Context { Timeout = TimeSpan.FromSeconds(20) }); + } + + internal FrostFSClient(WrapperPrm prm, SessionCache cache) + { + ClientCtx = new ClientContext( + client: this, + key: new ClientKey(prm.Key), + owner: FrostFsOwner.FromKey(prm.Key!), + channel: prm.GrpcChannelFactory(prm.Address), + version: new FrostFsVersion(2, 13)) + { + SessionCache = cache, + Interceptors = prm.Interceptors, + Callback = prm.Callback + }; + } #region ApeManagerImplementation public Task> AddChainAsync(PrmApeChainAdd args, CallContext ctx) @@ -254,8 +272,7 @@ public class FrostFSClient : IFrostFSClient public Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { - return GetObjectService().PutClientCutSingleObjectAsync(args, ctx); - // return GetObjectService().PutClientCutObjectAsync(args, ctx); + return GetObjectService().PutClientCutObjectAsync(args, ctx); } public Task PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx) @@ -430,5 +447,18 @@ public class FrostFSClient : IFrostFSClient } return ObjectServiceProvider; - } + } + + public async Task Dial(CallContext ctx) + { + var service = GetAccouningService(); + _ = await service.GetBallance(ctx).ConfigureAwait(false); + + return null; + } + + public bool RestartIfUnhealthy(CallContext ctx) + { + throw new NotImplementedException(); + } } diff --git a/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs b/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs index 43dc3ac..d546156 100644 --- a/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs +++ b/src/FrostFS.SDK.Client/Interfaces/IFrostFSClient.cs @@ -64,4 +64,6 @@ public interface IFrostFSClient #region Account Task GetBalanceAsync(CallContext ctx); #endregion + + public Task Dial(CallContext ctx); } diff --git a/src/FrostFS.SDK.Client/Mappers/ContainerId.cs b/src/FrostFS.SDK.Client/Mappers/ContainerId.cs index 55b6179..df27320 100644 --- a/src/FrostFS.SDK.Client/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.Client/Mappers/ContainerId.cs @@ -5,10 +5,16 @@ using FrostFS.SDK.Cryptography; using Google.Protobuf; +using Microsoft.Extensions.Caching.Memory; + namespace FrostFS.SDK.Client.Mappers.GRPC; public static class ContainerIdMapper { + private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromHours(1)) + .SetSize(1); + public static ContainerID ToMessage(this FrostFsContainerId model) { if (model is null) @@ -18,11 +24,15 @@ public static class ContainerIdMapper var containerId = model.GetValue() ?? throw new ArgumentNullException(nameof(model)); - var message = new ContainerID + if (!Caches.Containers.TryGetValue(containerId, out ContainerID? message)) { - Value = UnsafeByteOperations.UnsafeWrap(Base58.Decode(containerId)) - }; + message = new ContainerID + { + Value = ByteString.CopyFrom(Base58.Decode(containerId)) + }; + Caches.Containers.Set(containerId, message, _oneHourExpiration); + } return message!; } diff --git a/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs b/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs index b3fa5e3..e4ab8a2 100644 --- a/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs +++ b/src/FrostFS.SDK.Client/Mappers/MetaHeader.cs @@ -16,8 +16,8 @@ public static class MetaHeaderMapper return new RequestMetaHeader { Version = metaHeader.Version.ToMessage(), - Epoch = metaHeader.Epoch, - Ttl = metaHeader.Ttl + Epoch = (uint)metaHeader.Epoch, + Ttl = (uint)metaHeader.Ttl }; } } \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs index 4fa2edd..cd7a0b7 100644 --- a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs +++ b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs @@ -11,10 +11,8 @@ public static class PolicyMapper { return new Replica { - Count = replica.Count, - Selector = replica.Selector, - EcDataCount = replica.EcDataCount, - EcParityCount = replica.EcParityCount + Count = (uint)replica.Count, + Selector = replica.Selector }; } @@ -25,11 +23,7 @@ public static class PolicyMapper throw new ArgumentNullException(nameof(replica)); } - return new FrostFsReplica(replica.Count, replica.Selector) - { - EcDataCount = replica.EcDataCount, - EcParityCount = replica.EcParityCount - }; + return new FrostFsReplica((int)replica.Count, replica.Selector); } public static Selector ToMessage(this FrostFsSelector selector) diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs index fde13ff..2b4bd43 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs @@ -41,14 +41,9 @@ public static class ObjectHeaderMapper return model; } - + public static FrostFsSplit ToModel(this Header.Types.Split split) { - if (split is null) - { - throw new ArgumentNullException(nameof(split)); - } - var children = split!.Children.Count != 0 ? new ReadOnlyCollection([.. split.Children.Select(x => x.ToModel())]) : null; diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs index 5562be5..343e3de 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectId.cs @@ -17,7 +17,7 @@ public static class ObjectIdMapper return new ObjectID { - Value = UnsafeByteOperations.UnsafeWrap(objectId.ToHash()) + Value = ByteString.CopyFrom(objectId.ToHash()) }; } diff --git a/src/FrostFS.SDK.Client/Mappers/OwnerId.cs b/src/FrostFS.SDK.Client/Mappers/OwnerId.cs index 54f7a21..6739a0b 100644 --- a/src/FrostFS.SDK.Client/Mappers/OwnerId.cs +++ b/src/FrostFS.SDK.Client/Mappers/OwnerId.cs @@ -26,7 +26,7 @@ public static class OwnerIdMapper { message = new OwnerID { - Value = UnsafeByteOperations.UnsafeWrap(model.ToHash()) + Value = ByteString.CopyFrom(model.ToHash()) }; Caches.Owners.Set(model, message, _oneHourExpiration); diff --git a/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs b/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs new file mode 100644 index 0000000..d1307e8 --- /dev/null +++ b/src/FrostFS.SDK.Client/Mappers/Session/SessionMapper.cs @@ -0,0 +1,28 @@ +using System; + +using Google.Protobuf; + +namespace FrostFS.SDK.Client; + +public static class SessionMapper +{ + public static byte[] Serialize(this Session.SessionToken token) + { + if (token is null) + { + throw new ArgumentNullException(nameof(token)); + } + + byte[] bytes = new byte[token.CalculateSize()]; + using CodedOutputStream stream = new(bytes); + token.WriteTo(stream); + + return bytes; + } + + public static Session.SessionToken Deserialize(this Session.SessionToken token, byte[] bytes) + { + token.MergeFrom(bytes); + return token; + } +} diff --git a/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs b/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs index ffae746..88b4ac8 100644 --- a/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/SignatureMapper.cs @@ -23,9 +23,9 @@ public static class SignatureMapper return new Refs.Signature { - Key = UnsafeByteOperations.UnsafeWrap(signature.Key), + Key = ByteString.CopyFrom(signature.Key), Scheme = scheme, - Sign = UnsafeByteOperations.UnsafeWrap(signature.Sign) + Sign = ByteString.CopyFrom(signature.Sign) }; } } diff --git a/src/FrostFS.SDK.Client/Mappers/Version.cs b/src/FrostFS.SDK.Client/Mappers/Version.cs index af8738d..a66df6e 100644 --- a/src/FrostFS.SDK.Client/Mappers/Version.cs +++ b/src/FrostFS.SDK.Client/Mappers/Version.cs @@ -18,7 +18,7 @@ public static class VersionMapper throw new System.ArgumentNullException(nameof(model)); } - var key = (int)model.Major << 16 + (int)model.Minor; + var key = model.Major << 16 + model.Minor; if (!_cacheMessages.ContainsKey(key)) { @@ -28,8 +28,8 @@ public static class VersionMapper _spinlock.Enter(ref lockTaken); var message = new Version { - Major = model.Major, - Minor = model.Minor + Major = (uint)model.Major, + Minor = (uint)model.Minor }; _cacheMessages.Add(key, message); @@ -64,7 +64,7 @@ public static class VersionMapper try { _spinlock.Enter(ref lockTaken); - var model = new FrostFsVersion(message.Major, message.Minor); + var model = new FrostFsVersion((int)message.Major, (int)message.Minor); _cacheModels.Add(key, model); return model; diff --git a/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs b/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs index c7dc782..af4ffdc 100644 --- a/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs +++ b/src/FrostFS.SDK.Client/Models/Containers/FrostFsContainerInfo.cs @@ -91,13 +91,10 @@ public class FrostFsContainerInfo throw new ArgumentNullException("PlacementPolicy is null"); } - Span nonce = stackalloc byte[16]; - Nonce.ToBytes(nonce); - this.container = new Container.Container() { PlacementPolicy = PlacementPolicy.Value.GetPolicy(), - Nonce = ByteString.CopyFrom(nonce), + Nonce = ByteString.CopyFrom(Nonce.ToBytes()), OwnerId = Owner?.OwnerID, Version = Version?.VersionID }; diff --git a/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs b/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs index 2241227..fdeac99 100644 --- a/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs +++ b/src/FrostFS.SDK.Client/Models/Misc/CheckSum.cs @@ -11,7 +11,7 @@ public class CheckSum public static CheckSum CreateCheckSum(byte[] content) { - return new CheckSum { hash = DataHasher.Sha256(content.AsSpan()) }; + return new CheckSum { hash = content.Sha256() }; } public override string ToString() diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs index 82e2240..5ace048 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsReplica.cs @@ -4,12 +4,12 @@ namespace FrostFS.SDK; public struct FrostFsReplica : IEquatable { - public uint Count { get; set; } + public int Count { get; set; } public string Selector { get; set; } public uint EcDataCount { get; set; } public uint EcParityCount { get; set; } - public FrostFsReplica(uint count, string? selector = null) + public FrostFsReplica(int count, string? selector = null) { selector ??= string.Empty; @@ -31,12 +31,12 @@ public struct FrostFsReplica : IEquatable public readonly uint CountNodes() { - return Count != 0 ? Count : EcDataCount + EcParityCount; + return Count != 0 ? (uint)Count : EcDataCount + EcParityCount; } public override readonly int GetHashCode() { - return Count.GetHashCode() ^ Selector.GetHashCode() ^ (int)EcDataCount ^ (int)EcParityCount; + return (Count + Selector.GetHashCode()) ^ (int)EcDataCount ^ (int)EcParityCount; } public static bool operator ==(FrostFsReplica left, FrostFsReplica right) diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs index 9ed9522..81e642d 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsVersion.cs @@ -3,12 +3,12 @@ using FrostFS.SDK.Client.Mappers.GRPC; namespace FrostFS.SDK; -public class FrostFsVersion(uint major, uint minor) +public class FrostFsVersion(int major, int minor) { private Version? version; - public uint Major { get; set; } = major; - public uint Minor { get; set; } = minor; + public int Major { get; set; } = major; + public int Minor { get; set; } = minor; internal Version VersionID { diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs index aedfd35..cd287d0 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs @@ -353,7 +353,7 @@ internal struct Context var start = hasPrefix ? likeWildcard.Length : 0; var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length; - var str = f.Value.Substring(start, end - start); + var str = f.Value.Substring(start, end-start); if (hasPrefix && hasSuffix) return nodeInfo.Attributes[f.Key].Contains(str); diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs index e22016a..c39a587 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsObject.cs @@ -4,6 +4,10 @@ namespace FrostFS.SDK; public class FrostFsObject { + // private byte[]? _payloadBytes; + // private ReadOnlyMemory _payloadMemory; + // private bool _isInitPayloadMemory; + /// /// Creates new instance from ObjectHeader /// @@ -45,6 +49,25 @@ public class FrostFsObject public ReadOnlyMemory SingleObjectPayload { get; set; } + //public ReadOnlyMemory SingleObjectPayloadMemory + //{ + // get + // { + // if (!_isInitPayloadMemory) + // { + // _payloadMemory = _payloadBytes.AsMemory(); + // _isInitPayloadMemory = true; + // } + + // return _payloadMemory; + // } + // set + // { + // _payloadMemory = value; + // _isInitPayloadMemory = true; + // } + //} + /// /// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic /// diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs index 5afd46b..a173b34 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs @@ -22,7 +22,7 @@ public class FrostFsSplit(SplitId splitId, public FrostFsObjectId? Previous { get; } = previous; - public FrostFsObjectId? Parent { get; } = parent; + public FrostFsObjectId? Parent { get; internal set; } = parent; public FrostFsSignature? ParentSignature { get; } = parentSignature; @@ -39,7 +39,8 @@ public class FrostFsSplit(SplitId splitId, SplitId = SplitId?.GetSplitId(), Parent = Parent?.ToMessage(), ParentHeader = ParentHeader?.GetHeader(), - ParentSignature = ParentSignature?.ToMessage() + ParentSignature = ParentSignature?.ToMessage(), + Previous = Previous?.ToMessage(), }; if (Children != null) diff --git a/src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs b/src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs deleted file mode 100644 index 16ad326..0000000 --- a/src/FrostFS.SDK.Client/Models/Object/PartUploadedEventArgs.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace FrostFS.SDK.Client; - -public class PartUploadedEventArgs(ObjectPartInfo part) : EventArgs -{ - public ObjectPartInfo Part { get; } = part; -} diff --git a/src/FrostFS.SDK.Client/Models/Object/SplitId.cs b/src/FrostFS.SDK.Client/Models/Object/SplitId.cs index a113361..177817e 100644 --- a/src/FrostFS.SDK.Client/Models/Object/SplitId.cs +++ b/src/FrostFS.SDK.Client/Models/Object/SplitId.cs @@ -47,11 +47,16 @@ public class SplitId return this.id.ToString(); } + public byte[]? ToBinary() + { + if (this.id == Guid.Empty) + return null; + + return this.id.ToBytes(); + } + public ByteString? GetSplitId() { - Span span = stackalloc byte[16]; - id.ToBytes(span); - - return this.message ??= ByteString.CopyFrom(span); + return this.message ??= ByteString.CopyFrom(ToBinary()); } } diff --git a/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs b/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs deleted file mode 100644 index ca96c7d..0000000 --- a/src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -namespace FrostFS.SDK.Client; - -public readonly struct ObjectPartInfo(long offset, int length, FrostFsObjectId objectId) : System.IEquatable -{ - public long Offset { get; } = offset; - public int Length { get; } = length; - public FrostFsObjectId ObjectId { get; } = objectId; - - public override bool Equals(object obj) - { - if (obj == null || obj is not ObjectPartInfo) - return false; - - return Equals((ObjectPartInfo)obj); - } - - public override int GetHashCode() - { - return ((int)(Offset >> 32)) ^ (int)Offset ^ Length ^ ObjectId.Value.GetHashCode(); - } - - public static bool operator ==(ObjectPartInfo left, ObjectPartInfo right) - { - return left.Equals(right); - } - - public static bool operator !=(ObjectPartInfo left, ObjectPartInfo right) - { - return !(left == right); - } - - public bool Equals(ObjectPartInfo other) - { - return GetHashCode() == other.GetHashCode(); - } -} diff --git a/src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs b/src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs deleted file mode 100644 index a6fa905..0000000 --- a/src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; - -namespace FrostFS.SDK.Client; - -public class UploadProgressInfo -{ - private List _parts; - - public UploadProgressInfo(Guid splitId, int capacity = 8) - { - _parts = new List(capacity); - SplitId = new SplitId(splitId); - } - - public UploadProgressInfo(Guid splitId, Collection parts) - { - _parts = [.. parts]; - SplitId = new SplitId(splitId); - } - - public event EventHandler? Notify; - - public SplitId SplitId { get; } - - internal void AddPart(ObjectPartInfo part) - { - _parts.Add(part); - Notify?.Invoke(this, new PartUploadedEventArgs(part)); - } - - public ObjectPartInfo GetPart(int index) - { - return _parts[index]; - } - - public ObjectPartInfo GetLast() - { - return _parts.LastOrDefault(); - } - - public ReadOnlyCollection GetParts() - { - return new ReadOnlyCollection(_parts); - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs index 7b003e7..2f48ba3 100644 --- a/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs +++ b/src/FrostFS.SDK.Client/Models/Response/FrostFsResponseStatus.cs @@ -3,11 +3,10 @@ namespace FrostFS.SDK; public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = null, string? details = null) { public FrostFsStatusCode Code { get; set; } = code; - public string Message { get; set; } = message ?? string.Empty; - + public string Details { get; set; } = details ?? string.Empty; - + public bool IsSuccess => Code == FrostFsStatusCode.Success; public override string ToString() diff --git a/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs b/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs index 547b2b5..36dad09 100644 --- a/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs +++ b/src/FrostFS.SDK.Client/Models/Response/MetaHeader.cs @@ -1,10 +1,10 @@ namespace FrostFS.SDK; -public class MetaHeader(FrostFsVersion version, ulong epoch, uint ttl) +public class MetaHeader(FrostFsVersion version, int epoch, int ttl) { public FrostFsVersion Version { get; set; } = version; - public ulong Epoch { get; set; } = epoch; - public uint Ttl { get; set; } = ttl; + public int Epoch { get; set; } = epoch; + public int Ttl { get; set; } = ttl; public static MetaHeader Default() { diff --git a/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs b/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs index 2303776..720c3c4 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmApeChainRemove.cs @@ -25,7 +25,7 @@ public readonly struct PrmApeChainRemove( public readonly bool Equals(PrmApeChainRemove other) { return Target == other.Target - && ChainId.Equals(other.ChainId) + && ChainId.Equals(other.ChainId) && XHeaders == other.XHeaders; } diff --git a/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs b/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs index 46f25e0..706e367 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmObjectClientCutPut.cs @@ -8,8 +8,7 @@ public readonly struct PrmObjectClientCutPut( int bufferMaxSize = 0, FrostFsSessionToken? sessionToken = null, byte[]? customBuffer = null, - string[]? xheaders = null, - UploadProgressInfo? progress = null) : PrmObjectPutBase, System.IEquatable + string[]? xheaders = null) : PrmObjectPutBase, System.IEquatable { /// /// Need to provide values like ContainerId and ObjectType to create and object. @@ -42,8 +41,6 @@ public readonly struct PrmObjectClientCutPut( /// public string[] XHeaders { get; } = xheaders ?? []; - public UploadProgressInfo? Progress { get; } = progress; - internal PutObjectContext PutObjectContext { get; } = new(); public override readonly bool Equals(object obj) diff --git a/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs b/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs index 7b26f5b..fc02c7c 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs @@ -10,6 +10,7 @@ public readonly struct PrmObjectPatch( FrostFsSessionToken? sessionToken = null, bool replaceAttributes = false, FrostFsAttributePair[]? newAttributes = null, + FrostFsSplit? newSplitHeader = null, string[]? xheaders = null) : ISessionToken, System.IEquatable { public FrostFsAddress Address { get; } = address; @@ -23,6 +24,8 @@ public readonly struct PrmObjectPatch( public FrostFsAttributePair[]? NewAttributes { get; } = newAttributes; + public FrostFsSplit? NewSplitHeader { get; } = newSplitHeader; + public bool ReplaceAttributes { get; } = replaceAttributes; public int MaxChunkLength { get; } = maxChunkLength; diff --git a/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs b/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs new file mode 100644 index 0000000..c05c24c --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/ClientStatusMonitor.cs @@ -0,0 +1,163 @@ +using System; +using System.Threading; + +using Microsoft.Extensions.Logging; + +namespace FrostFS.SDK.Client; + +// clientStatusMonitor count error rate and other statistics for connection. +public class ClientStatusMonitor : IClientStatus +{ + private static readonly MethodIndex[] MethodIndexes = + [ + MethodIndex.methodBalanceGet, + MethodIndex.methodContainerPut, + MethodIndex.methodContainerGet, + MethodIndex.methodContainerList, + MethodIndex.methodContainerDelete, + MethodIndex.methodEndpointInfo, + MethodIndex.methodNetworkInfo, + MethodIndex.methodNetMapSnapshot, + MethodIndex.methodObjectPut, + MethodIndex.methodObjectDelete, + MethodIndex.methodObjectGet, + MethodIndex.methodObjectHead, + MethodIndex.methodObjectRange, + MethodIndex.methodObjectPatch, + MethodIndex.methodSessionCreate, + MethodIndex.methodAPEManagerAddChain, + MethodIndex.methodAPEManagerRemoveChain, + MethodIndex.methodAPEManagerListChains + ]; + + public static string GetMethodName(MethodIndex index) + { + return index switch + { + MethodIndex.methodBalanceGet => "BalanceGet", + MethodIndex.methodContainerPut => "ContainerPut", + MethodIndex.methodContainerGet => "ContainerGet", + MethodIndex.methodContainerList => "ContainerList", + MethodIndex.methodContainerDelete => "ContainerDelete", + MethodIndex.methodEndpointInfo => "EndpointInfo", + MethodIndex.methodNetworkInfo => "NetworkInfo", + MethodIndex.methodNetMapSnapshot => "NetMapSnapshot", + MethodIndex.methodObjectPut => "ObjectPut", + MethodIndex.methodObjectDelete => "ObjectDelete", + MethodIndex.methodObjectGet => "ObjectGet", + MethodIndex.methodObjectHead => "ObjectHead", + MethodIndex.methodObjectRange => "ObjectRange", + MethodIndex.methodObjectPatch => "ObjectPatch", + MethodIndex.methodSessionCreate => "SessionCreate", + MethodIndex.methodAPEManagerAddChain => "APEManagerAddChain", + MethodIndex.methodAPEManagerRemoveChain => "APEManagerRemoveChain", + MethodIndex.methodAPEManagerListChains => "APEManagerListChains", + _ => throw new ArgumentException("Unknown method", nameof(index)), + }; + } + + private readonly object _lock = new(); + + private readonly ILogger? logger; + private int healthy; + + public ClientStatusMonitor(ILogger? logger, string address) + { + this.logger = logger; + healthy = (int)HealthyStatus.Healthy; + + Address = address; + Methods = new MethodStatus[MethodIndexes.Length]; + + for (int i = 0; i < MethodIndexes.Length; i++) + { + Methods[i] = new MethodStatus(GetMethodName(MethodIndexes[i])); + } + } + + public string Address { get; } + + internal uint ErrorThreshold { get; set; } + + public uint CurrentErrorCount { get; set; } + + public ulong OverallErrorCount { get; set; } + + public MethodStatus[] Methods { get; private set; } + + public bool IsHealthy() + { + var res = Interlocked.CompareExchange(ref healthy, -1, -1) == (int)HealthyStatus.Healthy; + return res; + } + + public bool IsDialed() + { + return Interlocked.CompareExchange(ref healthy, -1, -1) != (int)HealthyStatus.UnhealthyOnDial; + } + + public void SetHealthy() + { + Interlocked.Exchange(ref healthy, (int)HealthyStatus.Healthy); + } + public void SetUnhealthy() + { + Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnRequest); + } + + public void SetUnhealthyOnDial() + { + Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnDial); + } + + public void IncErrorRate() + { + bool thresholdReached; + lock (_lock) + { + CurrentErrorCount++; + OverallErrorCount++; + + thresholdReached = CurrentErrorCount >= ErrorThreshold; + + if (thresholdReached) + { + SetUnhealthy(); + CurrentErrorCount = 0; + } + } + + if (thresholdReached && logger != null) + { + FrostFsMessages.ErrorЕhresholdReached(logger, Address, ErrorThreshold); + } + } + + public uint GetCurrentErrorRate() + { + lock (_lock) + { + return CurrentErrorCount; + } + } + + public ulong GetOverallErrorRate() + { + lock (_lock) + { + return OverallErrorCount; + } + } + + public StatusSnapshot[] MethodsStatus() + { + var result = new StatusSnapshot[Methods.Length]; + + for (int i = 0; i < result.Length; i++) + { + result[i] = Methods[i].Snapshot!; + } + + return result; + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs b/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs new file mode 100644 index 0000000..552ee2a --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/ClientWrapper.cs @@ -0,0 +1,135 @@ +using System; +using System.Threading.Tasks; + +using Grpc.Core; + +namespace FrostFS.SDK.Client; + +// clientWrapper is used by default, alternative implementations are intended for testing purposes only. +public class ClientWrapper : ClientStatusMonitor +{ + private readonly object _lock = new(); + private SessionCache sessionCache; + + internal ClientWrapper(WrapperPrm wrapperPrm, Pool pool) : base(wrapperPrm.Logger, wrapperPrm.Address) + { + WrapperPrm = wrapperPrm; + ErrorThreshold = wrapperPrm.ErrorThreshold; + + sessionCache = pool.SessionCache; + Client = new FrostFSClient(WrapperPrm, sessionCache); + } + + internal FrostFSClient? Client { get; private set; } + + internal WrapperPrm WrapperPrm { get; } + + internal FrostFSClient? GetClient() + { + lock (_lock) + { + if (IsHealthy()) + { + return Client; + } + + return null; + } + } + + // dial establishes a connection to the server from the FrostFS network. + // Returns an error describing failure reason. If failed, the client + // SHOULD NOT be used. + internal async Task Dial(CallContext ctx) + { + var client = GetClient() ?? throw new FrostFsInvalidObjectException("pool client unhealthy"); + + await client.Dial(ctx).ConfigureAwait(false); + } + + internal void HandleError(Exception ex) + { + if (ex is FrostFsResponseException responseException && responseException.Status != null) + { + switch (responseException.Status.Code) + { + case FrostFsStatusCode.Internal: + case FrostFsStatusCode.WrongMagicNumber: + case FrostFsStatusCode.SignatureVerificationFailure: + case FrostFsStatusCode.NodeUnderMaintenance: + IncErrorRate(); + return; + } + } + + IncErrorRate(); + } + + private async Task ScheduleGracefulClose() + { + if (Client == null) + return; + + await Task.Delay((int)WrapperPrm.GracefulCloseOnSwitchTimeout).ConfigureAwait(false); + } + + // restartIfUnhealthy checks healthy status of client and recreate it if status is unhealthy. + // Indicating if status was changed by this function call and returns error that caused unhealthy status. + internal async Task RestartIfUnhealthy(CallContext ctx) + { + bool wasHealthy; + + try + { + var response = await Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false); + return false; + } + catch (RpcException) + { + wasHealthy = true; + } + + // if connection is dialed before, to avoid routine/connection leak, + // pool has to close it and then initialize once again. + if (IsDialed()) + { + await ScheduleGracefulClose().ConfigureAwait(false); + } + + FrostFSClient? client = new(WrapperPrm, sessionCache); + + var error = await client.Dial(ctx).ConfigureAwait(false); + if (!string.IsNullOrEmpty(error)) + { + SetUnhealthyOnDial(); + return wasHealthy; + } + + lock (_lock) + { + Client = client; + } + + try + { + var res = await Client.GetNodeInfoAsync(ctx).ConfigureAwait(false); + } + catch (FrostFsException) + { + SetUnhealthy(); + + return wasHealthy; + } + + SetHealthy(); + return !wasHealthy; + } + + internal void IncRequests(ulong elapsed, MethodIndex method) + { + var methodStat = Methods[(int)method]; + + methodStat.IncRequests(elapsed); + } +} + diff --git a/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs b/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs new file mode 100644 index 0000000..d02e6bb --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/HealthyStatus.cs @@ -0,0 +1,18 @@ +namespace FrostFS.SDK.Client; + +// values for healthy status of clientStatusMonitor. +public enum HealthyStatus +{ + // statusUnhealthyOnDial is set when dialing to the endpoint is failed, + // so there is no connection to the endpoint, and pool should not close it + // before re-establishing connection once again. + UnhealthyOnDial, + + // statusUnhealthyOnRequest is set when communication after dialing to the + // endpoint is failed due to immediate or accumulated errors, connection is + // available and pool should close it before re-establishing connection once again. + UnhealthyOnRequest, + + // statusHealthy is set when connection is ready to be used by the pool. + Healthy +} diff --git a/src/FrostFS.SDK.Client/Pool/IClientStatus.cs b/src/FrostFS.SDK.Client/Pool/IClientStatus.cs new file mode 100644 index 0000000..0c08fac --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/IClientStatus.cs @@ -0,0 +1,28 @@ +namespace FrostFS.SDK.Client; + +public interface IClientStatus +{ + // isHealthy checks if the connection can handle requests. + bool IsHealthy(); + + // isDialed checks if the connection was created. + bool IsDialed(); + + // setUnhealthy marks client as unhealthy. + void SetUnhealthy(); + + // address return address of endpoint. + string Address { get; } + + // currentErrorRate returns current errors rate. + // After specific threshold connection is considered as unhealthy. + // Pool.startRebalance routine can make this connection healthy again. + uint GetCurrentErrorRate(); + + // overallErrorRate returns the number of all happened errors. + ulong GetOverallErrorRate(); + + // methodsStatus returns statistic for all used methods. + StatusSnapshot[] MethodsStatus(); +} + diff --git a/src/FrostFS.SDK.Client/Pool/InitParameters.cs b/src/FrostFS.SDK.Client/Pool/InitParameters.cs new file mode 100644 index 0000000..66da23f --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/InitParameters.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.ObjectModel; +using System.Security.Cryptography; + +using Grpc.Core; +using Grpc.Core.Interceptors; +using Grpc.Net.Client; + +using Microsoft.Extensions.Logging; + +namespace FrostFS.SDK.Client; + +// InitParameters contains values used to initialize connection Pool. +public class InitParameters(Func grpcChannelFactory) +{ + public ECDsa? Key { get; set; } + + public ulong NodeDialTimeout { get; set; } + + public ulong NodeStreamTimeout { get; set; } + + public ulong HealthcheckTimeout { get; set; } + + public ulong ClientRebalanceInterval { get; set; } + + public ulong SessionExpirationDuration { get; set; } + + public uint ErrorThreshold { get; set; } + + public NodeParam[]? NodeParams { get; set; } + + public GrpcChannelOptions[]? DialOptions { get; set; } + + public Func? ClientBuilder { get; set; } + + public ulong GracefulCloseOnSwitchTimeout { get; set; } + + public ILogger? Logger { get; set; } + + public Action? Callback { get; set; } + + public Collection Interceptors { get; } = []; + + public Func GrpcChannelFactory { get; set; } = grpcChannelFactory; +} diff --git a/src/FrostFS.SDK.Client/Pool/InnerPool.cs b/src/FrostFS.SDK.Client/Pool/InnerPool.cs new file mode 100644 index 0000000..f712552 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/InnerPool.cs @@ -0,0 +1,47 @@ +using FrostFS.SDK.Client; + +internal sealed class InnerPool +{ + private readonly object _lock = new(); + + internal InnerPool(Sampler sampler, ClientWrapper[] clients) + { + Clients = clients; + Sampler = sampler; + } + + internal Sampler Sampler { get; set; } + + internal ClientWrapper[] Clients { get; } + + internal ClientWrapper? Connection() + { + lock (_lock) + { + if (Clients.Length == 1) + { + var client = Clients[0]; + if (client.IsHealthy()) + { + return client; + } + } + else + { + var attempts = 3 * Clients.Length; + + for (int i = 0; i < attempts; i++) + { + int index = Sampler.Next(); + + if (Clients[index].IsHealthy()) + { + return Clients[index]; + } + } + } + + return null; + } + } +} diff --git a/src/FrostFS.SDK.Client/Pool/MethodIndex.cs b/src/FrostFS.SDK.Client/Pool/MethodIndex.cs new file mode 100644 index 0000000..53e5430 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/MethodIndex.cs @@ -0,0 +1,24 @@ +namespace FrostFS.SDK.Client; + +public enum MethodIndex +{ + methodBalanceGet, + methodContainerPut, + methodContainerGet, + methodContainerList, + methodContainerDelete, + methodEndpointInfo, + methodNetworkInfo, + methodNetMapSnapshot, + methodObjectPut, + methodObjectDelete, + methodObjectGet, + methodObjectHead, + methodObjectRange, + methodObjectPatch, + methodSessionCreate, + methodAPEManagerAddChain, + methodAPEManagerRemoveChain, + methodAPEManagerListChains, + methodLast +} diff --git a/src/FrostFS.SDK.Client/Pool/MethodStatus.cs b/src/FrostFS.SDK.Client/Pool/MethodStatus.cs new file mode 100644 index 0000000..8f40f3c --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/MethodStatus.cs @@ -0,0 +1,19 @@ +namespace FrostFS.SDK.Client; + +public class MethodStatus(string name) +{ + private readonly object _lock = new(); + + public string Name { get; } = name; + + public StatusSnapshot Snapshot { get; set; } = new StatusSnapshot(); + + internal void IncRequests(ulong elapsed) + { + lock (_lock) + { + Snapshot.AllTime += elapsed; + Snapshot.AllRequests++; + } + } +} diff --git a/src/FrostFS.SDK.Client/Pool/NodeParam.cs b/src/FrostFS.SDK.Client/Pool/NodeParam.cs new file mode 100644 index 0000000..92c2560 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/NodeParam.cs @@ -0,0 +1,12 @@ +namespace FrostFS.SDK.Client; + +// NodeParam groups parameters of remote node. +[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "")] +public readonly struct NodeParam(int priority, string address, float weight) +{ + public int Priority { get; } = priority; + + public string Address { get; } = address; + + public float Weight { get; } = weight; +} diff --git a/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs b/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs new file mode 100644 index 0000000..9323d93 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/NodeStatistic.cs @@ -0,0 +1,12 @@ +namespace FrostFS.SDK.Client; + +public class NodeStatistic +{ + public string? Address { get; internal set; } + + public StatusSnapshot[]? Methods { get; internal set; } + + public ulong OverallErrors { get; internal set; } + + public uint CurrentErrors { get; internal set; } +} diff --git a/src/FrostFS.SDK.Client/Pool/NodesParam.cs b/src/FrostFS.SDK.Client/Pool/NodesParam.cs new file mode 100644 index 0000000..be9f012 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/NodesParam.cs @@ -0,0 +1,12 @@ +using System.Collections.ObjectModel; + +namespace FrostFS.SDK.Client; + +public class NodesParam(int priority) +{ + public int Priority { get; } = priority; + + public Collection Addresses { get; } = []; + + public Collection Weights { get; } = []; +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Pool/Pool.cs b/src/FrostFS.SDK.Client/Pool/Pool.cs new file mode 100644 index 0000000..cd09d30 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/Pool.cs @@ -0,0 +1,677 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; + +using FrostFS.Refs; +using FrostFS.SDK.Client.Interfaces; +using FrostFS.SDK.Client.Mappers.GRPC; +using FrostFS.SDK.Cryptography; + +using Grpc.Core; + +using Microsoft.Extensions.Logging; + + +namespace FrostFS.SDK.Client; + +public partial class Pool : IFrostFSClient +{ + const int defaultSessionTokenExpirationDuration = 100; // in epochs + + const int defaultErrorThreshold = 100; + + const int defaultGracefulCloseOnSwitchTimeout = 10; //Seconds; + const int defaultRebalanceInterval = 15; //Seconds; + const int defaultHealthcheckTimeout = 4; //Seconds; + const int defaultDialTimeout = 5; //Seconds; + const int defaultStreamTimeout = 10; //Seconds; + + private readonly object _lock = new(); + + private InnerPool[]? InnerPools { get; set; } + + private ClientKey Key { get; set; } + + private OwnerID? _ownerId; + + private FrostFsVersion _version; + + private FrostFsOwner? _owner; + + private FrostFsOwner Owner + { + get + { + _owner ??= new FrostFsOwner(Key.ECDsaKey.PublicKey().PublicKeyToAddress()); + return _owner; + } + } + + private OwnerID OwnerId + { + get + { + if (_ownerId == null) + { + _owner = Key.Owner; + _ownerId = _owner.ToMessage(); + } + return _ownerId; + } + } + + internal CancellationTokenSource CancellationTokenSource { get; } = new CancellationTokenSource(); + + internal SessionCache SessionCache { get; set; } + + private ulong SessionTokenDuration { get; set; } + + private RebalanceParameters RebalanceParams { get; set; } + + private Func ClientBuilder; + + private bool disposedValue; + + private ILogger? logger { get; set; } + + private ulong MaxObjectSize { get; set; } + + public IClientStatus? ClientStatus { get; } + + // NewPool creates connection pool using parameters. + public Pool(InitParameters options) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (options.Key == null) + { + throw new ArgumentException($"Missed required parameter {nameof(options.Key)}"); + } + + _version = new FrostFsVersion(2, 13); + + var nodesParams = AdjustNodeParams(options.NodeParams); + + var cache = new SessionCache(options.SessionExpirationDuration); + + FillDefaultInitParams(options, this); + + Key = new ClientKey(options.Key); + + SessionCache = cache; + logger = options.Logger; + SessionTokenDuration = options.SessionExpirationDuration; + + RebalanceParams = new RebalanceParameters( + nodesParams.ToArray(), + options.HealthcheckTimeout, + options.ClientRebalanceInterval, + options.SessionExpirationDuration); + + ClientBuilder = options.ClientBuilder!; + + // ClientContext.PoolErrorHandler = client.HandleError; + + } + + // Dial establishes a connection to the servers from the FrostFS network. + // It also starts a routine that checks the health of the nodes and + // updates the weights of the nodes for balancing. + // Returns an error describing failure reason. + // + // If failed, the Pool SHOULD NOT be used. + // + // See also InitParameters.SetClientRebalanceInterval. + public async Task Dial(CallContext ctx) + { + var inner = new InnerPool[RebalanceParams.NodesParams.Length]; + + bool atLeastOneHealthy = false; + int i = 0; + foreach (var nodeParams in RebalanceParams.NodesParams) + { + var wrappers = new ClientWrapper[nodeParams.Weights.Count]; + + for (int j = 0; j < nodeParams.Addresses.Count; j++) + { + ClientWrapper? wrapper = null; + bool dialed = false; + try + { + wrapper = wrappers[j] = ClientBuilder(nodeParams.Addresses[j]); + + await wrapper.Dial(ctx).ConfigureAwait(false); + dialed = true; + + var token = await InitSessionForDuration(ctx, wrapper, RebalanceParams.SessionExpirationDuration, Key.ECDsaKey, false) + .ConfigureAwait(false); + + var key = FormCacheKey(nodeParams.Addresses[j], Key.PublicKey); + SessionCache.SetValue(key, token); + + atLeastOneHealthy = true; + } + catch (RpcException ex) + { + if (!dialed) + wrapper!.SetUnhealthyOnDial(); + else + wrapper!.SetUnhealthy(); + + if (logger != null) + { + FrostFsMessages.SessionCreationError(logger, wrapper!.WrapperPrm.Address, ex.Message); + } + } + catch (FrostFsInvalidObjectException) + { + break; + } + } + + var sampler = new Sampler(nodeParams.Weights.ToArray()); + + inner[i] = new InnerPool(sampler, wrappers); + + i++; + } + + if (!atLeastOneHealthy) + return "At least one node must be healthy"; + + InnerPools = inner; + + var res = await GetNetworkSettingsAsync(default).ConfigureAwait(false); + + MaxObjectSize = res.MaxObjectSize; + + StartRebalance(ctx); + + return null; + } + + private static IEnumerable AdjustNodeParams(NodeParam[]? nodeParams) + { + if (nodeParams == null || nodeParams.Length == 0) + { + throw new ArgumentException("No FrostFS peers configured"); + } + + Dictionary nodesParamsDict = new(nodeParams.Length); + foreach (var nodeParam in nodeParams) + { + if (!nodesParamsDict.TryGetValue(nodeParam.Priority, out var nodes)) + { + nodes = new NodesParam(nodeParam.Priority); + nodesParamsDict[nodeParam.Priority] = nodes; + } + + nodes.Addresses.Add(nodeParam.Address); + nodes.Weights.Add(nodeParam.Weight); + } + + var nodesParams = new List(nodesParamsDict.Count); + + foreach (var key in nodesParamsDict.Keys) + { + var nodes = nodesParamsDict[key]; + var newWeights = AdjustWeights([.. nodes.Weights]); + nodes.Weights.Clear(); + foreach (var weight in newWeights) + { + nodes.Weights.Add(weight); + } + + nodesParams.Add(nodes); + } + + return nodesParams.OrderBy(n => n.Priority); + } + + private static double[] AdjustWeights(double[] weights) + { + var adjusted = new double[weights.Length]; + + var sum = weights.Sum(); + + if (sum > 0) + { + for (int i = 0; i < weights.Length; i++) + { + adjusted[i] = weights[i] / sum; + } + } + + return adjusted; + } + + private static void FillDefaultInitParams(InitParameters parameters, Pool pool) + { + if (parameters.SessionExpirationDuration == 0) + parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration; + + if (parameters.ErrorThreshold == 0) + parameters.ErrorThreshold = defaultErrorThreshold; + + if (parameters.ClientRebalanceInterval <= 0) + parameters.ClientRebalanceInterval = defaultRebalanceInterval; + + if (parameters.GracefulCloseOnSwitchTimeout <= 0) + parameters.GracefulCloseOnSwitchTimeout = defaultGracefulCloseOnSwitchTimeout; + + if (parameters.HealthcheckTimeout <= 0) + parameters.HealthcheckTimeout = defaultHealthcheckTimeout; + + if (parameters.NodeDialTimeout <= 0) + parameters.NodeDialTimeout = defaultDialTimeout; + + if (parameters.NodeStreamTimeout <= 0) + parameters.NodeStreamTimeout = defaultStreamTimeout; + + if (parameters.SessionExpirationDuration == 0) + parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration; + + parameters.ClientBuilder ??= new Func((address) => + { + var wrapperPrm = new WrapperPrm() + { + Address = address, + Key = parameters.Key!, + Logger = parameters.Logger, + DialTimeout = parameters.NodeDialTimeout, + StreamTimeout = parameters.NodeStreamTimeout, + ErrorThreshold = parameters.ErrorThreshold, + GracefulCloseOnSwitchTimeout = parameters.GracefulCloseOnSwitchTimeout, + Callback = parameters.Callback, + Interceptors = parameters.Interceptors, + GrpcChannelFactory = parameters.GrpcChannelFactory + + }; + + return new ClientWrapper(wrapperPrm, pool); + } + ); + } + + private ClientWrapper Connection() + { + foreach (var pool in InnerPools!) + { + var client = pool.Connection(); + if (client != null) + { + return client; + } + } + + throw new FrostFsException("Cannot find alive client"); + } + + private static async Task InitSessionForDuration(CallContext ctx, ClientWrapper cw, ulong duration, ECDsa key, bool clientCut) + { + var client = cw.Client; + var networkInfo = await client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); + + var epoch = networkInfo.Epoch; + + ulong exp = ulong.MaxValue - epoch < duration + ? ulong.MaxValue + : epoch + duration; + + var prmSessionCreate = new PrmSessionCreate(exp); + + return await client.CreateSessionAsync(prmSessionCreate, ctx).ConfigureAwait(false); + } + + internal static string FormCacheKey(string address, string key) + { + return $"{address}{key}"; + } + + public void Close() + { + CancellationTokenSource.Cancel(); + + //if (InnerPools != null) + //{ + // // close all clients + // foreach (var innerPool in InnerPools) + // foreach (var client in innerPool.Clients) + // if (client.IsDialed()) + // client.Client?.Close(); + //} + } + + // startRebalance runs loop to monitor connection healthy status. + internal void StartRebalance(CallContext ctx) + { + var buffers = new double[RebalanceParams.NodesParams.Length][]; + + for (int i = 0; i < RebalanceParams.NodesParams.Length; i++) + { + var parameters = RebalanceParams.NodesParams[i]; + buffers[i] = new double[parameters.Weights.Count]; + + Task.Run(async () => + { + await Task.Delay((int)RebalanceParams.ClientRebalanceInterval).ConfigureAwait(false); + UpdateNodesHealth(ctx, buffers); + }); + } + } + + private void UpdateNodesHealth(CallContext ctx, double[][] buffers) + { + var tasks = new Task[InnerPools!.Length]; + + for (int i = 0; i < InnerPools.Length; i++) + { + var bufferWeights = buffers[i]; + + tasks[i] = Task.Run(() => UpdateInnerNodesHealth(ctx, i, bufferWeights)); + } + + Task.WaitAll(tasks); + } + + private async ValueTask UpdateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights) + { + if (poolIndex > InnerPools!.Length - 1) + { + return; + } + + var pool = InnerPools[poolIndex]; + + var options = RebalanceParams; + + int healthyChanged = 0; + + var tasks = new Task[pool.Clients.Length]; + + for (int j = 0; j < pool.Clients.Length; j++) + { + var client = pool.Clients[j]; + var healthy = false; + string? error = null; + var changed = false; + + try + { + // check timeout settings + changed = await client!.RestartIfUnhealthy(ctx).ConfigureAwait(false); + healthy = true; + bufferWeights[j] = options.NodesParams[poolIndex].Weights[j]; + } + // TODO: specify + catch (FrostFsException e) + { + error = e.Message; + bufferWeights[j] = 0; + + SessionCache.DeleteByPrefix(client.Address); + } + + if (changed) + { + if (!string.IsNullOrEmpty(error)) + { + if (logger != null) + { + FrostFsMessages.HealthChanged(logger, client.Address, healthy, error!); + } + + Interlocked.Exchange(ref healthyChanged, 1); + } + } + + await Task.WhenAll(tasks).ConfigureAwait(false); + + if (Interlocked.CompareExchange(ref healthyChanged, -1, -1) == 1) + { + var probabilities = AdjustWeights(bufferWeights); + + lock (_lock) + { + pool.Sampler = new Sampler(probabilities); + } + } + } + } + + + // TODO: remove + private bool CheckSessionTokenErr(Exception error, string address) + { + if (error == null) + { + return false; + } + + if (error is SessionNotFoundException || error is SessionExpiredException) + { + this.SessionCache.DeleteByPrefix(address); + return true; + } + + return false; + } + + public Statistic Statistic() + { + if (InnerPools == null) + { + throw new FrostFsInvalidObjectException(nameof(Pool)); + } + + var statistics = new Statistic(); + + foreach (var inner in InnerPools) + { + int valueIndex = 0; + var nodes = new string[inner.Clients.Length]; + + lock (_lock) + { + foreach (var client in inner.Clients) + { + if (client.IsHealthy()) + { + nodes[valueIndex] = client.Address; + } + + var node = new NodeStatistic + { + Address = client.Address, + Methods = client.MethodsStatus(), + OverallErrors = client.GetOverallErrorRate(), + CurrentErrors = client.GetCurrentErrorRate() + }; + + statistics.Nodes.Add(node); + + valueIndex++; + + statistics.OverallErrors += node.OverallErrors; + } + + if (statistics.CurrentNodes == null || statistics.CurrentNodes.Length == 0) + { + statistics.CurrentNodes = nodes; + } + } + } + + return statistics; + } + + public async Task GetNetmapSnapshotAsync(CallContext ctx) + { + var client = Connection(); + + return await client.Client!.GetNetmapSnapshotAsync(ctx).ConfigureAwait(false); + } + + public async Task GetNodeInfoAsync(CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false); + } + + public async Task GetNetworkSettingsAsync(CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); + } + + public async Task CreateSessionAsync(PrmSessionCreate args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.CreateSessionAsync(args, ctx).ConfigureAwait(false); + } + + public async Task> AddChainAsync(PrmApeChainAdd args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.AddChainAsync(args, ctx).ConfigureAwait(false); + } + + public async Task RemoveChainAsync(PrmApeChainRemove args, CallContext ctx) + { + var client = Connection(); + await client.Client!.RemoveChainAsync(args, ctx).ConfigureAwait(false); + } + + public async Task ListChainAsync(PrmApeChainList args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.ListChainAsync(args, ctx).ConfigureAwait(false); + } + + public async Task GetContainerAsync(PrmContainerGet args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetContainerAsync(args, ctx).ConfigureAwait(false); + } + + public IAsyncEnumerable ListContainersAsync(PrmContainerGetAll args, CallContext ctx) + { + var client = Connection(); + return client.Client!.ListContainersAsync(args, ctx); + } + + [Obsolete("Use PutContainerAsync method")] + public async Task CreateContainerAsync(PrmContainerCreate args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false); + } + + public async Task PutContainerAsync(PrmContainerCreate args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false); + } + + public async Task DeleteContainerAsync(PrmContainerDelete args, CallContext ctx) + { + var client = Connection(); + await client.Client!.DeleteContainerAsync(args, ctx).ConfigureAwait(false); + } + + public async Task GetObjectHeadAsync(PrmObjectHeadGet args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetObjectHeadAsync(args, ctx).ConfigureAwait(false); + } + + public async Task GetObjectAsync(PrmObjectGet args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetObjectAsync(args, ctx).ConfigureAwait(false); + } + + public async Task PutObjectAsync(PrmObjectPut args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PutObjectAsync(args, ctx).ConfigureAwait(false); + } + + public async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PutClientCutObjectAsync(args, ctx).ConfigureAwait(false); + } + + public async Task PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PutSingleObjectAsync(args, ctx).ConfigureAwait(false); + } + + public async Task PatchObjectAsync(PrmObjectPatch args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false); + } + + public async Task GetRangeAsync(PrmRangeGet args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetRangeAsync(args, ctx).ConfigureAwait(false); + } + + public async Task[]> GetRangeHashAsync(PrmRangeHashGet args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetRangeHashAsync(args, ctx).ConfigureAwait(false); + } + + public async Task PatchAsync(PrmObjectPatch args, CallContext ctx) + { + var client = Connection(); + return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false); + } + + public async Task DeleteObjectAsync(PrmObjectDelete args, CallContext ctx) + { + var client = Connection(); + await client.Client!.DeleteObjectAsync(args, ctx).ConfigureAwait(false); + } + + public IAsyncEnumerable SearchObjectsAsync(PrmObjectSearch args, CallContext ctx) + { + var client = Connection(); + return client.Client!.SearchObjectsAsync(args, ctx); + } + + public async Task GetBalanceAsync(CallContext ctx) + { + var client = Connection(); + return await client.Client!.GetBalanceAsync(ctx).ConfigureAwait(false); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + Close(); + } + + disposedValue = true; + } + } + + public void Dispose() + { + Dispose(disposing: true); + } +} diff --git a/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs b/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs new file mode 100644 index 0000000..988002c --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/RebalanceParameters.cs @@ -0,0 +1,16 @@ +namespace FrostFS.SDK.Client; + +public class RebalanceParameters( + NodesParam[] nodesParams, + ulong nodeRequestTimeout, + ulong clientRebalanceInterval, + ulong sessionExpirationDuration) +{ + public NodesParam[] NodesParams { get; set; } = nodesParams; + + public ulong NodeRequestTimeout { get; set; } = nodeRequestTimeout; + + public ulong ClientRebalanceInterval { get; set; } = clientRebalanceInterval; + + public ulong SessionExpirationDuration { get; set; } = sessionExpirationDuration; +} diff --git a/src/FrostFS.SDK.Client/Pool/RequestInfo.cs b/src/FrostFS.SDK.Client/Pool/RequestInfo.cs new file mode 100644 index 0000000..3b70650 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/RequestInfo.cs @@ -0,0 +1,14 @@ +using System; + +namespace FrostFS.SDK.Client; + +// RequestInfo groups info about pool request. +struct RequestInfo +{ + public string Address { get; set; } + + public MethodIndex MethodIndex { get; set; } + + public TimeSpan Elapsed { get; set; } +} + diff --git a/src/FrostFS.SDK.Client/Pool/Sampler.cs b/src/FrostFS.SDK.Client/Pool/Sampler.cs new file mode 100644 index 0000000..275f3f0 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/Sampler.cs @@ -0,0 +1,85 @@ +using System; + +namespace FrostFS.SDK.Client; + +internal sealed class Sampler +{ + private readonly object _lock = new(); + + private Random random = new(); + + internal double[] Probabilities { get; set; } + internal int[] Alias { get; set; } + + internal Sampler(double[] probabilities) + { + var small = new WorkList(); + var large = new WorkList(); + + var n = probabilities.Length; + + // sampler.randomGenerator = rand.New(source) + Probabilities = new double[n]; + Alias = new int[n]; + + // Compute scaled probabilities. + var p = new double[n]; + + for (int i = 0; i < n; i++) + { + p[i] = probabilities[i] * n; + if (p[i] < 1) + small.Add(i); + else + large.Add(i); + } + + while (small.Length > 0 && large.Length > 0) + { + var l = small.Remove(); + var g = large.Remove(); + + Probabilities[l] = p[l]; + Alias[l] = g; + + p[g] = p[g] + p[l] - 1; + + if (p[g] < 1) + small.Add(g); + else + large.Add(g); + } + + while (large.Length > 0) + { + var g = large.Remove(); + Probabilities[g] = 1; + } + + while (small.Length > 0) + { + var l = small.Remove(); + probabilities[l] = 1; + } + } + + internal int Next() + { + var n = Alias.Length; + + int i; + double f; + lock (_lock) + { + i = random.Next(0, n - 1); + f = random.NextDouble(); + } + + if (f < Probabilities[i]) + { + return i; + } + + return Alias[i]; + } +} diff --git a/src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs b/src/FrostFS.SDK.Client/Pool/SessionCache.cs similarity index 72% rename from src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs rename to src/FrostFS.SDK.Client/Pool/SessionCache.cs index a02436f..fae3de1 100644 --- a/src/FrostFS.SDK.Client/Services/Shared/SessionCache.cs +++ b/src/FrostFS.SDK.Client/Pool/SessionCache.cs @@ -1,4 +1,5 @@ -using System.Collections.Concurrent; +using System; +using System.Collections.Concurrent; namespace FrostFS.SDK.Client; @@ -35,4 +36,15 @@ internal sealed class SessionCache(ulong sessionExpirationDuration) _cache[key] = value; } } + + internal void DeleteByPrefix(string prefix) + { + foreach (var key in _cache.Keys) + { + if (key.StartsWith(prefix, StringComparison.Ordinal)) + { + _cache.TryRemove(key, out var _); + } + } + } } diff --git a/src/FrostFS.SDK.Client/Pool/Statistic.cs b/src/FrostFS.SDK.Client/Pool/Statistic.cs new file mode 100644 index 0000000..c4977d5 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/Statistic.cs @@ -0,0 +1,12 @@ +using System.Collections.ObjectModel; + +namespace FrostFS.SDK.Client; + +public sealed class Statistic +{ + public ulong OverallErrors { get; internal set; } + + public Collection Nodes { get; } = []; + + public string[]? CurrentNodes { get; internal set; } +} diff --git a/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs b/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs new file mode 100644 index 0000000..2156f99 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/StatusSnapshot.cs @@ -0,0 +1,8 @@ +namespace FrostFS.SDK.Client; + +public class StatusSnapshot() +{ + public ulong AllTime { get; internal set; } + + public ulong AllRequests { get; internal set; } +} diff --git a/src/FrostFS.SDK.Client/Pool/WorkList.cs b/src/FrostFS.SDK.Client/Pool/WorkList.cs new file mode 100644 index 0000000..7796e86 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/WorkList.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using System.Linq; + +namespace FrostFS.SDK.Client; + +internal sealed class WorkList +{ + private readonly List elements = []; + + internal int Length + { + get { return elements.Count; } + } + + internal void Add(int element) + { + elements.Add(element); + } + + internal int Remove() + { + int last = elements.LastOrDefault(); + elements.RemoveAt(elements.Count - 1); + return last; + } +} diff --git a/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs b/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs new file mode 100644 index 0000000..83ac869 --- /dev/null +++ b/src/FrostFS.SDK.Client/Pool/WrapperPrm.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.ObjectModel; +using System.Security.Cryptography; + +using Grpc.Core; +using Grpc.Core.Interceptors; + +using Microsoft.Extensions.Logging; + +namespace FrostFS.SDK.Client; + +[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "")] +public struct WrapperPrm +{ + internal ILogger? Logger { get; set; } + + internal string Address { get; set; } + + internal ECDsa Key { get; set; } + + internal ulong DialTimeout { get; set; } + + internal ulong StreamTimeout { get; set; } + + internal uint ErrorThreshold { get; set; } + + internal Action ResponseInfoCallback { get; set; } + + internal Action PoolRequestInfoCallback { get; set; } + + internal Func GrpcChannelFactory { get; set; } + + internal ulong GracefulCloseOnSwitchTimeout { get; set; } + + internal Action? Callback { get; set; } + + internal Collection? Interceptors { get; set; } +} + diff --git a/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs index ebd60de..8ad35d5 100644 --- a/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ApeManagerServiceProvider.cs @@ -20,6 +20,8 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor { var binary = RuleSerializer.Serialize(args.Chain); + var base64 = Convert.ToBase64String(binary); + AddChainRequest request = new() { Body = new() diff --git a/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs index c0f8b1a..59783cf 100644 --- a/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ContainerServiceProvider.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Security.Cryptography; using System.Threading.Tasks; using FrostFS.Container; @@ -82,7 +83,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService Body = new PutRequest.Types.Body { Container = grpcContainer, - Signature = ClientContext.Key.SignRFC6979(grpcContainer) + Signature = ClientContext.Key.ECDsaKey.SignRFC6979(grpcContainer) } }; @@ -112,8 +113,8 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService { Body = new DeleteRequest.Types.Body { - ContainerId = args.ContainerId.GetContainerID(), - Signature = ClientContext.Key.SignRFC6979(args.ContainerId.GetContainerID().Value) + ContainerId = args.ContainerId.ToMessage(), + Signature = ClientContext.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value) } }; diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index b3c1f50..27d8dfb 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -1,13 +1,11 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using FrostFS.Object; using FrostFS.Refs; -using FrostFS.SDK.Client; using FrostFS.SDK.Client.Interfaces; using FrostFS.SDK.Client.Mappers.GRPC; using FrostFS.Session; @@ -96,7 +94,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.GetContainerID(), + ContainerId = args.ContainerId.ToMessage(), ObjectId = args.ObjectId.ToMessage() } } @@ -124,7 +122,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.GetContainerID(), + ContainerId = args.ContainerId.ToMessage(), ObjectId = args.ObjectId.ToMessage() }, Range = new Object.Range @@ -159,7 +157,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.GetContainerID(), + ContainerId = args.ContainerId.ToMessage(), ObjectId = args.ObjectId.ToMessage() }, Type = ChecksumType.Sha256, @@ -204,7 +202,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Address = new Address { - ContainerId = args.ContainerId.GetContainerID(), + ContainerId = args.ContainerId.ToMessage(), ObjectId = args.ObjectId.ToMessage() } } @@ -231,7 +229,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Body = new SearchRequest.Types.Body { - ContainerId = args.ContainerId.GetContainerID(), + ContainerId = args.ContainerId.ToMessage(), Version = 1 // TODO: clarify this param } }; @@ -295,66 +293,83 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl internal async Task PatchObjectAsync(PrmObjectPatch args, CallContext ctx) { var chunkSize = args.MaxChunkLength; - + var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken); + var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false); var address = new Address { ObjectId = args.Address.ObjectId, ContainerId = args.Address.ContainerId }; + var request = new PatchRequest + { + Body = new() + { + Address = address, + } + }; + var protoToken = sessionToken.CreateObjectTokenContext( + address, + ObjectSessionContext.Types.Verb.Patch, + ClientContext.Key); - if (args.Payload != null && args.Payload.Length > 0) + request.AddMetaHeader(args.XHeaders, protoToken); + + if (args.NewAttributes is { Length: > 0 }) + { + request.Body.ReplaceAttributes = args.ReplaceAttributes; + foreach (var attr in args.NewAttributes) + { + request.Body.NewAttributes.Add(attr.ToMessage()); + } + } + + if (args.NewSplitHeader != null) + { + request.Body.NewSplitHeader = ObjectTools.CreateSplit(args.NewSplitHeader, ClientContext.Key, + ClientContext.Owner, ClientContext.Version); + if (request.Body.NewSplitHeader.Parent != null) + { + args.NewSplitHeader.Parent = request.Body.NewSplitHeader.Parent.ToModel(); + } + } + request.Sign(ClientContext.Key); + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); + + if (args.Payload != null) { byte[]? chunkBuffer = null; try { + // common chunkBuffer = ArrayPool.Shared.Rent(chunkSize); - - bool isFirstChunk = true; ulong currentPos = args.Range.Offset; while (true) { - var bytesCount = await args.Payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false); + var bytesCount = await args.Payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken) + .ConfigureAwait(false); if (bytesCount == 0) { break; } - PatchRequest request; - - if (isFirstChunk) + request = new PatchRequest() { - request = await CreateFirstRequest(args, ctx, address).ConfigureAwait(false); - - request.Body.Patch = new PatchRequest.Types.Body.Types.Patch + Body = new() { - Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), - SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount } - }; - - isFirstChunk = false; - } - else - { - request = new PatchRequest() - { - Body = new() + Address = address, + Patch = new PatchRequest.Types.Body.Types.Patch { - Address = address, - Patch = new PatchRequest.Types.Body.Types.Patch - { - Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), - SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount } - } + Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), + SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } } - }; - - request.AddMetaHeader(args.XHeaders); - } + } + }; + request.AddMetaHeader(args.XHeaders); request.Sign(ClientContext.Key); await call.RequestStream.WriteAsync(request).ConfigureAwait(false); @@ -370,14 +385,6 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } } } - else if (args.NewAttributes != null && args.NewAttributes.Length > 0) - { - PatchRequest request = await CreateFirstRequest(args, ctx, address).ConfigureAwait(false); - - request.Sign(ClientContext.Key); - - await call.RequestStream.WriteAsync(request).ConfigureAwait(false); - } await call.RequestStream.CompleteAsync().ConfigureAwait(false); var response = await call.ResponseAsync.ConfigureAwait(false); @@ -385,289 +392,95 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl Verifier.CheckResponse(response); return response.Body.ObjectId.ToModel(); - - async Task CreateFirstRequest(PrmObjectPatch args, CallContext ctx, Address address) - { - var body = new PatchRequest.Types.Body() { Address = address }; - - if (args.NewAttributes != null) - { - body.ReplaceAttributes = args.ReplaceAttributes; - - foreach (var attr in args.NewAttributes!) - { - body.NewAttributes.Add(attr.ToMessage()); - } - } - - var request = new PatchRequest() { Body = body }; - - var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false); - - var protoToken = sessionToken.CreateObjectTokenContext( - address, - ObjectSessionContext.Types.Verb.Patch, - ClientContext.Key); - - request.AddMetaHeader(args.XHeaders, protoToken); - return request; - } } internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { - if (args.Payload == null) - throw new ArgumentException(nameof(args.Payload)); + var stream = args.Payload!; + var header = args.Header!; - if (args.Header == null) - throw new ArgumentException(nameof(args.Header)); + if (header.PayloadLength > 0) + args.PutObjectContext.FullLength = header.PayloadLength; + else if (stream.CanSeek) + args.PutObjectContext.FullLength = (ulong)stream.Length; + else + throw new ArgumentException("The stream does not have a length and payload length is not defined"); + + if (args.PutObjectContext.FullLength == 0) + throw new ArgumentException("The stream has zero length"); var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - int partSize = (int)networkSettings.MaxObjectSize; + var partSize = (int)networkSettings.MaxObjectSize; - int chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize; + var restBytes = args.PutObjectContext.FullLength; - ulong fullLength; + var objectSize = (int)Math.Min((ulong)partSize, restBytes); - // Information about the uploaded parts. - var progressInfo = args.Progress; // - var offset = 0L; + // define collection capacity + var objectsCount = (int)(restBytes / (ulong)objectSize) + ((restBytes % (ulong)objectSize) > 0 ? 1 : 0); - if (progressInfo != null && progressInfo.GetParts().Count > 0) - { - if (!args.Payload.CanSeek) - { - throw new FrostFsException("Cannot resume client cut upload for this stream. Seek must be supported."); - } - - var lastPart = progressInfo.GetLast(); - args.Payload.Position = lastPart.Offset + lastPart.Length; - fullLength = (ulong)(args.Payload.Length - args.Payload.Position); - offset = args.Payload.Position; - } - else - { - if (args.Header.PayloadLength > 0) - fullLength = args.Header.PayloadLength; - else if (args.Payload.CanSeek) - fullLength = (ulong)args.Payload.Length; - else - throw new ArgumentException("The stream does not have a length and payload length is not defined"); - } - - //define collection capacity - var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0; - var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; - - progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); - - var remain = fullLength; - - byte[]? buffer = null; - bool isRentBuffer = false; - - try - { - if (args.CustomBuffer != null) - { - if (args.CustomBuffer.Length < chunkSize) - throw new ArgumentException($"Buffer size is too small. At least {chunkSize} required"); - - buffer = args.CustomBuffer; - } - else - { - buffer = ArrayPool.Shared.Rent(chunkSize); - isRentBuffer = true; - } - - FrostFsObjectId? resultObjectId = null; - FrostFsObjectHeader? parentHeader = null; - - while (remain > 0) - { - var bytesToWrite = Math.Min((ulong)partSize, remain); - var isLastPart = remain <= (ulong)partSize; - - // When the last part of the object is uploaded, all metadata for the object must be added - if (isLastPart && objectsCount > 1) - { - parentHeader = new FrostFsObjectHeader(args.Header.ContainerId, FrostFsObjectType.Regular) - { - Attributes = args.Header.Attributes, - PayloadLength = fullLength - }; - } - - // Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter - var header = objectsCount == 1 ? args.Header : new FrostFsObjectHeader( - args.Header.ContainerId, - FrostFsObjectType.Regular, - [], - new FrostFsSplit(progressInfo.SplitId, progressInfo.GetLast().ObjectId, parentHeader: parentHeader)); - - var prm = new PrmObjectPut(header); - using var stream = await PutStreamObjectAsync(prm, ctx).ConfigureAwait(false); - var uploaded = 0; - - // If an error occurs while uploading a part of the object, there is no need to re-upload the parts - // that were successfully uploaded before. It is sufficient to re-upload only the failed part - - var thisPartRest = (int)Math.Min((ulong)partSize, remain); - while (thisPartRest > 0) - { - var nextChunkSize = Math.Min(thisPartRest, chunkSize); - var size = await args.Payload.ReadAsync(buffer, 0, nextChunkSize).ConfigureAwait(false); - - if (size == 0) - break; - - await stream.WriteAsync(buffer.AsMemory(0, size)).ConfigureAwait(false); - uploaded += size; - thisPartRest -= size; - } - - var objectId = await stream.CompleteAsync().ConfigureAwait(false); - var part = new ObjectPartInfo(offset, uploaded, objectId); - offset += uploaded; - progressInfo.AddPart(part); - - remain -= bytesToWrite; - - if (isLastPart) - { - if (objectsCount == 1) - { - return progressInfo.GetPart(0).ObjectId; - } - - if (parentHeader == null) continue; - - // Once all parts of the object are uploaded, they must be linked into a single entity - var linkObject = new FrostFsLinkObject(header.ContainerId, progressInfo.SplitId, parentHeader, - [.. progressInfo.GetParts().Select(p => p.ObjectId)]); - - await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false); - - // Retrieve the ID of the linked object - resultObjectId = FrostFsObjectId.FromHash(prm.Header!.GetHeader().Split!.Parent.Value.Span); - return resultObjectId; - } - } - - throw new FrostFsException("Unexpected error: cannot send object"); - } - finally - { - if (isRentBuffer && buffer != null) - { - ArrayPool.Shared.Return(buffer); - } - } - } - - internal async Task PutClientCutSingleObjectAsync(PrmObjectClientCutPut args, CallContext ctx) - { - if (args.Payload == null) - throw new ArgumentException(nameof(args.Payload)); - - if (args.Header == null) - throw new ArgumentException(nameof(args.Header)); - - var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false); - int partSize = (int)networkSettings.MaxObjectSize; - - int chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize; - - ulong fullLength; - - // Information about the uploaded parts. - var progressInfo = args.Progress; // - var offset = 0L; - - if (progressInfo != null && progressInfo.GetParts().Count > 0) - { - if (!args.Payload.CanSeek) - { - throw new FrostFsException("Cannot resume client cut upload for this stream. Seek must be supported."); - } - - var lastPart = progressInfo.GetLast(); - args.Payload.Position = lastPart.Offset + lastPart.Length; - fullLength = (ulong)(args.Payload.Length - args.Payload.Position); - offset = args.Payload.Position; - } - else - { - if (args.Header.PayloadLength > 0) - fullLength = args.Header.PayloadLength; - else if (args.Payload.CanSeek) - fullLength = (ulong)args.Payload.Length; - else - throw new ArgumentException("The stream does not have a length and payload length is not defined"); - } - - //define collection capacity - var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0; - var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0; - - progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount); - - // if the object fits one part, it can be loaded as non-complex object, but if it is not upload resuming - if (objectsCount == 1 && progressInfo.GetLast().Length == 0) + // if the object fits one part, it can be loaded as non-complex object + if (objectsCount == 1) { args.PutObjectContext.MaxObjectSizeCache = partSize; - args.PutObjectContext.FullLength = fullLength; var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false); return singlePartResult.ObjectId; } - var remain = fullLength; + List parts = new(objectsCount); + SplitId splitId = new(); + + // keep attributes for the large object + var attributes = args.Header!.Attributes.ToArray(); + header.Attributes = null; + + var remain = args.PutObjectContext.FullLength; + + FrostFsObjectHeader? parentHeader = null; + + var lastIndex = objectsCount - 1; + + bool rentBuffer = false; byte[]? buffer = null; - bool isRentBuffer = false; try { - if (args.CustomBuffer != null) + for (int i = 0; i < objectsCount; i++) { - if (args.CustomBuffer.Length < partSize) + if (args.CustomBuffer != null) { - throw new ArgumentException($"Buffer size is too small. A buffer with capacity {partSize} is required"); + if (args.CustomBuffer.Length < partSize) + { + throw new ArgumentException($"Buffer size is too small. A buffer with capacity {partSize} is required"); + } + + buffer = args.CustomBuffer; + } + else + { + buffer = ArrayPool.Shared.Rent(partSize); + rentBuffer = true; } - buffer = args.CustomBuffer; - } - else - { - buffer = ArrayPool.Shared.Rent(partSize); - isRentBuffer = true; - } - - FrostFsObjectHeader? parentHeader = null; - - for (int i = 0; i < objectsCount;) - { - i++; var bytesToWrite = Math.Min((ulong)partSize, remain); - var size = await args.Payload.ReadAsync(buffer, 0, (int)bytesToWrite).ConfigureAwait(false); + var size = await stream.ReadAsync(buffer, 0, (int)bytesToWrite).ConfigureAwait(false); - if (i == objectsCount) + if (i == lastIndex) { - parentHeader = new FrostFsObjectHeader(args.Header.ContainerId, FrostFsObjectType.Regular) + parentHeader = new FrostFsObjectHeader(header.ContainerId, FrostFsObjectType.Regular, attributes) { - PayloadLength = args.PutObjectContext.FullLength, - Attributes = args.Header.Attributes + PayloadLength = args.PutObjectContext.FullLength }; } // Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter var partHeader = new FrostFsObjectHeader( - args.Header.ContainerId, + header.ContainerId, FrostFsObjectType.Regular, [], - new FrostFsSplit(progressInfo.SplitId, progressInfo.GetLast().ObjectId, + new FrostFsSplit(splitId, parts.LastOrDefault(), parentHeader: parentHeader)) { PayloadLength = (ulong)size @@ -680,20 +493,15 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var prm = new PrmSingleObjectPut(obj); - var objectId = await PutSingleObjectAsync(prm, ctx).ConfigureAwait(false); + var objId = await PutSingleObjectAsync(prm, ctx).ConfigureAwait(false); - var part = new ObjectPartInfo(offset, size, objectId); - progressInfo.AddPart(part); + parts.Add(objId); - offset += size; - - if (i < objectsCount) - { + if (i < lastIndex) continue; - } - + // Once all parts of the object are uploaded, they must be linked into a single entity - var linkObject = new FrostFsLinkObject(args.Header.ContainerId, progressInfo.SplitId, parentHeader!, [.. progressInfo.GetParts().Select(p => p.ObjectId)]); + var linkObject = new FrostFsLinkObject(header.ContainerId, splitId, parentHeader!, parts); _ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false); @@ -705,7 +513,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } finally { - if (isRentBuffer && buffer != null) + if (rentBuffer && buffer != null) { ArrayPool.Shared.Return(buffer); } diff --git a/src/FrostFS.SDK.Client/Tools/ClientContext.cs b/src/FrostFS.SDK.Client/Tools/ClientContext.cs index 104cb3d..e2522c9 100644 --- a/src/FrostFS.SDK.Client/Tools/ClientContext.cs +++ b/src/FrostFS.SDK.Client/Tools/ClientContext.cs @@ -39,7 +39,7 @@ public class ClientContext(FrostFSClient client, ClientKey key, FrostFsOwner own { if (sessionKey == null && Key != null && Address != null) { - sessionKey = $"{Address}{Key}"; + sessionKey = Pool.FormCacheKey(Address, Key.PublicKey); } return sessionKey; diff --git a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs index c8c5caf..701ba05 100644 --- a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs +++ b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs @@ -1,6 +1,6 @@ using System; using System.Linq; -using System.Security.Cryptography; + using FrostFS.Object; using FrostFS.Refs; using FrostFS.SDK.Client.Mappers.GRPC; @@ -44,11 +44,7 @@ public static class ObjectTools if (header.Split != null) SetSplitValues(grpcHeader, header.Split, owner, version, key); - using var sha256 = SHA256.Create(); - using HashStream stream = new(sha256); - grpcHeader.WriteTo(stream); - - return new FrostFsObjectId(Base58.Encode(stream.Hash())); + return new ObjectID { Value = grpcHeader.Sha256() }.ToModel(); } internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx) @@ -79,7 +75,7 @@ public static class ObjectTools var obj = new Object.Object { Header = grpcHeader, - ObjectId = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcHeader.Sha256()) }, + ObjectId = new ObjectID { Value = grpcHeader.Sha256() }, Payload = UnsafeByteOperations.UnsafeWrap(@object.SingleObjectPayload) }; @@ -121,9 +117,9 @@ public static class ObjectTools if (split.ParentHeader is not null) { - var grpcParentHeader = CreateHeader(split.ParentHeader, DataHasher.Sha256([]), owner, version); + var grpcParentHeader = CreateHeader(split.ParentHeader, Array.Empty().Sha256(), owner, version); - grpcHeader.Split.Parent = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcParentHeader.Sha256()) }; + grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() }; grpcHeader.Split.ParentHeader = grpcParentHeader; grpcHeader.Split.ParentSignature = new Signature { @@ -135,6 +131,26 @@ public static class ObjectTools grpcHeader.Split.Previous = split.Previous?.ToMessage(); } + internal static Header.Types.Split CreateSplit(FrostFsSplit split, ClientKey key, FrostFsOwner owner, FrostFsVersion version) + { + if (split.ParentHeader == null) + { + return split.GetSplit(); + } + split.ParentHeader.PayloadCheckSum ??= Array.Empty().Sha256(); + var grpcParentHeader = CreateHeader(split.ParentHeader, split.ParentHeader.PayloadCheckSum, owner, version); + + var res = split.GetSplit(); + res.ParentHeader = grpcParentHeader; + res.Parent ??= new ObjectID { Value = grpcParentHeader.Sha256() }; + res.ParentSignature ??= new Signature + { + Key = key.PublicKeyProto, + Sign = key.ECDsaKey.SignData(res.Parent.ToByteArray()), + }; + return res; + } + internal static Header CreateHeader( FrostFsObjectHeader header, ReadOnlyMemory payloadChecksum, @@ -151,12 +167,21 @@ public static class ObjectTools return grpcHeader; } + internal static Checksum Sha256Checksum(byte[] data) + { + return new Checksum + { + Type = ChecksumType.Sha256, + Sum = ByteString.CopyFrom(data.Sha256()) + }; + } + internal static Checksum Sha256Checksum(ReadOnlyMemory data) { return new Checksum { Type = ChecksumType.Sha256, - Sum = UnsafeByteOperations.UnsafeWrap(DataHasher.Sha256(data)) + Sum = ByteString.CopyFrom(data.Sha256()) }; } diff --git a/src/FrostFS.SDK.Client/Tools/RequestSigner.cs b/src/FrostFS.SDK.Client/Tools/RequestSigner.cs index 8df1b7c..e2aceef 100644 --- a/src/FrostFS.SDK.Client/Tools/RequestSigner.cs +++ b/src/FrostFS.SDK.Client/Tools/RequestSigner.cs @@ -33,7 +33,6 @@ public static class RequestSigner var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N); var privateKey = new ECPrivateKeyParameters(new BigInteger(1, key.PrivateKey()), ecParameters); var signer = new ECDsaSigner(new HMacDsaKCalculator(digest)); - var hash = new byte[digest.GetDigestSize()]; digest.BlockUpdate(data, 0, data.Length); @@ -55,21 +54,21 @@ public static class RequestSigner return ByteString.CopyFrom(signature); } - internal static SignatureRFC6979 SignRFC6979(this ClientKey key, IMessage message) + internal static SignatureRFC6979 SignRFC6979(this ECDsa key, IMessage message) { return new SignatureRFC6979 { - Key = key.PublicKeyProto, - Sign = key.ECDsaKey.SignRFC6979(message.ToByteArray()), + Key = ByteString.CopyFrom(key.PublicKey()), + Sign = key.SignRFC6979(message.ToByteArray()), }; } - internal static SignatureRFC6979 SignRFC6979(this ClientKey key, ByteString data) + internal static SignatureRFC6979 SignRFC6979(this ECDsa key, ByteString data) { return new SignatureRFC6979 { - Key = key.PublicKeyProto, - Sign = key.ECDsaKey.SignRFC6979(data.ToByteArray()), + Key = ByteString.CopyFrom(key.PublicKey()), + Sign = key.SignRFC6979(data.ToByteArray()), }; } @@ -83,11 +82,11 @@ public static class RequestSigner Span result = stackalloc byte[65]; result[0] = 0x04; - key.SignHash(DataHasher.Sha512(data)).AsSpan().CopyTo(result.Slice(1)); + key.SignHash(data.Sha512()).AsSpan().CopyTo(result.Slice(1)); return ByteString.CopyFrom(result); } - + public static ByteString SignDataByHash(this ECDsa key, byte[] hash) { if (key is null) @@ -113,9 +112,8 @@ public static class RequestSigner Sign = key.ECDsaKey.SignData(ReadOnlyMemory.Empty), }; } - - using var sha512 = SHA512.Create(); - using HashStream stream = new(sha512); + + using HashStream stream = new(); data.WriteTo(stream); var sig = new Signature diff --git a/src/FrostFS.SDK.Client/Tools/Verifier.cs b/src/FrostFS.SDK.Client/Tools/Verifier.cs index 90eb3ce..c110431 100644 --- a/src/FrostFS.SDK.Client/Tools/Verifier.cs +++ b/src/FrostFS.SDK.Client/Tools/Verifier.cs @@ -9,13 +9,61 @@ using FrostFS.Session; using Google.Protobuf; +using Org.BouncyCastle.Asn1.Sec; +using Org.BouncyCastle.Crypto.Digests; +using Org.BouncyCastle.Crypto.Parameters; +using Org.BouncyCastle.Crypto.Signers; +using Org.BouncyCastle.Math; + namespace FrostFS.SDK.Client; public static class Verifier { public const int RFC6979SignatureSize = 64; - public static bool VerifyData(this ECDsa key, IMessage data, ByteString sig) + private static BigInteger[] DecodeSignature(byte[] sig) + { + if (sig.Length != RFC6979SignatureSize) + throw new FormatException($"Wrong signature size, expect={RFC6979SignatureSize}, actual={sig.Length}"); + + var rs = new BigInteger[2]; + rs[0] = new BigInteger(1, sig.AsSpan(0, 32).ToArray()); + rs[1] = new BigInteger(1, sig.AsSpan(32).ToArray()); + + return rs; + } + + public static bool VerifyRFC6979(this byte[] publicKey, byte[] data, byte[] sig) + { + if (publicKey is null || data is null || sig is null) + return false; + + var rs = DecodeSignature(sig); + var digest = new Sha256Digest(); + var signer = new ECDsaSigner(new HMacDsaKCalculator(digest)); + var secp256R1 = SecNamedCurves.GetByName("secp256r1"); + var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N); + var bcPublicKey = new ECPublicKeyParameters(secp256R1.Curve.DecodePoint(publicKey), ecParameters); + var hash = new byte[digest.GetDigestSize()]; + + digest.BlockUpdate(data, 0, data.Length); + digest.DoFinal(hash, 0); + signer.Init(false, bcPublicKey); + + return signer.VerifySignature(hash, rs[0], rs[1]); + } + + public static bool VerifyRFC6979(this SignatureRFC6979 signature, IMessage message) + { + if (signature is null) + { + throw new ArgumentNullException(nameof(signature)); + } + + return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray()); + } + + public static bool VerifyData(this ECDsa key, ReadOnlyMemory data, byte[] sig) { if (key is null) throw new ArgumentNullException(nameof(key)); @@ -23,18 +71,7 @@ public static class Verifier if (sig is null) throw new ArgumentNullException(nameof(sig)); - var signature = sig.Span.Slice(1).ToArray(); - using var sha = SHA512.Create(); - - if (data is null) - { - return key.VerifyHash(DataHasher.Sha512(new Span([])), signature); - } - - using var stream = new HashStream(sha); - data.WriteTo(stream); - - return key.VerifyHash(stream.Hash(), signature); + return key.VerifyHash(data.Sha512(), sig.AsSpan(1).ToArray()); } public static bool VerifyMessagePart(this Signature sig, IMessage data) @@ -43,8 +80,9 @@ public static class Verifier return false; using var key = sig.Key.ToByteArray().LoadPublicKey(); + var data2Verify = data is null ? [] : data.ToByteArray(); - return key.VerifyData(data, sig.Sign); + return key.VerifyData(data2Verify, sig.Sign.ToByteArray()); } internal static bool VerifyMatryoskaLevel(IMessage body, IMetaHeader meta, IVerificationHeader verification) diff --git a/src/FrostFS.SDK.Client/Tools/WalletTools.cs b/src/FrostFS.SDK.Client/Tools/WalletTools.cs deleted file mode 100644 index 45532a7..0000000 --- a/src/FrostFS.SDK.Client/Tools/WalletTools.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Text.Json; -using System; -using FrostFS.SDK.Client.Wallets; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Client; - -public static class WalletTools -{ - public static string GetWifFromWallet(string walletJsonText, byte[] password, int accountIndex = 0) - { - var wallet = JsonSerializer.Deserialize(walletJsonText) ?? throw new ArgumentException("Wrong wallet format"); - - if (wallet.Accounts == null || wallet.Accounts.Length < accountIndex + 1) - { - throw new ArgumentException("Wrong wallet content"); - } - - var encryptedKey = wallet.Accounts[accountIndex].Key; - - if (string.IsNullOrEmpty(encryptedKey)) - { - throw new ArgumentException("Cannot get encrypted WIF"); - } - - var privateKey = CryptoWallet.GetKeyFromEncodedWif(password, encryptedKey!); - var wif = privateKey.GetWIFFromPrivateKey(); - - return wif; - } -} diff --git a/src/FrostFS.SDK.Client/Wallets/Account.cs b/src/FrostFS.SDK.Client/Wallets/Account.cs deleted file mode 100644 index 1d5b3eb..0000000 --- a/src/FrostFS.SDK.Client/Wallets/Account.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FrostFS.SDK.Client.Wallets; - -public class Account -{ - [JsonPropertyName("address")] - public string? Address { get; set; } - - [JsonPropertyName("key")] - public string? Key { get; set; } - - [JsonPropertyName("label")] - public string? Label { get; set; } - - [JsonPropertyName("contract")] - public Contract? Contract { get; set; } - - [JsonPropertyName("lock")] - public bool Lock { get; set; } - - [JsonPropertyName("isDefault")] - public bool IsDefault { get; set; } -} diff --git a/src/FrostFS.SDK.Client/Wallets/Contract.cs b/src/FrostFS.SDK.Client/Wallets/Contract.cs deleted file mode 100644 index b15d21d..0000000 --- a/src/FrostFS.SDK.Client/Wallets/Contract.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FrostFS.SDK.Client.Wallets; - -public class Contract -{ - [JsonPropertyName("script")] - public string? Script { get; set; } - - [JsonPropertyName("parameters")] - public Parameter[]? Parameters { get; set; } - - [JsonPropertyName("deployed")] - public bool Deployed { get; set; } -} diff --git a/src/FrostFS.SDK.Client/Wallets/Extra.cs b/src/FrostFS.SDK.Client/Wallets/Extra.cs deleted file mode 100644 index bff8b8d..0000000 --- a/src/FrostFS.SDK.Client/Wallets/Extra.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace FrostFS.SDK.Client.Wallets; - -public class Extra -{ - public string? Tokens { get; set; } -} diff --git a/src/FrostFS.SDK.Client/Wallets/Parameter.cs b/src/FrostFS.SDK.Client/Wallets/Parameter.cs deleted file mode 100644 index c831e36..0000000 --- a/src/FrostFS.SDK.Client/Wallets/Parameter.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FrostFS.SDK.Client.Wallets; - -public class Parameter -{ - [JsonPropertyName("name")] - public string? Name { get; set; } - - [JsonPropertyName("type")] - public string? Type { get; set; } -} diff --git a/src/FrostFS.SDK.Client/Wallets/ScryptValue.cs b/src/FrostFS.SDK.Client/Wallets/ScryptValue.cs deleted file mode 100644 index 2548b32..0000000 --- a/src/FrostFS.SDK.Client/Wallets/ScryptValue.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FrostFS.SDK.Client.Wallets; - -public class ScryptValue -{ - [JsonPropertyName("n")] - public int N { get; set; } - - [JsonPropertyName("r")] - public int R { get; set; } - - [JsonPropertyName("p")] - public int P { get; set; } -} diff --git a/src/FrostFS.SDK.Client/Wallets/Wallet.cs b/src/FrostFS.SDK.Client/Wallets/Wallet.cs deleted file mode 100644 index e1439e0..0000000 --- a/src/FrostFS.SDK.Client/Wallets/Wallet.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FrostFS.SDK.Client.Wallets; - -public class Wallet -{ - [JsonPropertyName("version")] - public string? Version { get; set; } - - [JsonPropertyName("accounts")] - public Account[]? Accounts { get; set; } - - [JsonPropertyName("scrypt")] - public ScryptValue? Scrypt { get; set; } - - [JsonPropertyName("extra")] - public Extra? Extra { get; set; } -} diff --git a/src/FrostFS.SDK.Cryptography/ArrayHelper.cs b/src/FrostFS.SDK.Cryptography/ArrayHelper.cs index c88134d..00c356d 100644 --- a/src/FrostFS.SDK.Cryptography/ArrayHelper.cs +++ b/src/FrostFS.SDK.Cryptography/ArrayHelper.cs @@ -21,18 +21,4 @@ internal static class ArrayHelper return dst; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void GetRevertedArray(ReadOnlySpan source, byte[] data) - { - if (source.Length != 0) - { - int i = 0; - int j = source.Length - 1; - while (i < source.Length) - { - data[i++] = source[j--]; - } - } - } } \ No newline at end of file diff --git a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs index 64bc8e9..c03d604 100644 --- a/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Cryptography/AssemblyInfo.cs @@ -1,7 +1,8 @@ using System.Reflection; -[assembly: AssemblyCompany("TrueCloudLab")] -[assembly: AssemblyFileVersion("1.0.7.0")] +[assembly: AssemblyCompany("FrostFS.SDK.Cryptography")] +[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Cryptography")] [assembly: AssemblyTitle("FrostFS.SDK.Cryptography")] -[assembly: AssemblyVersion("1.0.7.0")] +[assembly: AssemblyVersion("1.0.3.0")] diff --git a/src/FrostFS.SDK.Cryptography/Base58.cs b/src/FrostFS.SDK.Cryptography/Base58.cs index 4dc6187..636eb0a 100644 --- a/src/FrostFS.SDK.Cryptography/Base58.cs +++ b/src/FrostFS.SDK.Cryptography/Base58.cs @@ -19,20 +19,19 @@ public static class Base58 if (buffer.Length < 4) throw new FormatException(); - var check = buffer.AsSpan(0, buffer.Length - 4); - byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(check).AsSpan()); + var check = buffer.AsSpan(0, buffer.Length - 4).ToArray(); + byte[] checksum = check.Sha256().Sha256(); if (!buffer.AsSpan(buffer.Length - 4).SequenceEqual(checksum.AsSpan(0, 4))) throw new FormatException(); - var result = check.ToArray(); Array.Clear(buffer, 0, buffer.Length); - return result; + return check; } - public static string Base58CheckEncode(this Span data) + public static string Base58CheckEncode(this ReadOnlySpan data) { - byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan()); + byte[] checksum = data.ToArray().Sha256().Sha256(); Span buffer = stackalloc byte[data.Length + 4]; data.CopyTo(buffer); @@ -60,9 +59,10 @@ public static class Base58 } int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count(); + var leadingZeros = new byte[leadingZeroCount]; if (bi.IsZero) - return new byte[leadingZeroCount]; + return leadingZeros; var bytesBigEndian = bi.ToByteArray().Reverse().ToArray(); @@ -73,26 +73,12 @@ public static class Base58 var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray(); - var result = new byte[leadingZeroCount + bytesBigEndian.Length - firstNonZeroIndex]; - - int p = 0; - while (p < leadingZeroCount) - result[p++] = 0; - - for (int j = firstNonZeroIndex; j < bytesBigEndian.Length; j++) - result[p++] = bytesBigEndian[j]; - - return result; + return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros); } public static string Encode(ReadOnlySpan input) { - var data = new byte[input.Length + 1]; - - ArrayHelper.GetRevertedArray(input, data); - - data[input.Length] = 0; - + var data = input.ToArray().Reverse().Concat(new byte[] { 0 }).ToArray(); BigInteger value = new(data); // Encode BigInteger to Base58 string diff --git a/src/FrostFS.SDK.Cryptography/DataHasher.cs b/src/FrostFS.SDK.Cryptography/DataHasher.cs deleted file mode 100644 index fceeeaa..0000000 --- a/src/FrostFS.SDK.Cryptography/DataHasher.cs +++ /dev/null @@ -1,121 +0,0 @@ -using System; -using System.Buffers; -using System.Security.Cryptography; - -namespace FrostFS.SDK.Cryptography; - -public static class DataHasher -{ - private const int LargeBlockSize = 1024 * 1024; - private const int SmallBlockSize = 256; - - public static byte[] Hash(this ReadOnlyMemory bytes, HashAlgorithm algorithm) - { - if (algorithm is null) - { - throw new ArgumentNullException(nameof(algorithm)); - } - - if (bytes.Length == 0) - { - return algorithm.ComputeHash([]); - } - - int rest, pos = 0; - - var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize; - - byte[] buffer = ArrayPool.Shared.Rent(blockSize); - - try - { - while ((rest = bytes.Length - pos) > 0) - { - var size = Math.Min(rest, blockSize); - - bytes.Slice(pos, size).CopyTo(buffer); - - algorithm.TransformBlock(buffer, 0, size, buffer, 0); - - pos += size; - } - - algorithm.TransformFinalBlock([], 0, 0); - return algorithm.Hash; - } - finally - { - if (buffer != null) - { - ArrayPool.Shared.Return(buffer); - } - } - } - - public static byte[] Hash(this ReadOnlySpan bytes, HashAlgorithm algorithm) - { - if (algorithm is null) - { - throw new ArgumentNullException(nameof(algorithm)); - } - - if (bytes.Length == 0) - { - return algorithm.ComputeHash([]); - } - - int rest, pos = 0; - - var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize; - - byte[] buffer = ArrayPool.Shared.Rent(blockSize); - - try - { - while ((rest = bytes.Length - pos) > 0) - { - var size = Math.Min(rest, blockSize); - - bytes.Slice(pos, size).CopyTo(buffer); - - algorithm.TransformBlock(buffer, 0, size, buffer, 0); - - pos += size; - } - - algorithm.TransformFinalBlock([], 0, 0); - return algorithm.Hash; - } - finally - { - if (buffer != null) - { - ArrayPool.Shared.Return(buffer); - } - } - } - - public static byte[] Sha256(ReadOnlyMemory value) - { - using SHA256 sha = SHA256.Create(); - return Hash(value, sha); - } - - public static byte[] Sha256(ReadOnlySpan value) - { - using SHA256 sha = SHA256.Create(); - return Hash(value, sha); - } - - public static byte[] Sha512(ReadOnlyMemory value) - { - using SHA512 sha = SHA512.Create(); - return Hash(value, sha); - } - - public static byte[] Sha512(ReadOnlySpan value) - { - using SHA512 sha = SHA512.Create(); - return Hash(value, sha); - } -} diff --git a/src/FrostFS.SDK.Cryptography/Extentions.cs b/src/FrostFS.SDK.Cryptography/Extentions.cs index 5f4fdd9..40ad6f0 100644 --- a/src/FrostFS.SDK.Cryptography/Extentions.cs +++ b/src/FrostFS.SDK.Cryptography/Extentions.cs @@ -1,9 +1,20 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using CommunityToolkit.HighPerformance; using Org.BouncyCastle.Crypto.Digests; namespace FrostFS.SDK.Cryptography; public static class Extentions { + private static readonly SHA256 _sha256 = SHA256.Create(); + private static SpinLock _spinlockSha256; + + private static readonly SHA512 _sha512 = SHA512.Create(); + private static SpinLock _spinlockSha512; + internal static byte[] RIPEMD160(this byte[] value) { var hash = new byte[20]; @@ -13,4 +24,72 @@ public static class Extentions digest.DoFinal(hash, 0); return hash; } + + public static byte[] Sha256(this byte[] value) + { + bool lockTaken = false; + try + { + _spinlockSha256.Enter(ref lockTaken); + + return _sha256.ComputeHash(value); + } + finally + { + if (lockTaken) + { + _spinlockSha256.Exit(false); + } + } + } + + public static byte[] Sha256(this ReadOnlyMemory value) + { + bool lockTaken = false; + try + { + _spinlockSha256.Enter(ref lockTaken); + + return _sha256.ComputeHash(value.AsStream()); + } + finally + { + if (lockTaken) + { + _spinlockSha256.Exit(false); + } + } + } + + public static byte[] Sha512(this ReadOnlyMemory value) + { + bool lockTaken = false; + try + { + _spinlockSha512.Enter(ref lockTaken); + + return _sha512.ComputeHash(value.AsStream()); + } + finally + { + if (lockTaken) + _spinlockSha512.Exit(false); + } + } + + public static byte[] Sha512(this Stream stream) + { + bool lockTaken = false; + try + { + _spinlockSha512.Enter(ref lockTaken); + + return _sha512.ComputeHash(stream); + } + finally + { + if (lockTaken) + _spinlockSha512.Exit(false); + } + } } diff --git a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj index faeabb1..e4c7824 100644 --- a/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj +++ b/src/FrostFS.SDK.Cryptography/FrostFS.SDK.Cryptography.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Cryptography - 1.0.7 + 1.0.3 Cryptography tools for C# SDK @@ -26,18 +26,16 @@ false - True - .\\..\\..\\keyfile.snk + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/src/FrostFS.SDK.Cryptography/HashStream.cs b/src/FrostFS.SDK.Cryptography/HashStream.cs index 2893f26..d2ede1a 100644 --- a/src/FrostFS.SDK.Cryptography/HashStream.cs +++ b/src/FrostFS.SDK.Cryptography/HashStream.cs @@ -3,11 +3,11 @@ using System.Security.Cryptography; namespace FrostFS.SDK.Cryptography; -public sealed class HashStream(HashAlgorithm algorithm) : Stream +public sealed class HashStream() : Stream { private long position; - private readonly HashAlgorithm _hash = algorithm; + private readonly SHA512 _hash = SHA512.Create(); public override bool CanRead => false; diff --git a/src/FrostFS.SDK.Cryptography/Key.cs b/src/FrostFS.SDK.Cryptography/Key.cs index 9eca104..072f155 100644 --- a/src/FrostFS.SDK.Cryptography/Key.cs +++ b/src/FrostFS.SDK.Cryptography/Key.cs @@ -16,7 +16,7 @@ public static class KeyExtension private const int UncompressedPublicKeyLength = 65; private static readonly uint CheckSigDescriptor = - BinaryPrimitives.ReadUInt32LittleEndian(DataHasher.Sha256(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").AsSpan())); + BinaryPrimitives.ReadUInt32LittleEndian(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").Sha256()); public static byte[] Compress(this byte[] publicKey) { @@ -60,15 +60,10 @@ public static class KeyExtension $"expected length={CompressedPublicKeyLength}, actual={publicKey.Length}" ); - var signDescriptor = BitConverter.GetBytes(CheckSigDescriptor); - - var script = new byte[3 + publicKey.Length + signDescriptor.Length]; - - script[0] = 0x0c; - script[1] = CompressedPublicKeyLength; //PUSHDATA1 33 - Buffer.BlockCopy(publicKey, 0, script, 2, publicKey.Length); - script[publicKey.Length + 2] = 0x41; //SYSCALL - Buffer.BlockCopy(signDescriptor, 0, script, publicKey.Length + 3, signDescriptor.Length); + var script = new byte[] { 0x0c, CompressedPublicKeyLength }; //PUSHDATA1 33 + script = ArrayHelper.Concat(script, publicKey); + script = ArrayHelper.Concat(script, [0x41]); //SYSCALL + script = ArrayHelper.Concat(script, BitConverter.GetBytes(CheckSigDescriptor)); //Neo_Crypto_CheckSig return script; } @@ -79,26 +74,7 @@ public static class KeyExtension throw new ArgumentNullException(nameof(publicKey)); var script = publicKey.CreateSignatureRedeemScript(); - return DataHasher.Sha256(script.AsSpan()).RIPEMD160(); - } - - public static string GetWIFFromPrivateKey(this byte[] privateKey) - { - if (privateKey == null || privateKey.Length != 32) - { - throw new ArgumentNullException(nameof(privateKey)); - } - - Span wifSpan = stackalloc byte[34]; - - wifSpan[0] = 0x80; - wifSpan[33] = 0x01; - - privateKey.AsSpan().CopyTo(wifSpan.Slice(1)); - - var wif = Base58.Base58CheckEncode(wifSpan); - - return wif; + return script.Sha256().RIPEMD160(); } private static string ToAddress(this byte[] scriptHash, byte version) @@ -126,6 +102,11 @@ public static class KeyExtension return privateKey; } + public static string Address(this ECDsa key) + { + return key.PublicKey().PublicKeyToAddress(); + } + public static string PublicKeyToAddress(this byte[] publicKey) { if (publicKey == null) @@ -151,12 +132,10 @@ public static class KeyExtension var pos = 33 - param.Q.X.Length; param.Q.X.CopyTo(pubkey, pos); - - var y = new byte[33]; - ArrayHelper.GetRevertedArray(param.Q.Y, y); - y[32] = 0; - - pubkey[0] = new BigInteger(y).IsEven ? (byte)0x2 : (byte)0x3; + if (new BigInteger(param.Q.Y.Reverse().Concat(new byte[] { 0x0 }).ToArray()).IsEven) + pubkey[0] = 0x2; + else + pubkey[0] = 0x3; return pubkey; } @@ -173,9 +152,7 @@ public static class KeyExtension { var secp256R1 = SecNamedCurves.GetByName("secp256r1"); var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)) - .GetEncoded(false) - .Skip(1) - .ToArray(); + .GetEncoded(false).Skip(1).ToArray(); var key = ECDsa.Create(new ECParameters { diff --git a/src/FrostFS.SDK.Cryptography/WalletExtractor.cs b/src/FrostFS.SDK.Cryptography/WalletExtractor.cs deleted file mode 100644 index 3e940f5..0000000 --- a/src/FrostFS.SDK.Cryptography/WalletExtractor.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System; -using System.Text; -using System.Linq; -using System.Security.Cryptography; -using Org.BouncyCastle.Crypto.Generators; - -namespace FrostFS.SDK.Cryptography; - -public static class CryptoWallet -{ - private static int N = 16384; - private static int R = 8; - private static int P = 8; - - private enum CipherAction - { - Encrypt, - Decrypt - } - - public static byte[] GetKeyFromEncodedWif(byte[] password, string encryptedWIF) - { - var nep2Data = Base58.Base58CheckDecode(encryptedWIF); - - if (nep2Data.Length == 39 && nep2Data[0] == 1 && nep2Data[1] == 66 && nep2Data[2] == 224) - { - var addressHash = nep2Data.AsSpan(3, 4).ToArray(); - var derivedKey = SCrypt.Generate(password, addressHash, N, R, P, 64); - var derivedKeyHalf1 = derivedKey.Take(32).ToArray(); - var derivedKeyHalf2 = derivedKey.Skip(32).ToArray(); - var encrypted = nep2Data.AsSpan(7, 32).ToArray(); - var decrypted = Aes(encrypted, derivedKeyHalf2, CipherAction.Decrypt); - var plainPrivateKey = XorRange(decrypted, derivedKeyHalf1, 0, decrypted.Length); - - return plainPrivateKey; - } - else - { - throw new ArgumentException("Not valid NEP2 prefix."); - } - } - - public static string EncryptWif(string plainText, ECDsa key) - { - var addressHash = GetAddressHash(key); - var derivedKey = SCrypt.Generate(Encoding.UTF8.GetBytes(plainText), addressHash, N, R, P, 64); - var derivedHalf1 = derivedKey.Take(32).ToArray(); - var derivedHalf2 = derivedKey.Skip(32).ToArray(); - var encryptedHalf1 = Aes(XorRange(key.PrivateKey(), derivedHalf1, 0, 16), derivedHalf2, CipherAction.Encrypt); - var encryptedHalf2 = Aes(XorRange(key.PrivateKey(), derivedHalf1, 16, 32), derivedHalf2, CipherAction.Encrypt); - var prefixes = new byte[] { 1, 66, 224 }; - var concatenation = ArrayHelper.Concat([prefixes, addressHash, encryptedHalf1, encryptedHalf2]); - - return Base58.Base58CheckEncode(concatenation); - } - - public static byte[] GetAddressHash(ECDsa key) - { - string address = key.PublicKey().PublicKeyToAddress(); - - using SHA256 sha256 = SHA256.Create(); - byte[] addressHashed = sha256.ComputeHash(Encoding.UTF8.GetBytes(address)); - return [.. addressHashed.Take(4)]; - } - - private static byte[] XorRange(byte[] arr1, byte[] arr2, int from, int length) - { - byte[] result = new byte[length]; - - var j = 0; - for (var i = from; i < length; ++i) - { - result[j++] = (byte)(arr1[i] ^ arr2[i]); - } - - return result; - } - - private static byte[] Aes(byte[] data, byte[] key, CipherAction action) - { - using var aes = System.Security.Cryptography.Aes.Create(); - aes.Key = key; - aes.Mode = CipherMode.ECB; - aes.Padding = PaddingMode.None; - - if (action == CipherAction.Encrypt) - { - return aes.CreateEncryptor().TransformFinalBlock(data, 0, data.Length); - } - - if (action == CipherAction.Decrypt) - { - return aes.CreateDecryptor().TransformFinalBlock(data, 0, data.Length); - } - - throw new ArgumentException("Wrong cippher action", nameof(action)); - } -} diff --git a/src/FrostFS.SDK.Protos/AssemblyInfo.cs b/src/FrostFS.SDK.Protos/AssemblyInfo.cs index d7fab05..11ace79 100644 --- a/src/FrostFS.SDK.Protos/AssemblyInfo.cs +++ b/src/FrostFS.SDK.Protos/AssemblyInfo.cs @@ -1,7 +1,8 @@ using System.Reflection; -[assembly: AssemblyCompany("TrueCloudLab")] -[assembly: AssemblyFileVersion("1.0.7.0")] +[assembly: AssemblyCompany("FrostFS.SDK.Protos")] +[assembly: AssemblyFileVersion("1.0.2.0")] +[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")] [assembly: AssemblyProduct("FrostFS.SDK.Protos")] [assembly: AssemblyTitle("FrostFS.SDK.Protos")] -[assembly: AssemblyVersion("1.0.7.0")] +[assembly: AssemblyVersion("1.0.3.0")] diff --git a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj index 222acd1..5207bbc 100644 --- a/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj +++ b/src/FrostFS.SDK.Protos/FrostFS.SDK.Protos.csproj @@ -5,7 +5,7 @@ 12.0 enable FrostFS.SDK.Protos - 1.0.7 + 1.0.3 Protobuf client for C# SDK @@ -13,21 +13,19 @@ - true + true - <_SkipUpgradeNetAnalyzersNuGetWarning>true + <_SkipUpgradeNetAnalyzersNuGetWarning>true - true + true false - True - .\\..\\..\\keyfile.snk diff --git a/src/FrostFS.SDK.Protos/object/service.proto b/src/FrostFS.SDK.Protos/object/service.proto index 2b8042b..d490dca 100644 --- a/src/FrostFS.SDK.Protos/object/service.proto +++ b/src/FrostFS.SDK.Protos/object/service.proto @@ -887,6 +887,10 @@ message PatchRequest { // key, then it just replaces it while merging the lists. bool replace_attributes = 3; + // New split header for the object. This defines how the object will relate + // to other objects in a split operation. + neo.fs.v2.object.Header.Split new_split_header = 5; + // The patch for the object's payload. message Patch { // The range of the source object for which the payload is replaced by the diff --git a/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj b/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj index 37bbace..30c0564 100644 --- a/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj +++ b/src/FrostFS.SDK.Tests/FrostFS.SDK.Tests.csproj @@ -10,16 +10,11 @@ - true + true - true - - - - True - .\\..\\..\\keyfile.snk + true @@ -47,4 +42,5 @@ PreserveNewest + diff --git a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs index 58a9d40..0a3aec6 100644 --- a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs @@ -3,6 +3,7 @@ using System.Security.Cryptography; using FrostFS.Object; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; +using FrostFS.SDK.Cryptography; using FrostFS.Session; using Google.Protobuf; @@ -42,7 +43,7 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader) Signature = new Refs.Signature { Key = Key.PublicKeyProto, - Sign = Key.ECDsaKey.SignData(header.ToByteArray()), + Sign = Key.ECDsaKey. SignData(header.ToByteArray()), } } }, diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs index eafdda5..e6f25c2 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs @@ -1,3 +1,5 @@ +using System.Security.Cryptography; + using FrostFS.Container; using FrostFS.Object; using FrostFS.SDK.Client; diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs index 7491950..0ad6ab0 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs @@ -23,16 +23,13 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) var grpcVersion = Version.ToMessage(); - Span ContainerGuidSpan = stackalloc byte[16]; - ContainerGuid.ToBytes(ContainerGuidSpan); - PutResponse putResponse = new() { Body = new PutResponse.Types.Body { ContainerId = new ContainerID { - Value = ByteString.CopyFrom(ContainerGuidSpan) + Value = ByteString.CopyFrom(ContainerGuid.ToBytes()) } }, MetaHeader = new ResponseMetaHeader @@ -72,7 +69,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key) Container = new Container.Container { Version = grpcVersion, - Nonce = ByteString.CopyFrom(ContainerGuidSpan), + Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()), PlacementPolicy = PlacementPolicy.GetPolicy() } }, diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index 772f92e..49a0f6d 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -4,6 +4,7 @@ using System.Security.Cryptography; using FrostFS.Object; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; +using FrostFS.SDK.Cryptography; using Google.Protobuf; @@ -41,8 +42,6 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) public Collection RangeHashResponses { get; } = []; - public Action? Callback; - public override Mock GetMock() { var mock = new Mock(); @@ -167,14 +166,9 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) It.IsAny())) .Returns((PutSingleRequest r, Metadata m, DateTime? dt, CancellationToken ct) => { - Callback?.Invoke(); Verifier.CheckRequest(r); - var req = r.Clone(); - - // Clone method does not clone the payload but keeps a reference - req.Body.Object.Payload = ByteString.CopyFrom(r.Body.Object.Payload.ToByteArray()); - PutSingleRequests.Add(req); + PutSingleRequests.Add(r); return new AsyncUnaryCall( Task.FromResult(putSingleResponse), diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs index f1032db..37a8c13 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ContainerTests/ContainerTests.cs @@ -40,9 +40,9 @@ public class ContainerTests : SmokeTestsBase { var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); await Cleanup(client); - + client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); - + FrostFsContainerId containerId = await CreateContainer(client, ctx: default, token: null, @@ -50,25 +50,22 @@ public class ContainerTests : SmokeTestsBase backupFactor: 1, selectors: [], filter: [], - containerAttributes: [new("testKey1", "testValue1")], + containerAttributes: [new ("testKey1", "testValue1")], new FrostFsReplica(3)); Assert.NotNull(containerId); var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default); - + Assert.NotNull(container); Assert.NotNull(container.Attributes); + Assert.Equal(2, container.Attributes.Count); Assert.Equal("testKey1", container.Attributes[0].Key); Assert.Equal("testValue1", container.Attributes[0].Value); - - //Assert.Equal("true", container.Attributes[1].Value); - - // for cluster - //Assert.Equal(2, container.Attributes.Count); - //Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key); - + Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key); + Assert.Equal("true", container.Attributes[1].Value); + Assert.True(container.PlacementPolicy.HasValue); Assert.Equal(1u, container.PlacementPolicy.Value.BackupFactor); @@ -77,7 +74,7 @@ public class ContainerTests : SmokeTestsBase Assert.Empty(container.PlacementPolicy.Value.Filters); Assert.Single(container.PlacementPolicy.Value.Replicas); - Assert.Equal(3u, container.PlacementPolicy.Value.Replicas[0].Count); + Assert.Equal(3, container.PlacementPolicy.Value.Replicas[0].Count); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount); Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector); @@ -85,7 +82,7 @@ public class ContainerTests : SmokeTestsBase Assert.Equal(OwnerId!.ToString(), container.Owner!.Value); Assert.NotNull(container.Version); - + Assert.Equal(Version!.Major, container.Version.Major); Assert.Equal(Version.Minor, container.Version.Minor); } @@ -104,8 +101,8 @@ public class ContainerTests : SmokeTestsBase new ("filter2", "filterKey2", 2, "testValue2", [new ("subFilter2", "subFilterKey2", 3, "testValue3",[])]) ]; - Collection selectors = [ - new ("selector1") { + Collection selectors = [ + new ("selector1") { Count = 1, Clause = 1, Attribute = "attribute1", @@ -132,19 +129,19 @@ public class ContainerTests : SmokeTestsBase Assert.NotNull(containerId); var container = await client.GetContainerAsync(new PrmContainerGet(containerId), default); - + Assert.NotNull(container); Assert.NotNull(container.Attributes); - //Assert.Single(container.Attributes); - //Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key); - //Assert.Equal("true", container.Attributes[0].Value); + Assert.Single(container.Attributes); + Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key); + Assert.Equal("true", container.Attributes[0].Value); Assert.True(container.PlacementPolicy.HasValue); Assert.Equal(2u, container.PlacementPolicy.Value.BackupFactor); Assert.False(container.PlacementPolicy.Value.Unique); - + Assert.NotEmpty(container.PlacementPolicy.Value.Selectors); Assert.Equal(2, container.PlacementPolicy.Value.Selectors.Count); @@ -191,7 +188,7 @@ public class ContainerTests : SmokeTestsBase Assert.Empty(subFilter.Filters); Assert.Single(container.PlacementPolicy.Value.Replicas); - Assert.Equal(1u, container.PlacementPolicy.Value.Replicas[0].Count); + Assert.Equal(1, container.PlacementPolicy.Value.Replicas[0].Count); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount); Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount); Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector); @@ -199,7 +196,7 @@ public class ContainerTests : SmokeTestsBase Assert.Equal(OwnerId!.ToString(), container.Owner!.Value); Assert.NotNull(container.Version); - + Assert.Equal(Version!.Major, container.Version.Major); Assert.Equal(Version.Minor, container.Version.Minor); } @@ -236,12 +233,12 @@ public class ContainerTests : SmokeTestsBase }); } -#pragma warning disable xUnit1031 // Timeout is used + #pragma warning disable xUnit1031 // Timeout is used if (!Task.WaitAll(tasks, 20000)) { Assert.Fail("cannot create containers"); } -#pragma warning restore xUnit1031 + #pragma warning restore xUnit1031 var containers = client.ListContainersAsync(new PrmContainerGetAll(), default); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs b/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs index 9d2fd63..664d522 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/MiscTests/InterceptorTest.cs @@ -29,8 +29,8 @@ public class InterceptorTests() : SmokeTestsBase Assert.True(callbackInvoked); Assert.True(interceptorInvoked); - Assert.Equal(2u, result.Version.Major); - Assert.Equal(13u, result.Version.Minor); + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); Assert.NotNull(result.Addresses); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs index 229531a..f1fce3c 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/NetworkTests/NetworkTests.cs @@ -28,14 +28,12 @@ public class SmokeClientTests : SmokeTestsBase var result = await client.GetNodeInfoAsync(default); - Assert.Equal(2u, result.Version.Major); - Assert.Equal(13u, result.Version.Minor); + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); - Assert.Single(result.Addresses); - Assert.Equal(9, result.Attributes.Count); - //Assert.Equal(2, result.Addresses.Count); - //Assert.Equal(11, result.Attributes.Count); + Assert.Equal(2, result.Addresses.Count); + Assert.Equal(11, result.Attributes.Count); } [Fact] @@ -67,13 +65,13 @@ public class SmokeClientTests : SmokeTestsBase Assert.True(result.HomomorphicHashingDisabled); Assert.True(result.MaintenanceModeAllowed); Assert.True(0u < result.MagicNumber); - + Assert.Equal(0u, result.AuditFee); Assert.Equal(0u, result.BasicIncomeRate); Assert.Equal(0u, result.ContainerAliasFee); Assert.Equal(0u, result.ContainerFee); Assert.Equal(75u, result.EpochDuration); - Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee); + Assert.Equal(10_000_000_000u, result.InnerRingCandidateFee); Assert.Equal(12u, result.MaxECDataCount); Assert.Equal(4u, result.MaxECParityCount); Assert.Equal(5242880u, result.MaxObjectSize); @@ -103,8 +101,8 @@ public class SmokeClientTests : SmokeTestsBase Assert.True(callbackInvoked); Assert.True(interceptorInvoked); - Assert.Equal(2u, result.Version.Major); - Assert.Equal(13u, result.Version.Minor); + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); Assert.Equal(NodeState.Online, result.State); Assert.Equal(33, result.PublicKey.Length); Assert.NotNull(result.Addresses); diff --git a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs index 473b076..03a9169 100644 --- a/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs @@ -26,7 +26,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase [InlineData(false, 2, 3)] [InlineData(true, 2, 1)] [InlineData(false, 2, 1)] - public async void FullScenario(bool unique, uint backupFactor, uint replicas) + public async void FullScenario(bool unique, uint backupFactor, int replicas) { var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); _testOutputHelper.WriteLine("client created"); @@ -58,7 +58,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase int[] objectSizes = [1, 257, 5 * 1024 * 1024, 20 * 1024 * 1024]; string[] objectTypes = [clientCut, serverCut, singleObject]; - + foreach (var objectSize in objectSizes) { _testOutputHelper.WriteLine($"test set for object size {objectSize}"); @@ -121,16 +121,9 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase await ValidateFilters(client, containerId, objectId, null, (ulong)bytes.Length); _testOutputHelper.WriteLine($"\tfilters validated"); - if (type != clientCut && bytes.Length > 1024 + 64 && bytes.Length < 20 * 1024 * 1024) + if (type != clientCut) { - // patch payload only - await ValidatePatch(client, containerId, bytes, true, objectId, [], false); - - // patch attributes only - await ValidatePatch(client, containerId, bytes, false, objectId, [new("a1", "v1"), new("a2", "v2")], false); - - // patch payload and attributes - await ValidatePatch(client, containerId, bytes, true, objectId, [new("a3", "v3"), new("a4", "v4")], true); + await ValidatePatch(client, containerId, bytes, objectId); _testOutputHelper.WriteLine($"\tpatch validated"); } @@ -164,7 +157,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.NotNull(x); Assert.True(x.Length > 0); - // Assert.True(expectedHash.SequenceEqual(h.ToArray())); + // Assert.True(expectedHash.SequenceEqual(h.ToArray())); } } @@ -206,73 +199,51 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase _testOutputHelper.WriteLine($"\t\trange {range.Offset};{range.Length} validated"); } - private static async Task ValidatePatch( - IFrostFSClient client, - FrostFsContainerId containerId, - byte[] bytes, - bool patchPayload, - FrostFsObjectId objectId, - FrostFsAttributePair[] attributes, - bool replaceAttributes) + private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId) { - byte[]? patch = null; - FrostFsRange range = new(); - if (patchPayload) - { - patch = new byte[1024]; - for (int i = 0; i < patch.Length; i++) - { - patch[i] = 32; - } + if (bytes.Length < 1024 + 64 || bytes.Length > 5900) + return; - range = new FrostFsRange(64, (ulong)patch.Length); + var patch = new byte[1024]; + for (int i = 0; i < patch.Length; i++) + { + patch[i] = 32; } + var range = new FrostFsRange(64, (ulong)patch.Length); + var patchParams = new PrmObjectPatch( new FrostFsAddress(containerId, objectId), - payload: new MemoryStream(patch ?? []), + payload: new MemoryStream(patch), maxChunkLength: 1024, - range: range, - replaceAttributes: replaceAttributes, - newAttributes: attributes); + range: range); var newIbjId = await client.PatchObjectAsync(patchParams, default); var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, newIbjId), default); - if (patchPayload) + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) { - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - for (int i = 0; i < (int)range.Offset; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); - - var rangeEnd = range.Offset + range.Length; - - for (int i = (int)range.Offset; i < (int)rangeEnd; i++) - Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); - - for (int i = (int)rangeEnd; i < bytes.Length; i++) - Assert.Equal(downloadedBytes[i], bytes[i]); + ms.Write(chunk.Value.Span); } - if (attributes != null && attributes.Length > 0) - { - foreach (var newAttr in attributes) - { - var i = @object!.Header!.Attributes!.Count(p => p.Key == newAttr.Key && p.Value == newAttr.Value); - Assert.Equal(1, i); - } - } + for (int i = 0; i < (int)range.Offset; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + var rangeEnd = range.Offset + range.Length; + + for (int i = (int)range.Offset; i < (int)rangeEnd; i++) + Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); + + for (int i = (int)rangeEnd; i < bytes.Length; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); } + private async Task ValidateFilters(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId, SplitId? splitId, ulong length) { var ecdsaKey = keyString.LoadWif(); @@ -347,7 +318,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.NotNull(objHeader); Assert.Equal(containerId.GetValue(), objHeader.ContainerId.GetValue()); - + Assert.Equal(expected.HeaderInfo!.OwnerId!.Value, objHeader.OwnerId!.Value); Assert.Equal(expected.HeaderInfo.Version!.Major, objHeader.Version!.Major); Assert.Equal(expected.HeaderInfo.Version!.Minor, objHeader.Version!.Minor); @@ -367,6 +338,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase Assert.Null(objHeader.Split); } + private static async Task CreateObjectServerCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes) { var header = new FrostFsObjectHeader( diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs new file mode 100644 index 0000000..831f37e --- /dev/null +++ b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultiThreadTestsBase.cs @@ -0,0 +1,42 @@ +using System.Security.Cryptography; + +using FrostFS.SDK.Client; +using FrostFS.SDK.Cryptography; + +namespace FrostFS.SDK.Tests.Smoke; + +public abstract class MultiThreadTestsBase +{ + private TestNodeInfo[] nodes; + + protected CallContext? Ctx { get; } + + protected MultiThreadTestsBase() + { + nodes = new TestNodeInfo[4]; + + nodes[0] = new(new Uri(""), ""); + nodes[1] = new(new Uri(""), ""); + nodes[2] = new(new Uri(""), ""); + nodes[3] = new(new Uri(""), ""); + } +} + +public class TestNodeInfo +{ + internal Uri Uri; + + protected ECDsa? Key { get; } + + protected FrostFsOwner? OwnerId { get; } + + protected FrostFsVersion? Version { get; } + + public TestNodeInfo(Uri uri, string keyString) + { + Uri = uri; + Key = keyString.LoadWif(); + OwnerId = FrostFsOwner.FromKey(Key); + Version = new FrostFsVersion(2, 13); + } +} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs new file mode 100644 index 0000000..194ef98 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadPoolSmokeTests.cs @@ -0,0 +1,544 @@ +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; + +using FrostFS.SDK.Client; +using FrostFS.SDK.Cryptography; + +namespace FrostFS.SDK.Tests.Smoke; + +[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] +[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] +public class MultithreadPoolSmokeTests : SmokeTestsBase +{ + private InitParameters GetDefaultParams() + { + return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) + { + Key = keyString.LoadWif(), + NodeParams = [new(1, url, 100.0f)], + ClientBuilder = null, + GracefulCloseOnSwitchTimeout = 30_000_000, + Logger = null + }; + } + + [Fact] + public async void NetworkMapTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)); + + Assert.Null(error); + + var result = await pool.GetNetmapSnapshotAsync(default); + + Assert.True(result.Epoch > 0); + Assert.Single(result.NodeInfoCollection); + + var item = result.NodeInfoCollection[0]; + Assert.Equal(2, item.Version.Major); + Assert.Equal(13, item.Version.Minor); + Assert.Equal(NodeState.Online, item.State); + Assert.True(item.PublicKey.Length > 0); + Assert.Single(item.Addresses); + Assert.Equal(9, item.Attributes.Count); + } + + [Fact] + public async void NodeInfoTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)); + + Assert.Null(error); + + var result = await pool.GetNodeInfoAsync(default); + + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); + Assert.Equal(NodeState.Online, result.State); + Assert.Equal(33, result.PublicKey.Length); + Assert.Single(result.Addresses); + Assert.Equal(9, result.Attributes.Count); + } + + [Fact] + public async void NodeInfoStatisticsTwoNodesTest() + { + var callbackText = string.Empty; + + var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) + { + Key = keyString.LoadWif(), + NodeParams = [ + new(1, url, 100.0f), + new(2, url.Replace('0', '1'), 100.0f) + ], + ClientBuilder = null, + GracefulCloseOnSwitchTimeout = 30_000_000, + Logger = null, + Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds" + }; + + var pool = new Pool(options); + + var ctx = new CallContext(TimeSpan.Zero); + + var error = await pool.Dial(ctx).ConfigureAwait(true); + + Assert.Null(error); + + var result = await pool.GetNodeInfoAsync(default); + + var statistics = pool.Statistic(); + + Assert.False(string.IsNullOrEmpty(callbackText)); + Assert.Contains(" took ", callbackText, StringComparison.Ordinal); + } + + [Fact] + public async void NodeInfoStatisticsTest() + { + var options = GetDefaultParams(); + + var callbackText = string.Empty; + options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; + + var pool = new Pool(options); + + var ctx = new CallContext(TimeSpan.Zero); + + var error = await pool.Dial(ctx).ConfigureAwait(true); + + Assert.Null(error); + + var result = await pool.GetNodeInfoAsync(default); + + Assert.False(string.IsNullOrEmpty(callbackText)); + Assert.Contains(" took ", callbackText, StringComparison.Ordinal); + } + + [Fact] + public async void GetSessionTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + var prm = new PrmSessionCreate(100); + + var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true); + + var ownerHash = Base58.Decode(OwnerId!.Value); + + Assert.NotNull(token); + Assert.NotEqual(Guid.Empty, token.Id); + Assert.Equal(33, token.SessionKey.Length); + } + + [Fact] + public async void CreateObjectWithSessionToken() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["key1", "value1"]); + + var containerId = await pool.PutContainerAsync(createContainerParam, default); + + var bytes = GetRandomBytes(1024); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + sessionToken: token); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(pool); + } + + [Fact] + public async void FilterTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + lightWait); + + var containerId = await pool.PutContainerAsync(createContainerParam, default); + + var bytes = new byte[] { 1, 2, 3 }; + + var ParentHeader = new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular) + { + PayloadLength = 3 + }; + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")], + new FrostFsSplit())); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId, false), default); + + var ecdsaKey = keyString.LoadWif(); + + var networkInfo = await pool.GetNetmapSnapshotAsync(default); + + await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); + + await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); + + await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); + + await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); + + await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); + + await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); + + await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); + + await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); + + var checkSum = CheckSum.CreateCheckSum(bytes); + + await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); + + await CheckFilter(pool, containerId, new FilterByPhysicallyStored()); + } + + private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter) + { + var resultObjectsCount = 0; + + PrmObjectSearch searchParam = new(containerId, null, [], filter); + + await foreach (var objId in pool.SearchObjectsAsync(searchParam, default)) + { + resultObjectsCount++; + var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); + } + + Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); + } + + [Theory] + [InlineData(1)] + [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB + [InlineData(6 * 1024 * 1024 + 100)] + public async void SimpleScenarioTest(int objectSize) + { + bool callbackInvoked = false; + + var options = GetDefaultParams(); + + options.Callback = new((cs) => + { + callbackInvoked = true; + Assert.True(cs.ElapsedMicroSeconds > 0); + }); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["testKey", "testValue"]); + + var createdContainer = await pool.PutContainerAsync(createContainerParam, default); + + var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default); + Assert.NotNull(container); + Assert.True(callbackInvoked); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")])); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) + { + hasObject = true; + + var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes!); + Assert.Equal("fileName", objHeader.Attributes!.First().Key); + Assert.Equal("test", objHeader.Attributes!.First().Value); + } + + Assert.True(hasObject); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(pool); + + await foreach (var _ in pool.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB + [InlineData(6 * 1024 * 1024 + 100)] + public async void SimpleScenarioWithSessionTest(int objectSize) + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0)); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); + + await Cleanup(pool); + + var ctx = new CallContext(TimeSpan.FromSeconds(20)); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams); + + var container = await pool.PutContainerAsync(createContainerParam, ctx); + + var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx); + Assert.NotNull(containerInfo); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: container, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + sessionToken: token); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default)) + { + hasObject = true; + + var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); + + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes.First().Key); + Assert.Equal("test", objHeader.Attributes.First().Value); + } + + Assert.True(hasObject); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(pool); + + await foreach (var _ in pool.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB + [InlineData(64 * 1024 * 1024 - 1)] + [InlineData(64 * 1024 * 1024 + 1)] + [InlineData(2 * 64 * 1024 * 1024 + 256)] + [InlineData(200)] + public async void ClientCutScenarioTest(int objectSize) + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + lightWait); + + var containerId = await pool.PutContainerAsync(createContainerParam, default); + + var ctx = new CallContext(TimeSpan.FromSeconds(10)); + + // ctx.Interceptors.Add(new CallbackInterceptor()); + + var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx); + + Assert.NotNull(container); + + byte[] bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectClientCutPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + payload: new MemoryStream(bytes)); + + var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) + { + hasObject = true; + + var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); + + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes[0].Key); + Assert.Equal("test", objHeader.Attributes[0].Value); + } + + Assert.True(hasObject); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await CheckFilter(pool, containerId, new FilterByRootObject()); + + await Cleanup(pool); + + await foreach (var cid in pool.ListContainersAsync(default, default)) + { + Assert.Fail($"Container {cid.GetValue()} exist"); + } + } +} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs new file mode 100644 index 0000000..7de8155 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Smoke/PoolTests/Multithread/MultithreadSmokeClientTests.cs @@ -0,0 +1,684 @@ +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; + +using FrostFS.SDK.Client; +using FrostFS.SDK.Client.Interfaces; +using FrostFS.SDK.Cryptography; +using FrostFS.SDK.SmokeTests; + +namespace FrostFS.SDK.Tests.Smoke; + +[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] +[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] +public class MultithreadSmokeClientTests : SmokeTestsBase +{ + [Fact] + public async void AccountTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + var result = await client.GetBalanceAsync(default); + + Assert.NotNull(result); + Assert.True(result.Value == 0); + } + + [Fact] + public async void NetworkMapTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + var result = await client.GetNetmapSnapshotAsync(default); + + Assert.True(result.Epoch > 0); + Assert.Single(result.NodeInfoCollection); + + var item = result.NodeInfoCollection[0]; + Assert.Equal(2, item.Version.Major); + Assert.Equal(13, item.Version.Minor); + Assert.Equal(NodeState.Online, item.State); + Assert.True(item.PublicKey.Length > 0); + Assert.Single(item.Addresses); + Assert.Equal(9, item.Attributes.Count); + } + + + [Fact] + public async void NodeInfoTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + var result = await client.GetNodeInfoAsync(default); + + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); + Assert.Equal(NodeState.Online, result.State); + Assert.Equal(33, result.PublicKey.Length); + Assert.Single(result.Addresses); + Assert.Equal(9, result.Attributes.Count); + } + + [Fact] + public async void NodeInfoStatisticsTest() + { + var options = ClientOptions; + + var callbackContent = string.Empty; + options.Value.Callback = (cs) => callbackContent = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; + + var client = FrostFSClient.GetInstance(options, GrpcChannel); + + var result = await client.GetNodeInfoAsync(default); + + Assert.NotEmpty(callbackContent); + } + + [Fact] + public async void GetSessionTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + var token = await client.CreateSessionAsync(new(100), default); + + Assert.NotNull(token); + Assert.NotEqual(Guid.Empty, token.Id); + Assert.Equal(33, token.SessionKey.Length); + } + + [Fact] + public async void CreateObjectWithSessionToken() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + await Cleanup(client); + + var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["key1", "value1"]); + + var containerId = await client.PutContainerAsync(createContainerParam, default); + + var bytes = GetRandomBytes(1024); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + sessionToken: token); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default) + .ConfigureAwait(true); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk().ConfigureAwait(true)) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(client).ConfigureAwait(true); + } + + [Fact] + public async void FilterTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + lightWait); + + var containerId = await client.PutContainerAsync(createContainerParam, default); + + var bytes = new byte[] { 1, 2, 3 }; + + var ParentHeader = new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular) + { + PayloadLength = 3 + }; + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")], + new FrostFsSplit())); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var head = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); + + var ecdsaKey = keyString.LoadWif(); + + var networkInfo = await client.GetNetmapSnapshotAsync(default); + + await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); + + await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); + + await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); + + await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); + + await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); + + await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); + + await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); + + await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); + + var checkSum = CheckSum.CreateCheckSum(bytes); + + await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); + + await CheckFilter(client, containerId, new FilterByPhysicallyStored()); + } + + private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter) + { + var resultObjectsCount = 0; + + PrmObjectSearch searchParam = new(containerId, null, [], filter); + + await foreach (var objId in client.SearchObjectsAsync(searchParam, default)) + { + resultObjectsCount++; + var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); + } + + Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); + } + + [Theory] + [InlineData(1)] + [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB + [InlineData(6 * 1024 * 1024 + 100)] + public async void SimpleScenarioTest(int objectSize) + { + bool callbackInvoked = false; + + var options = ClientOptions; + + options.Value.Callback = new((cs) => + { + callbackInvoked = true; + Assert.True(cs.ElapsedMicroSeconds > 0); + }); + + var client = FrostFSClient.GetInstance(options, GrpcChannel); + + await Cleanup(client); + + var ctx = new CallContext(TimeSpan.FromSeconds(20)); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["testKey", "testValue"]); + + var createdContainer = await client.PutContainerAsync(createContainerParam, ctx); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); + Assert.NotNull(container); + Assert.True(callbackInvoked); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")])); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) + { + hasObject = true; + + var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes.First().Key); + Assert.Equal("test", objHeader.Attributes.First().Value); + } + + Assert.True(hasObject); + + var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Fact] + public async void PatchTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["testKey", "testValue"]); + + var createdContainer = await client.PutContainerAsync(createContainerParam, default); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); + Assert.NotNull(container); + + var bytes = new byte[1024]; + for (int i = 0; i < 1024; i++) + { + bytes[i] = 31; + } + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")])); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var patch = new byte[16]; + for (int i = 0; i < 16; i++) + { + patch[i] = 32; + } + + var range = new FrostFsRange(8, (ulong)patch.Length); + + var patchParams = new PrmObjectPatch( + new FrostFsAddress(createdContainer, objectId), + payload: new MemoryStream(patch), + maxChunkLength: 32, + range: range); + + var newIbjId = await client.PatchObjectAsync(patchParams, default); + + var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + for (int i = 0; i < (int)range.Offset; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + var rangeEnd = range.Offset + range.Length; + + for (int i = (int)range.Offset; i < (int)rangeEnd; i++) + Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); + + for (int i = (int)rangeEnd; i < bytes.Length; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Fact] + public async void RangeTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["testKey", "testValue"]); + + var createdContainer = await client.PutContainerAsync(createContainerParam, default); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); + Assert.NotNull(container); + + var bytes = new byte[256]; + for (int i = 0; i < 256; i++) + { + bytes[i] = (byte)i; + } + + var param = new PrmObjectPut( + new FrostFsObjectHeader(containerId: createdContainer, type: FrostFsObjectType.Regular)); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64)); + + var rangeReader = await client.GetRangeAsync(rangeParam, default); + + var downloadedBytes = new byte[rangeParam.Range.Length]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await rangeReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes)); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Fact] + public async void RangeHashTest() + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["testKey", "testValue"]); + + var createdContainer = await client.PutContainerAsync(createContainerParam, default); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default); + Assert.NotNull(container); + + var bytes = new byte[256]; + for (int i = 0; i < 256; i++) + { + bytes[i] = (byte)i; + } + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular)); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [new FrostFsRange(100, 64)], bytes); + + var hashes = await client.GetRangeHashAsync(rangeParam, default); + + foreach (var hash in hashes) + { + var x = hash[..32].ToArray(); + } + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB + [InlineData(6 * 1024 * 1024 + 100)] + public async void SimpleScenarioWithSessionTest(int objectSize) + { + var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel); + //Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0)) + var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); + + await Cleanup(client); + + var ctx = new CallContext(TimeSpan.FromSeconds(20)); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams); + + var container = await client.PutContainerAsync(createContainerParam, ctx); + + var containerInfo = await client.GetContainerAsync(new PrmContainerGet(container), ctx); + Assert.NotNull(containerInfo); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: container, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + sessionToken: token); + + var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in client.SearchObjectsAsync( + new PrmObjectSearch(container, token, [], filter), default)) + { + hasObject = true; + + var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); + + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes.First().Key); + Assert.Equal("test", objHeader.Attributes.First().Value); + } + + Assert.True(hasObject); + + var @object = await client.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB + [InlineData(64 * 1024 * 1024 - 1)] + [InlineData(64 * 1024 * 1024 + 1)] + [InlineData(2 * 64 * 1024 * 1024 + 256)] + [InlineData(200)] + public async void ClientCutScenarioTest(int objectSize) + { + var options = ClientOptions; + options.Value.Interceptors.Add(new CallbackInterceptor()); + + var client = FrostFSClient.GetInstance(options, GrpcChannel); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + lightWait); + + var containerId = await client.PutContainerAsync(createContainerParam, default); + + var ctx = new CallContext(TimeSpan.FromSeconds(10)); + + var container = await client.GetContainerAsync(new PrmContainerGet(containerId), ctx); + + Assert.NotNull(container); + + byte[] bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectClientCutPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + payload: new MemoryStream(bytes)); + + var objectId = await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) + { + hasObject = true; + + var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); + + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes[0].Key); + Assert.Equal("test", objHeader.Attributes[0].Value); + } + + Assert.True(hasObject); + + var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await CheckFilter(client, containerId, new FilterByRootObject()); + + await Cleanup(client); + + var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5)); + + IAsyncEnumerator? enumerator = null; + do + { + if (deadline <= DateTime.UtcNow) + { + Assert.Fail("Containers exist"); + break; + } + + enumerator = client.ListContainersAsync(default, default).GetAsyncEnumerator(); + await Task.Delay(500); + } + while (await enumerator!.MoveNextAsync()); + } + + [Fact] + public async void NodeInfoCallbackAndInterceptorTest() + { + bool callbackInvoked = false; + bool intercepterInvoked = false; + + var options = ClientOptions; + options.Value.Callback = (cs) => + { + callbackInvoked = true; + Assert.True(cs.ElapsedMicroSeconds > 0); + }; + + options.Value.Interceptors.Add(new CallbackInterceptor(s => intercepterInvoked = true)); + + var client = FrostFSClient.GetInstance(options, GrpcChannel); + + var result = await client.GetNodeInfoAsync(default); + + Assert.True(callbackInvoked); + Assert.True(intercepterInvoked); + + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); + Assert.Equal(NodeState.Online, result.State); + Assert.Equal(33, result.PublicKey.Length); + Assert.Single(result.Addresses); + Assert.Equal(9, result.Attributes.Count); + } +} diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs new file mode 100644 index 0000000..9e7ea80 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Smoke/PoolTests/PoolSmokeTests.cs @@ -0,0 +1,546 @@ +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; + +using FrostFS.SDK.Client; +using FrostFS.SDK.Cryptography; + +namespace FrostFS.SDK.Tests.Smoke; + +[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] +[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] +public class PoolSmokeTests : SmokeTestsBase +{ + private InitParameters GetDefaultParams() + { + return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) + { + Key = keyString.LoadWif(), + NodeParams = [new(1, url, 100.0f)], + ClientBuilder = null, + GracefulCloseOnSwitchTimeout = 30_000_000, + Logger = null + }; + } + + [Fact] + public async void NetworkMapTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)); + + Assert.Null(error); + + var result = await pool.GetNetmapSnapshotAsync(default); + + Assert.True(result.Epoch > 0); + Assert.Single(result.NodeInfoCollection); + + var item = result.NodeInfoCollection[0]; + Assert.Equal(2, item.Version.Major); + Assert.Equal(13, item.Version.Minor); + Assert.Equal(NodeState.Online, item.State); + Assert.True(item.PublicKey.Length > 0); + Assert.Single(item.Addresses); + Assert.Equal(9, item.Attributes.Count); + } + + [Fact] + public async void NodeInfoTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)); + + Assert.Null(error); + + var result = await pool.GetNodeInfoAsync(default); + + Assert.Equal(2, result.Version.Major); + Assert.Equal(13, result.Version.Minor); + Assert.Equal(NodeState.Online, result.State); + Assert.Equal(33, result.PublicKey.Length); + Assert.Single(result.Addresses); + Assert.Equal(9, result.Attributes.Count); + } + + [Fact] + public async void NodeInfoStatisticsTwoNodesTest() + { + var callbackText = string.Empty; + + var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url))) + { + Key = keyString.LoadWif(), + NodeParams = [ + new(1, url, 100.0f), + new(2, url.Replace('0', '1'), 100.0f) + ], + ClientBuilder = null, + GracefulCloseOnSwitchTimeout = 30_000_000, + Logger = null, + Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds" + }; + + var pool = new Pool(options); + + var ctx = new CallContext(TimeSpan.Zero); + + var error = await pool.Dial(ctx).ConfigureAwait(true); + + Assert.Null(error); + + var result = await pool.GetNodeInfoAsync(default); + + var statistics = pool.Statistic(); + + Assert.False(string.IsNullOrEmpty(callbackText)); + Assert.Contains(" took ", callbackText, StringComparison.Ordinal); + } + + [Fact] + public async void NodeInfoStatisticsTest() + { + var options = GetDefaultParams(); + + var callbackText = string.Empty; + options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"; + + var pool = new Pool(options); + + var ctx = new CallContext(TimeSpan.Zero); + + var error = await pool.Dial(ctx).ConfigureAwait(true); + + Assert.Null(error); + + var result = await pool.GetNodeInfoAsync(default); + + Assert.False(string.IsNullOrEmpty(callbackText)); + Assert.Contains(" took ", callbackText, StringComparison.Ordinal); + } + + [Fact] + public async void GetSessionTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + var prm = new PrmSessionCreate(100); + + var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true); + + var ownerHash = Base58.Decode(OwnerId!.Value); + + Assert.NotNull(token); + Assert.NotEqual(Guid.Empty, token.Id); + Assert.Equal(33, token.SessionKey.Length); + } + + [Fact] + public async void CreateObjectWithSessionToken() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["key1", "value1"]); + + var containerId = await pool.PutContainerAsync(createContainerParam, default); + + var bytes = GetRandomBytes(1024); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + sessionToken: token); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(pool); + } + + [Fact] + public async void FilterTest() + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + lightWait); + + var containerId = await pool.PutContainerAsync(createContainerParam, default); + + var bytes = new byte[] { 1, 2, 3 }; + + var ParentHeader = new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular) + { + PayloadLength = 3 + }; + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")], + new FrostFsSplit())); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); + + var ecdsaKey = keyString.LoadWif(); + + var networkInfo = await pool.GetNetmapSnapshotAsync(default); + + await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId)); + + await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey))); + + await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId)); + + await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test")); + + await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId)); + + await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version)); + + await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch)); + + await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3)); + + var checkSum = CheckSum.CreateCheckSum(bytes); + + await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum)); + + await CheckFilter(pool, containerId, new FilterByPhysicallyStored()); + } + + private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter) + { + var resultObjectsCount = 0; + + PrmObjectSearch searchParam = new(containerId, null, [], filter); + + await foreach (var objId in pool.SearchObjectsAsync(searchParam, default)) + { + resultObjectsCount++; + var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default); + } + + Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work"); + } + + [Theory] + [InlineData(1)] + [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB + [InlineData(6 * 1024 * 1024 + 100)] + public async void SimpleScenarioTest(int objectSize) + { + bool callbackInvoked = false; + + var options = GetDefaultParams(); + + options.Callback = new((cs) => + { + callbackInvoked = true; + Assert.True(cs.ElapsedMicroSeconds > 0); + }); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var ctx = new CallContext(TimeSpan.Zero); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams, + xheaders: ["testKey", "testValue"]); + + var createdContainer = await pool.PutContainerAsync(createContainerParam, ctx); + + var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default); + Assert.NotNull(container); + Assert.True(callbackInvoked); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")])); + + var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default)) + { + hasObject = true; + + var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default); + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes!); + Assert.Equal("fileName", objHeader.Attributes!.First().Key); + Assert.Equal("test", objHeader.Attributes!.First().Value); + } + + Assert.True(hasObject); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(pool); + + await foreach (var _ in pool.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB + [InlineData(6 * 1024 * 1024 + 100)] + public async void SimpleScenarioWithSessionTest(int objectSize) + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0)); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); + + await Cleanup(pool); + + var ctx = new CallContext(TimeSpan.FromSeconds(20)); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + PrmWait.DefaultParams); + + var container = await pool.PutContainerAsync(createContainerParam, ctx); + + var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx); + Assert.NotNull(containerInfo); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut( + new FrostFsObjectHeader( + containerId: container, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + sessionToken: token); + + var stream = await pool.PutObjectAsync(param, new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + await stream.WriteAsync(bytes.AsMemory()); + var objectId = await stream.CompleteAsync(); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + var objs = pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default); + await foreach (var objId in objs) + { + hasObject = true; + + var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default); + + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes.First().Key); + Assert.Equal("test", objHeader.Attributes.First().Value); + } + + Assert.True(hasObject); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(pool); + + await foreach (var _ in pool.ListContainersAsync(default, default)) + { + Assert.Fail("Containers exist"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB + [InlineData(64 * 1024 * 1024 - 1)] + [InlineData(64 * 1024 * 1024 + 1)] + [InlineData(2 * 64 * 1024 * 1024 + 256)] + [InlineData(200)] + public async void ClientCutScenarioTest(int objectSize) + { + var options = GetDefaultParams(); + + var pool = new Pool(options); + + var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true); + + Assert.Null(error); + + await Cleanup(pool); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), + lightWait); + + var containerId = await pool.PutContainerAsync(createContainerParam, default); + + var ctx = new CallContext(TimeSpan.FromSeconds(10)); + + // ctx.Interceptors.Add(new CallbackInterceptor()); + + var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx); + + Assert.NotNull(container); + + byte[] bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectClientCutPut( + new FrostFsObjectHeader( + containerId: containerId, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + payload: new MemoryStream(bytes)); + + var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default)) + { + hasObject = true; + + var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default); + + var objHeader = res.HeaderInfo; + Assert.NotNull(objHeader); Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes[0].Key); + Assert.Equal("test", objHeader.Attributes[0].Value); + } + + Assert.True(hasObject); + + var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await CheckFilter(pool, containerId, new FilterByRootObject()); + + await Cleanup(pool); + + await foreach (var cid in pool.ListContainersAsync(default, default)) + { + Assert.Fail($"Container {cid.GetValue()} exist"); + } + } +} diff --git a/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs b/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs index 5e6998b..59099a9 100644 --- a/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Smoke/SmokeTestsBase.cs @@ -16,13 +16,19 @@ namespace FrostFS.SDK.Tests.Smoke; [SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")] public abstract class SmokeTestsBase { + // cluster Ori + // internal readonly string url = "http://10.78.128.207:8080"; + // internal readonly string keyString = "L4JWLdedUd4b21sriRHtCPGkjG2Mryz2AWLiVqTBSNyxxyAUcc7s"; + // cluster - // internal readonly string url = "http://10.78.128.190:8080"; - // internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba"; + internal readonly string url = "http://10.78.128.190:8080"; + internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba"; // WSL2 - internal readonly string url = "http://172.29.238.97:8080"; // "http://172.20.8.23:8080"; - internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; + // internal readonly string url = "http://172.29.238.97:8080"; + // internal readonly string keyString = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; // "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; + + //"KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; protected ECDsa? Key { get; } @@ -71,7 +77,7 @@ public abstract class SmokeTestsBase if (networkSettings.HomomorphicHashingDisabled) attributes.Add(new("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", "true")); - + var containerInfo = new FrostFsContainerInfo( new FrostFsPlacementPolicy(unique, backupFactor, selectors, filter, replicas), [.. attributes]); diff --git a/src/FrostFS.SDK.Tests/TestData/wallet.json b/src/FrostFS.SDK.Tests/TestData/wallet.json deleted file mode 100644 index 7a4bdd4..0000000 --- a/src/FrostFS.SDK.Tests/TestData/wallet.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": "1.0", - "accounts": [ - { - "address": "NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw", - "key": "6PYVCcS2yp89JpcfR61FGhdhhzyYjSErNedmpZErnybNTxUZMRdhzJLrek", - "label": "", - "contract": { - "script": "DCEDJOdiiPy5ABANAYAqFO+XfMpFrQc1YSMERt8Us0TIWLZBVuezJw==", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "isDefault": false - } - ], - "scrypt": { - "n": 16384, - "r": 8, - "p": 8 - }, - "extra": { - "Tokens": null - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs index ce1d06c..db1e193 100644 --- a/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ContainerTest.cs @@ -1,5 +1,5 @@ using System.Diagnostics.CodeAnalysis; -using FrostFS.Netmap; + using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; using FrostFS.SDK.Cryptography; @@ -11,27 +11,6 @@ namespace FrostFS.SDK.Tests.Unit; [SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] public class ContainerTest : ContainerTestsBase { - [Theory] - [InlineData(1, "test", 0, 0)] - - public void ReplicaToMessagelTest(uint count, string selector, uint ecDataCount, uint ecParityCount) - { - FrostFsReplica replica = new() - { - Count = count, - Selector = selector, - EcDataCount = ecDataCount, - EcParityCount = ecParityCount - }; - - Replica message = replica.ToMessage(); - - Assert.Equal(count, message.Count); - Assert.Equal(selector, message.Selector); - Assert.Equal(ecDataCount, message.EcDataCount); - Assert.Equal(ecParityCount, message.EcParityCount); - } - [Fact] public async void CreateContainerTest() { @@ -41,16 +20,13 @@ public class ContainerTest : ContainerTestsBase Assert.NotNull(result); Assert.NotNull(result.GetValue()); - - var bytes = Mocker.ContainerGuid.ToByteArray(true); - - Assert.True(Base58.Encode(new Span(bytes)) == result.GetValue()); + Assert.True(Base58.Encode(Mocker.ContainerGuid.ToBytes()) == result.GetValue()); } [Fact] public async void GetContainerTest() { - var cid = new FrostFsContainerId(Base58.Encode(new Span(Mocker.ContainerGuid.ToByteArray(true)))); + var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); var result = await GetClient().GetContainerAsync(new PrmContainerGet(cid), default); @@ -85,7 +61,7 @@ public class ContainerTest : ContainerTestsBase public async void DeleteContainerAsyncTest() { Mocker.ReturnContainerRemoved = true; - var cid = new FrostFsContainerId(Base58.Encode(new Span(Mocker.ContainerGuid.ToByteArray(true)))); + var cid = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); await GetClient().DeleteContainerAsync(new PrmContainerDelete(cid, PrmWait.DefaultParams), default); diff --git a/src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs b/src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs deleted file mode 100644 index 3a71d78..0000000 --- a/src/FrostFS.SDK.Tests/Unit/MetaheaderTests.cs +++ /dev/null @@ -1,29 +0,0 @@ -using FrostFS.SDK.Client; -using FrostFS.SDK.Client.Mappers.GRPC; - -namespace FrostFS.SDK.Tests.Unit; - -public class MetaheaderTests -{ - [Theory] - [InlineData(2, 13, 1, 1)] - [InlineData(200, 0, 1000000, 8)] - public void MetaheaderTest(int major, int minor, int epoch, int ttl) - { - MetaHeader metaHeader = new MetaHeader( - new FrostFsVersion( - major: 2, - minor: 13 - ), - epoch: 0, - ttl: 2 - ); - - var result = metaHeader.ToMessage(); - - Assert.Equal((ulong)metaHeader.Epoch, result.Epoch); - Assert.Equal((ulong)metaHeader.Ttl, result.Ttl); - Assert.Equal((ulong)metaHeader.Version.Major, result.Version.Major); - Assert.Equal((ulong)metaHeader.Version.Minor, result.Version.Minor); - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs b/src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs deleted file mode 100644 index 233e12e..0000000 --- a/src/FrostFS.SDK.Tests/Unit/NetmapSnapshotTests.cs +++ /dev/null @@ -1,101 +0,0 @@ -using FrostFS.Netmap; -using FrostFS.SDK.Client; - -using Google.Protobuf; - -namespace FrostFS.SDK.Tests.Unit; - -public class NetmapSnapshotTests : NetworkTestsBase -{ - [Theory] - [InlineData(false)] - [InlineData(true)] - - public async void NetmapSnapshotTest(bool useContext) - { - var body = new NetmapSnapshotResponse.Types.Body - { - Netmap = new Netmap.Netmap { Epoch = 99 } - }; - - var nodeInfo1 = new NodeInfo - { - State = NodeInfo.Types.State.Online, - PublicKey = ByteString.CopyFrom([1, 2, 3]) - }; - - nodeInfo1.Addresses.Add("address1"); - nodeInfo1.Addresses.Add("address2"); - nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); - nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); - - var nodeInfo2 = new NodeInfo - { - State = NodeInfo.Types.State.Offline, - PublicKey = ByteString.CopyFrom([3, 4, 5]) - }; - - nodeInfo2.Addresses.Add("address3"); - nodeInfo2.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key3", Value = "value3" }); - - body.Netmap.Nodes.Add(nodeInfo1); - body.Netmap.Nodes.Add(nodeInfo2); - - Mocker.NetmapSnapshotResponse = new NetmapSnapshotResponse { Body = body }; - - var ctx = useContext - ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) - : default; - - var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); - - var result = await GetClient(DefaultSettings).GetNetmapSnapshotAsync(ctx); - - var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); - - Assert.NotNull(result); - - Assert.Equal(99u, result.Epoch); - Assert.Equal(2, result.NodeInfoCollection.Count); - - var node1 = result.NodeInfoCollection[0]; - Assert.Equal(NodeState.Online, node1.State); - Assert.Equal(2, node1.Addresses.Count); - Assert.Equal("address1", node1.Addresses.ElementAt(0)); - Assert.Equal("address2", node1.Addresses.ElementAt(1)); - - Assert.Equal(2, node1.Attributes.Count); - - Assert.Equal("key1", node1.Attributes.ElementAt(0).Key); - Assert.Equal("value1", node1.Attributes.ElementAt(0).Value); - Assert.Equal("key2", node1.Attributes.ElementAt(1).Key); - Assert.Equal("value2", node1.Attributes.ElementAt(1).Value); - - var node2 = result.NodeInfoCollection[1]; - Assert.Equal(NodeState.Offline, node2.State); - Assert.Single(node2.Addresses); - Assert.Equal("address3", node2.Addresses.ElementAt(0)); - - Assert.Single(node2.Attributes); - - Assert.Equal("key3", node2.Attributes.ElementAt(0).Key); - Assert.Equal("value3", node2.Attributes.ElementAt(0).Value); - - if (useContext) - { - Assert.NotNull(Mocker.NetmapSnapshotRequest); - Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); - - Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); - Assert.NotNull(Mocker.DateTime); - Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); - Assert.True(Mocker.DateTime.Value <= validTimeoutTo); - } - else - { - Assert.NotNull(Mocker.NetmapSnapshotRequest); - Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); - Assert.Null(Mocker.DateTime); - } - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs b/src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs deleted file mode 100644 index 3f3fb2f..0000000 --- a/src/FrostFS.SDK.Tests/Unit/NetworkSettingsTests.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using FrostFS.SDK.Client; - -namespace FrostFS.SDK.Tests.Unit; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -public class NetworkSettingsTests : NetworkTestsBase -{ - [Theory] - [InlineData(false)] - [InlineData(true)] - public async void NetworkSettingsTest(bool useContext) - { - Mocker.Parameters.Add("AuditFee", [1]); - Mocker.Parameters.Add("BasicIncomeRate", [2]); - Mocker.Parameters.Add("ContainerFee", [3]); - Mocker.Parameters.Add("ContainerAliasFee", [4]); - Mocker.Parameters.Add("EpochDuration", [5]); - Mocker.Parameters.Add("InnerRingCandidateFee", [6]); - Mocker.Parameters.Add("MaxECDataCount", [7]); - Mocker.Parameters.Add("MaxECParityCount", [8]); - Mocker.Parameters.Add("MaxObjectSize", [9]); - Mocker.Parameters.Add("WithdrawFee", [10]); - Mocker.Parameters.Add("HomomorphicHashingDisabled", [1]); - Mocker.Parameters.Add("MaintenanceModeAllowed", [1]); - - var ctx = useContext - ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) - : default; - - var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); - - var result = await GetClient(DefaultSettings).GetNetworkSettingsAsync(ctx); - - var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); - - Assert.NotNull(result); - - Assert.Equal(Mocker.Parameters["AuditFee"], [(byte)result.AuditFee]); - Assert.Equal(Mocker.Parameters["BasicIncomeRate"], [(byte)result.BasicIncomeRate]); - Assert.Equal(Mocker.Parameters["ContainerFee"], [(byte)result.ContainerFee]); - Assert.Equal(Mocker.Parameters["ContainerAliasFee"], [(byte)result.ContainerAliasFee]); - Assert.Equal(Mocker.Parameters["EpochDuration"], [(byte)result.EpochDuration]); - Assert.Equal(Mocker.Parameters["InnerRingCandidateFee"], [(byte)result.InnerRingCandidateFee]); - Assert.Equal(Mocker.Parameters["MaxECDataCount"], [(byte)result.MaxECDataCount]); - Assert.Equal(Mocker.Parameters["MaxECParityCount"], [(byte)result.MaxECParityCount]); - Assert.Equal(Mocker.Parameters["MaxObjectSize"], [(byte)result.MaxObjectSize]); - Assert.Equal(Mocker.Parameters["WithdrawFee"], [(byte)result.WithdrawFee]); - - Assert.True(result.HomomorphicHashingDisabled); - Assert.True(result.MaintenanceModeAllowed); - - if (useContext) - { - Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); - Assert.NotNull(Mocker.DateTime); - - Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); - Assert.True(Mocker.DateTime.Value <= validTimeoutTo); - } - else - { - Assert.NotNull(Mocker.NetworkInfoRequest); - Assert.Empty(Mocker.NetworkInfoRequest.MetaHeader.XHeaders); - Assert.Null(Mocker.DateTime); - } - } -} diff --git a/src/FrostFS.SDK.Tests/Unit/NetworkTest.cs b/src/FrostFS.SDK.Tests/Unit/NetworkTest.cs new file mode 100644 index 0000000..b1146f7 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Unit/NetworkTest.cs @@ -0,0 +1,225 @@ +using System.Diagnostics.CodeAnalysis; + +using FrostFS.Netmap; +using FrostFS.SDK.Client; + +using Google.Protobuf; + +namespace FrostFS.SDK.Tests.Unit; + +[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] +public class NetworkTest : NetworkTestsBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async void NetworkSettingsTest(bool useContext) + { + Mocker.Parameters.Add("AuditFee", [1]); + Mocker.Parameters.Add("BasicIncomeRate", [2]); + Mocker.Parameters.Add("ContainerFee", [3]); + Mocker.Parameters.Add("ContainerAliasFee", [4]); + Mocker.Parameters.Add("EpochDuration", [5]); + Mocker.Parameters.Add("InnerRingCandidateFee", [6]); + Mocker.Parameters.Add("MaxECDataCount", [7]); + Mocker.Parameters.Add("MaxECParityCount", [8]); + Mocker.Parameters.Add("MaxObjectSize", [9]); + Mocker.Parameters.Add("WithdrawFee", [10]); + Mocker.Parameters.Add("HomomorphicHashingDisabled", [1]); + Mocker.Parameters.Add("MaintenanceModeAllowed", [1]); + + var ctx = useContext + ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) + : default; + + var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); + + var result = await GetClient(DefaultSettings).GetNetworkSettingsAsync(ctx); + + var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); + + Assert.NotNull(result); + + Assert.Equal(Mocker.Parameters["AuditFee"], [(byte)result.AuditFee]); + Assert.Equal(Mocker.Parameters["BasicIncomeRate"], [(byte)result.BasicIncomeRate]); + Assert.Equal(Mocker.Parameters["ContainerFee"], [(byte)result.ContainerFee]); + Assert.Equal(Mocker.Parameters["ContainerAliasFee"], [(byte)result.ContainerAliasFee]); + Assert.Equal(Mocker.Parameters["EpochDuration"], [(byte)result.EpochDuration]); + Assert.Equal(Mocker.Parameters["InnerRingCandidateFee"], [(byte)result.InnerRingCandidateFee]); + Assert.Equal(Mocker.Parameters["MaxECDataCount"], [(byte)result.MaxECDataCount]); + Assert.Equal(Mocker.Parameters["MaxECParityCount"], [(byte)result.MaxECParityCount]); + Assert.Equal(Mocker.Parameters["MaxObjectSize"], [(byte)result.MaxObjectSize]); + Assert.Equal(Mocker.Parameters["WithdrawFee"], [(byte)result.WithdrawFee]); + + Assert.True(result.HomomorphicHashingDisabled); + Assert.True(result.MaintenanceModeAllowed); + + if (useContext) + { + Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); + Assert.NotNull(Mocker.DateTime); + + Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); + Assert.True(Mocker.DateTime.Value <= validTimeoutTo); + } + else + { + Assert.NotNull(Mocker.NetworkInfoRequest); + Assert.Empty(Mocker.NetworkInfoRequest.MetaHeader.XHeaders); + Assert.Null(Mocker.DateTime); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + + public async void NetmapSnapshotTest(bool useContext) + { + var body = new NetmapSnapshotResponse.Types.Body + { + Netmap = new Netmap.Netmap { Epoch = 99 } + }; + + var nodeInfo1 = new NodeInfo + { + State = NodeInfo.Types.State.Online, + PublicKey = ByteString.CopyFrom([1, 2, 3]) + }; + + nodeInfo1.Addresses.Add("address1"); + nodeInfo1.Addresses.Add("address2"); + nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); + nodeInfo1.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); + + var nodeInfo2 = new NodeInfo + { + State = NodeInfo.Types.State.Offline, + PublicKey = ByteString.CopyFrom([3, 4, 5]) + }; + + nodeInfo2.Addresses.Add("address3"); + nodeInfo2.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key3", Value = "value3" }); + + body.Netmap.Nodes.Add(nodeInfo1); + body.Netmap.Nodes.Add(nodeInfo2); + + Mocker.NetmapSnapshotResponse = new NetmapSnapshotResponse { Body = body }; + + var ctx = useContext + ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) + : default; + + var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); + + var result = await GetClient(DefaultSettings).GetNetmapSnapshotAsync(ctx); + + var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); + + Assert.NotNull(result); + + Assert.Equal(99u, result.Epoch); + Assert.Equal(2, result.NodeInfoCollection.Count); + + var node1 = result.NodeInfoCollection[0]; + Assert.Equal(NodeState.Online, node1.State); + Assert.Equal(2, node1.Addresses.Count); + Assert.Equal("address1", node1.Addresses.ElementAt(0)); + Assert.Equal("address2", node1.Addresses.ElementAt(1)); + + Assert.Equal(2, node1.Attributes.Count); + + Assert.Equal("key1", node1.Attributes.ElementAt(0).Key); + Assert.Equal("value1", node1.Attributes.ElementAt(0).Value); + Assert.Equal("key2", node1.Attributes.ElementAt(1).Key); + Assert.Equal("value2", node1.Attributes.ElementAt(1).Value); + + var node2 = result.NodeInfoCollection[1]; + Assert.Equal(NodeState.Offline, node2.State); + Assert.Single(node2.Addresses); + Assert.Equal("address3", node2.Addresses.ElementAt(0)); + + Assert.Single(node2.Attributes); + + Assert.Equal("key3", node2.Attributes.ElementAt(0).Key); + Assert.Equal("value3", node2.Attributes.ElementAt(0).Value); + + if (useContext) + { + Assert.NotNull(Mocker.NetmapSnapshotRequest); + Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); + + Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); + Assert.NotNull(Mocker.DateTime); + Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); + Assert.True(Mocker.DateTime.Value <= validTimeoutTo); + } + else + { + Assert.NotNull(Mocker.NetmapSnapshotRequest); + Assert.Empty(Mocker.NetmapSnapshotRequest.MetaHeader.XHeaders); + Assert.Null(Mocker.DateTime); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async void NodeInfoTest(bool useContext) + { + var body = new LocalNodeInfoResponse.Types.Body + { + NodeInfo = new NodeInfo() + { + State = NodeInfo.Types.State.Online, + PublicKey = ByteString.CopyFrom([1, 2, 3]) + }, + Version = new Refs.Version { Major = 2, Minor = 12 } + }; + + body.NodeInfo.Addresses.Add("address1"); + body.NodeInfo.Addresses.Add("address2"); + body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); + body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); + + Mocker.NodeInfoResponse = new LocalNodeInfoResponse { Body = body }; + + var ctx = useContext + ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) + : default; + + var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); + + var result = await GetClient(DefaultSettings).GetNodeInfoAsync(ctx); + + var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); + + Assert.NotNull(result); + + Assert.Equal(NodeState.Online, result.State); + + Assert.Equal(2, result.Addresses.Count); + Assert.Equal("address1", result.Addresses.ElementAt(0)); + Assert.Equal("address2", result.Addresses.ElementAt(1)); + + Assert.Equal(2, result.Attributes.Count); + Assert.Equal("value1", result.Attributes["key1"]); + Assert.Equal("value2", result.Attributes["key2"]); + + Assert.NotNull(Mocker.LocalNodeInfoRequest); + if (useContext) + { + Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); + Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); + Assert.NotNull(Mocker.DateTime); + + Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); + Assert.True(Mocker.DateTime.Value <= validTimeoutTo); + } + else + { + Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); + Assert.Null(Mocker.DateTime); + } + } +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs b/src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs deleted file mode 100644 index 6449beb..0000000 --- a/src/FrostFS.SDK.Tests/Unit/NodeInfoTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -using FrostFS.Netmap; -using FrostFS.SDK.Client; -using Google.Protobuf; - -namespace FrostFS.SDK.Tests.Unit; - -public class NodeInfoTests : NetworkTestsBase -{ - [Theory] - [InlineData(false)] - [InlineData(true)] - public async void NodeInfoTest(bool useContext) - { - var body = new LocalNodeInfoResponse.Types.Body - { - NodeInfo = new NodeInfo() - { - State = NodeInfo.Types.State.Online, - PublicKey = ByteString.CopyFrom([1, 2, 3]) - }, - Version = new Refs.Version { Major = 2, Minor = 12 } - }; - - body.NodeInfo.Addresses.Add("address1"); - body.NodeInfo.Addresses.Add("address2"); - body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key1", Value = "value1" }); - body.NodeInfo.Attributes.Add(new NodeInfo.Types.Attribute { Key = "key2", Value = "value2" }); - - Mocker.NodeInfoResponse = new LocalNodeInfoResponse { Body = body }; - - var ctx = useContext - ? new CallContext(TimeSpan.FromSeconds(20), Mocker.CancellationTokenSource.Token) - : default; - - var validTimeoutFrom = DateTime.UtcNow.AddSeconds(20); - - var result = await GetClient(DefaultSettings).GetNodeInfoAsync(ctx); - - var validTimeoutTo = DateTime.UtcNow.AddSeconds(20); - - Assert.NotNull(result); - - Assert.Equal(NodeState.Online, result.State); - - Assert.Equal(2, result.Addresses.Count); - Assert.Equal("address1", result.Addresses.ElementAt(0)); - Assert.Equal("address2", result.Addresses.ElementAt(1)); - - Assert.Equal(2, result.Attributes.Count); - Assert.Equal("value1", result.Attributes["key1"]); - Assert.Equal("value2", result.Attributes["key2"]); - - Assert.NotNull(Mocker.LocalNodeInfoRequest); - if (useContext) - { - Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); - Assert.Equal(Mocker.CancellationTokenSource.Token, Mocker.CancellationToken); - Assert.NotNull(Mocker.DateTime); - - Assert.True(Mocker.DateTime.Value >= validTimeoutFrom); - Assert.True(Mocker.DateTime.Value <= validTimeoutTo); - } - else - { - Assert.Empty(Mocker.LocalNodeInfoRequest.MetaHeader.XHeaders); - Assert.Null(Mocker.DateTime); - } - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs index 985f02e..479c6ba 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTest.cs @@ -6,9 +6,8 @@ using System.Text; using FrostFS.Refs; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; -using FrostFS.SDK.Cryptography; + using Google.Protobuf; -using Org.BouncyCastle.Utilities; namespace FrostFS.SDK.Tests.Unit; @@ -61,7 +60,7 @@ public class ObjectTest : ObjectTestsBase var param = new PrmObjectClientCutPut( Mocker.ObjectHeader, payload: new MemoryStream(bytes), - bufferMaxSize: blockSize); + bufferMaxSize: 1024); Random rnd = new(); @@ -88,8 +87,8 @@ public class ObjectTest : ObjectTestsBase // PART1 Assert.Equal(blockSize, objects[0].Payload.Length); - Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); - + Assert.True(bytes.AsMemory(0, blockSize).ToArray().SequenceEqual(objects[0].Payload)); + Assert.NotNull(objects[0].Header.Split.SplitId); Assert.Null(objects[0].Header.Split.Previous); Assert.True(objects[0].Header.Attributes.Count == 0); @@ -97,7 +96,7 @@ public class ObjectTest : ObjectTestsBase // PART2 Assert.Equal(blockSize, objects[1].Payload.Length); - Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); + Assert.True(bytes.AsMemory(blockSize, blockSize).ToArray().SequenceEqual(objects[1].Payload)); Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); Assert.True(objects[1].Header.Attributes.Count == 0); @@ -105,7 +104,7 @@ public class ObjectTest : ObjectTestsBase // last part Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); - Assert.Equal(bytes.AsMemory(2 * blockSize).ToArray(), objects[2].Payload); + Assert.True(bytes.AsMemory(2*blockSize).ToArray().SequenceEqual(objects[2].Payload)); Assert.NotNull(objects[3].Header.Split.Parent); Assert.NotNull(objects[3].Header.Split.ParentHeader); @@ -129,351 +128,6 @@ public class ObjectTest : ObjectTestsBase Assert.Equal(result.Value, modelObjId.ToString()); } - [Fact] - public async void ClientCutWithInterruptionOnFirstPartTest() - { - NetworkMocker.Parameters.Add("MaxObjectSize", [0x0, 0xa]); - - var blockSize = 2560; - byte[] bytes = File.ReadAllBytes(@".\..\..\..\cat.jpg"); - var fileLength = bytes.Length; - - var splitId = Guid.NewGuid(); - var progress = new UploadProgressInfo(splitId); - - var param = new PrmObjectClientCutPut( - Mocker.ObjectHeader, - payload: new MemoryStream(bytes), - bufferMaxSize: blockSize, - progress: progress); - - Random rnd = new(); - - Collection objIds = new([new byte[32], new byte[32], new byte[32]]); - rnd.NextBytes(objIds.ElementAt(0)); - rnd.NextBytes(objIds.ElementAt(1)); - rnd.NextBytes(objIds.ElementAt(2)); - - foreach (var objId in objIds) - Mocker.ResultObjectIds!.Add(objId); - - int sentBlockCount = 0; - Mocker.Callback = () => - { - if (++sentBlockCount == 1) - throw new FrostFsException("some error"); - }; - - bool gotException = false; - try - { - var result = await GetClient().PutClientCutObjectAsync(param, default); - } - catch (FrostFsException ex) - { - if (ex.Message == "some error") - gotException = true; - } - - Assert.True(gotException); - - var singleObjects = Mocker.PutSingleRequests.ToArray(); - - Assert.Empty(singleObjects); - } - - [Fact] - public async void ClientCutWithInterruptionOnMiddlePartTest() - { - NetworkMocker.Parameters.Add("MaxObjectSize", [0x0, 0xa]); - - var blockSize = 2560; - byte[] bytes = File.ReadAllBytes(@".\..\..\..\cat.jpg"); - var fileLength = bytes.Length; - - var splitId = Guid.Parse("67e4bbe9-86ca-474d-9385-6569ce89db61"); - var progress = new UploadProgressInfo(splitId); - - var param = new PrmObjectClientCutPut( - Mocker.ObjectHeader, - payload: new MemoryStream(bytes), - bufferMaxSize: blockSize, - progress: progress); - - Random rnd = new(); - - Collection objIds = new([new byte[32], new byte[32], new byte[32]]); - rnd.NextBytes(objIds.ElementAt(0)); - rnd.NextBytes(objIds.ElementAt(1)); - rnd.NextBytes(objIds.ElementAt(2)); - - foreach (var objId in objIds) - Mocker.ResultObjectIds!.Add(objId); - - int sentBlockCount = 0; - Mocker.Callback = () => - { - if (++sentBlockCount == 2) - throw new FrostFsException("some error"); - }; - - bool gotException = false; - try - { - _ = await GetClient().PutClientCutObjectAsync(param, default); - } - catch (FrostFsException ex) - { - if (ex.Message == "some error") - gotException = true; - } - - Assert.True(gotException); - - var singleObjects = Mocker.PutSingleRequests.ToArray(); - - Assert.NotNull(Mocker.ClientStreamWriter?.Messages); - - var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); - - Assert.Single(objects); - - Assert.Single(progress.GetParts()); - - var part = progress.GetPart(0); - Assert.Equal(0, part.Offset); - Assert.Equal(2560, part.Length); - - // PART1 - Assert.Equal(blockSize, objects[0].Payload.Length); - Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); - - Assert.NotNull(objects[0].Header.Split.SplitId); - Assert.Null(objects[0].Header.Split.Previous); - Assert.True(objects[0].Header.Attributes.Count == 0); - Assert.Null(objects[0].Header.Split.Parent); - - // resume uploading - sentBlockCount = 10; - - var result = await GetClient().PutClientCutObjectAsync(param, default); - - singleObjects = Mocker.PutSingleRequests.ToArray(); - - Assert.NotNull(Mocker.ClientStreamWriter?.Messages); - - objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); - - Assert.Equal(4, objects.Length); - - // PART1 - Assert.Equal(blockSize, objects[0].Payload.Length); - Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); - - Assert.NotNull(objects[0].Header.Split.SplitId); - Assert.Null(objects[0].Header.Split.Previous); - Assert.True(objects[0].Header.Attributes.Count == 0); - Assert.Null(objects[0].Header.Split.Parent); - - // PART2 - Assert.Equal(blockSize, objects[1].Payload.Length); - Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); - - Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); - Assert.True(objects[1].Header.Attributes.Count == 0); - Assert.Null(objects[1].Header.Split.Parent); - - // last part - Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); - Assert.Equal(bytes.AsMemory(2 * blockSize).ToArray(), objects[2].Payload); - - Assert.NotNull(objects[3].Header.Split.Parent); - Assert.NotNull(objects[3].Header.Split.ParentHeader); - Assert.NotNull(objects[3].Header.Split.ParentSignature); - Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); - Assert.True(objects[2].Header.Attributes.Count == 0); - - // link object - Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent); - Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader); - Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); - Assert.Equal(0, (int)objects[3].Header.PayloadLength); - Assert.True(objects[3].Header.Attributes.Count == 0); - - Assert.Single(objects[3].Header.Split.ParentHeader.Attributes); - Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key); - Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value); - - var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray()); - - Assert.Equal(result.Value, modelObjId.ToString()); - - Assert.Equal(3, progress.GetParts().Count); - part = progress.GetPart(0); - Assert.Equal(0, part.Offset); - Assert.Equal(2560, part.Length); - - part = progress.GetPart(1); - Assert.Equal(2560, part.Offset); - Assert.Equal(2560, part.Length); - - part = progress.GetPart(2); - Assert.Equal(2 * 2560, part.Offset); - Assert.Equal(1620, part.Length); - } - - [Fact] - public async void ClientCutWithInterruptionOnLastPartTest() - { - NetworkMocker.Parameters.Add("MaxObjectSize", [0x0, 0xa]); - - var blockSize = 2560; - byte[] bytes = File.ReadAllBytes(@".\..\..\..\cat.jpg"); - var fileLength = bytes.Length; - - var splitId = Guid.Parse("67e4bbe9-86ca-474d-9385-6569ce89db61"); - var progress = new UploadProgressInfo(splitId); - - var param = new PrmObjectClientCutPut( - Mocker.ObjectHeader, - payload: new MemoryStream(bytes), - bufferMaxSize: blockSize, - progress: progress); - - Random rnd = new(); - - Collection objIds = new([new byte[32], new byte[32], new byte[32]]); - rnd.NextBytes(objIds.ElementAt(0)); - rnd.NextBytes(objIds.ElementAt(1)); - rnd.NextBytes(objIds.ElementAt(2)); - - foreach (var objId in objIds) - Mocker.ResultObjectIds!.Add(objId); - - int sentBlockCount = 0; - Mocker.Callback = () => - { - if (++sentBlockCount == 3) - throw new FrostFsException("some error"); - }; - - bool gotException = false; - try - { - _ = await GetClient().PutClientCutObjectAsync(param, default); - } - catch (FrostFsException ex) - { - if (ex.Message == "some error") - gotException = true; - } - - Assert.True(gotException); - - var singleObjects = Mocker.PutSingleRequests.ToArray(); - - Assert.NotNull(Mocker.ClientStreamWriter?.Messages); - - var objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); - - Assert.Equal(2, objects.Length); - - Assert.Equal(2, progress.GetParts().Count); - - var part = progress.GetPart(0); - Assert.Equal(0, part.Offset); - Assert.Equal(2560, part.Length); - - part = progress.GetPart(1); - Assert.Equal(2560, part.Offset); - Assert.Equal(2560, part.Length); - - // PART1 - Assert.Equal(blockSize, objects[0].Payload.Length); - Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); - - Assert.NotNull(objects[0].Header.Split.SplitId); - Assert.Null(objects[0].Header.Split.Previous); - Assert.True(objects[0].Header.Attributes.Count == 0); - Assert.Null(objects[0].Header.Split.Parent); - - // PART2 - Assert.Equal(blockSize, objects[1].Payload.Length); - Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); - - Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); - Assert.True(objects[1].Header.Attributes.Count == 0); - Assert.Null(objects[1].Header.Split.Parent); - - // resume uploading - sentBlockCount = 10; - - var result = await GetClient().PutClientCutObjectAsync(param, default); - - singleObjects = Mocker.PutSingleRequests.ToArray(); - - Assert.NotNull(Mocker.ClientStreamWriter?.Messages); - - objects = Mocker.PutSingleRequests.Select(o => o.Body.Object).ToArray(); - - Assert.Equal(4, objects.Length); - - // PART1 - Assert.Equal(blockSize, objects[0].Payload.Length); - Assert.Equal(bytes.AsMemory(0, blockSize).ToArray(), objects[0].Payload); - - Assert.NotNull(objects[0].Header.Split.SplitId); - Assert.Null(objects[0].Header.Split.Previous); - Assert.True(objects[0].Header.Attributes.Count == 0); - Assert.Null(objects[0].Header.Split.Parent); - - // PART2 - Assert.Equal(blockSize, objects[1].Payload.Length); - Assert.Equal(bytes.AsMemory(blockSize, blockSize).ToArray(), objects[1].Payload); - - Assert.Equal(objects[0].Header.Split.SplitId, objects[1].Header.Split.SplitId); - Assert.True(objects[1].Header.Attributes.Count == 0); - Assert.Null(objects[1].Header.Split.Parent); - - // last part - Assert.Equal(bytes.Length % blockSize, objects[2].Payload.Length); - Assert.Equal(bytes.AsMemory(2 * blockSize).ToArray(), objects[2].Payload); - - Assert.NotNull(objects[3].Header.Split.Parent); - Assert.NotNull(objects[3].Header.Split.ParentHeader); - Assert.NotNull(objects[3].Header.Split.ParentSignature); - Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); - Assert.True(objects[2].Header.Attributes.Count == 0); - - // link object - Assert.Equal(objects[2].Header.Split.Parent, objects[3].Header.Split.Parent); - Assert.Equal(objects[2].Header.Split.ParentHeader, objects[3].Header.Split.ParentHeader); - Assert.Equal(objects[2].Header.Split.SplitId, objects[3].Header.Split.SplitId); - Assert.Equal(0, (int)objects[3].Header.PayloadLength); - Assert.True(objects[3].Header.Attributes.Count == 0); - - Assert.Single(objects[3].Header.Split.ParentHeader.Attributes); - Assert.Equal("k", objects[3].Header.Split.ParentHeader.Attributes[0].Key); - Assert.Equal("v", objects[3].Header.Split.ParentHeader.Attributes[0].Value); - - var modelObjId = FrostFsObjectId.FromHash(objects[3].Header.Split.Parent.Value.ToByteArray()); - - Assert.Equal(result.Value, modelObjId.ToString()); - - Assert.Equal(3, progress.GetParts().Count); - part = progress.GetPart(0); - Assert.Equal(0, part.Offset); - Assert.Equal(2560, part.Length); - - part = progress.GetPart(1); - Assert.Equal(2560, part.Offset); - Assert.Equal(2560, part.Length); - - part = progress.GetPart(2); - Assert.Equal(2 * 2560, part.Offset); - Assert.Equal(1620, part.Length); - } - [Fact] public async void DeleteObject() { diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs index c3681ce..18e3981 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs @@ -35,7 +35,7 @@ public abstract class ObjectTestsBase ContainerGuid = Guid.NewGuid() }; - ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToByteArray(true))); + ContainerId = new FrostFsContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes())); Mocker.ObjectHeader = new( ContainerId, diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs b/src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs deleted file mode 100644 index 3c0245b..0000000 --- a/src/FrostFS.SDK.Tests/Unit/ObjectToolsTests.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Security.Cryptography; -using System.Text; -using FrostFS.SDK.Client; -using FrostFS.SDK.Cryptography; - -namespace FrostFS.SDK.Tests.Unit; - -public class ObjectToolsTests -{ - internal readonly string key = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; - - [Fact] - public void CalculateObjectIdTest() - { - var payload = Encoding.UTF8.GetBytes("testPayload"); - - var payloadHash = SHA256.HashData(payload); - - FrostFsContainerId containerId = new("test"); - FrostFsObjectHeader header = new(containerId); - - var ecdsaKey = key.LoadWif(); - var owner = FrostFsOwner.FromKey(ecdsaKey); - - var clientKey = new ClientKey(ecdsaKey); - - var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); - - Assert.NotNull(objId.Value); - Assert.Equal("HuAojwCYi62iUKr1FtSCCkMLLWv1uAnznF8iSb1bRV1N", objId.Value); - } - - [Fact] - public void CalculateObjectIdTest1() - { - var payload = Encoding.UTF8.GetBytes("testPayload"); - - var payloadHash = SHA256.HashData(payload); - var ecdsaKey = key.LoadWif(); - var owner = FrostFsOwner.FromKey(ecdsaKey); - - var clientKey = new ClientKey(ecdsaKey); - FrostFsContainerId containerId = new("test"); - FrostFsObjectHeader header = new(containerId, FrostFsObjectType.Regular, null, null, owner, new FrostFsVersion(2, 13)); - - var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); - - Assert.NotNull(objId.Value); - Assert.Equal("HuAojwCYi62iUKr1FtSCCkMLLWv1uAnznF8iSb1bRV1N", objId.Value); - } - - [Fact] - public void CalculateObjectIdWithAttrTest() - { - var payload = Encoding.UTF8.GetBytes("testPayload"); - - var payloadHash = SHA256.HashData(payload); - var ecdsaKey = key.LoadWif(); - var owner = FrostFsOwner.FromKey(ecdsaKey); - - var clientKey = new ClientKey(ecdsaKey); - FrostFsContainerId containerId = new("test"); - - FrostFsAttributePair[] attribs = [new("key", "val")]; - - FrostFsObjectHeader header = new(containerId, FrostFsObjectType.Regular, attribs, null, owner, new FrostFsVersion(2, 13)); - - var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); - - Assert.NotNull(objId.Value); - Assert.Equal("4zq5NYEbzkrfmdKne3GnpavE24gU2PnuV17ZExb9hcn3", objId.Value); - } - - [Fact] - public void CalculateObjectIdWithSplitIdTest() - { - var payload = Encoding.UTF8.GetBytes("testPayload"); - - var payloadHash = SHA256.HashData(payload); - var ecdsaKey = key.LoadWif(); - var owner = FrostFsOwner.FromKey(ecdsaKey); - - var clientKey = new ClientKey(ecdsaKey); - FrostFsContainerId containerId = new("test"); - - FrostFsAttributePair[] attribs = [new("key", "val")]; - - var guid = Guid.Parse("790a8d04-f5c3-4cd6-b46f-a78ee7e325f2"); - SplitId splitId = new(guid); - FrostFsSplit split = new (splitId); - - FrostFsObjectHeader header = new(containerId, FrostFsObjectType.Regular, attribs, split, owner, new FrostFsVersion(2, 13)); - - var objId = ObjectTools.CalculateObjectId(header, payloadHash, owner, new FrostFsVersion(2, 13), clientKey); - - Assert.NotNull(objId.Value); - Assert.Equal("HCYzsuXyfe5LmQzi58hPQxExGPAFv7dU5TzEACLxM1os", objId.Value); - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs b/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs deleted file mode 100644 index 46f7a3d..0000000 --- a/src/FrostFS.SDK.Tests/Unit/PlacementPolicyTests.cs +++ /dev/null @@ -1,307 +0,0 @@ -using System.Xml.Linq; -using FrostFS.Netmap; -using FrostFS.SDK.Client; -using Google.Protobuf.WellKnownTypes; - -namespace FrostFS.SDK.Tests.Unit; - -public class PlacementPolicyTests : NetworkTestsBase -{ - [Theory] - [InlineData(true, 1)] - [InlineData(true, 3)] - [InlineData(true, 5)] - [InlineData(false, 1)] - [InlineData(false, 3)] - [InlineData(false, 5)] - public void PlacementPolicySimpleFullTest(bool unique, uint backupFactor) - { - PlacementPolicy policy = new() - { - ContainerBackupFactor = backupFactor, - Unique = unique - }; - - var result = policy.ToModel(); - - Assert.Equal(backupFactor, result.BackupFactor); - Assert.Equal(unique, result.Unique); - Assert.Empty(result.Filters); - Assert.Empty(result.Replicas); - Assert.Empty(result.Selectors); - } - - [Fact] - public void PlacementPolicyFullTest() - { - PlacementPolicy policy = new() - { - ContainerBackupFactor = 3, - Unique = true - }; - - policy.Filters.AddRange( - [ - new () { Name = "filter1", Key = "filterKey1", Op =Operation.Eq, Value = "testValue1" }, - new () { Name = "filter2", Key = "filterKey2", Op =Operation.And, Value = "testValue2" } - ]); - - policy.Selectors.AddRange( - [ - new () { Name = "name1", Attribute = "attrib1", Clause = Clause.Same, Count = 5, Filter = "filter1" }, - new () { Name = "name2", Attribute = "attrib2", Clause = Clause.Distinct, Count = 4, Filter = "filter2" } - ]); - - policy.Replicas.AddRange( - [ - new () { EcDataCount = 2, EcParityCount = 3, Count = 4, Selector = "selector1"}, - new () { EcDataCount = 5, EcParityCount = 6, Count = 7, Selector = "selector2"}, - ]); - - var result = policy.ToModel(); - - Assert.Equal(3L, result.BackupFactor); - Assert.True(result.Unique); - Assert.Equal(2, result.Filters.Count); - Assert.Equal(2, result.Replicas.Length); - Assert.Equal(2, result.Selectors.Count); - - var rep0 = result.Replicas[0]; - Assert.Equal(2u, rep0.EcDataCount); - Assert.Equal(3u, rep0.EcParityCount); - Assert.Equal(4u, rep0.Count); - Assert.Equal("selector1", rep0.Selector); - - var rep1 = result.Replicas[1]; - Assert.Equal(5u, rep1.EcDataCount); - Assert.Equal(6u, rep1.EcParityCount); - Assert.Equal(7u, rep1.Count); - Assert.Equal("selector2", rep1.Selector); - - var f0 = result.Filters[0]; - Assert.Equal("filterKey1", f0.Key); - Assert.Equal("filter1", f0.Name); - Assert.Equal(1, f0.Operation); - Assert.Equal("testValue1", f0.Value); - - var f1 = result.Filters[1]; - Assert.Equal("filterKey2", f1.Key); - Assert.Equal("filter2", f1.Name); - Assert.Equal(8, f1.Operation); - Assert.Equal("testValue2", f1.Value); - - var s0 = result.Selectors[0]; - Assert.Equal("name1", s0.Name); - Assert.Equal("attrib1", s0.Attribute); - Assert.Equal(1, s0.Clause); - Assert.Equal(5L, s0.Count); - Assert.Equal("filter1", s0.Filter); - - var s1 = result.Selectors[1]; - Assert.Equal("name2", s1.Name); - Assert.Equal("attrib2", s1.Attribute); - Assert.Equal(2, s1.Clause); - Assert.Equal(4L, s1.Count); - Assert.Equal("filter2", s1.Filter); - } - - - [Theory] - [InlineData(1, "test" , 0, 0)] - [InlineData(1, "", 1000, 9999)] - [InlineData(1, "some long text to test reasonable length of the selector name", 100000000, 100000001)] - [InlineData(100, "test2", 1, 1)] - [InlineData(1, " ", 2, 3)] - [InlineData(10, "!", 0, 0)] - [InlineData(1, "123", 0, 0)] - public void ReplicaToModelTest(uint count, string selector, uint ecDataCount, uint ecParityCount) - { - Replica replica = new () - { - Count = count, - Selector = selector, - EcDataCount = ecDataCount, - EcParityCount = ecParityCount - }; - - FrostFsReplica model = replica.ToModel(); - - Assert.Equal(count, model.Count); - Assert.Equal(selector, model.Selector); - Assert.Equal(ecDataCount, model.EcDataCount); - Assert.Equal(ecParityCount, model.EcParityCount); - } - - [Theory] - [InlineData(1, "test", 0, 0)] - [InlineData(1, "", 1000, 9999)] - [InlineData(1, "some long text to test reasonable length of the selector name", 100000000, 100000001)] - [InlineData(100, "test2", 1, 1)] - [InlineData(1, " ", 2, 3)] - [InlineData(10, "!", 0, 0)] - [InlineData(1, "123", 0, 0)] - public void ReplicaToMessagelTest(uint count, string selector, uint ecDataCount, uint ecParityCount) - { - FrostFsReplica replica = new () - { - Count = count, - Selector = selector, - EcDataCount = ecDataCount, - EcParityCount = ecParityCount - }; - - Replica message = replica.ToMessage(); - - Assert.Equal(count, message.Count); - Assert.Equal(selector, message.Selector); - Assert.Equal(ecDataCount, message.EcDataCount); - Assert.Equal(ecParityCount, message.EcParityCount); - } - - [Theory] - [InlineData("test", 1, 2, "attribute", "filter")] - [InlineData("test", 0, 0, "longlonglonglonglonglonglonglonglonglonglonglonglong attribute", "longlonglonglonglonglonglonglonglonglonglonglonglong filter")] - [InlineData("test", 0, 1, "attribute", "filter")] - public void SelectorToMessageTest(string name, uint count, int clause, string attr, string filter) - { - FrostFsSelector selector = new (name) - { - Count = count, - Clause = clause, - Attribute = attr, - Filter = filter, - }; - - var message = selector.ToMessage(); - - Assert.Equal(name, message.Name); - Assert.Equal(count, message.Count); - Assert.Equal(clause, (int)message.Clause); - Assert.Equal(attr, message.Attribute); - Assert.Equal(filter, message.Filter); - - } - - [Theory] - [InlineData("test", 1, Clause.Same, "attribute", "filter")] - [InlineData("test", 0, Clause.Distinct, "longlonglonglonglonglonglonglonglonglonglonglonglong attribute", "longlonglonglonglonglonglonglonglonglonglonglonglong filter")] - public void SelectorToModelTest(string name, uint count, Clause clause, string attr, string filter) - { - Selector selector = new () - { - Name = name, - Count = count, - Clause = clause, - Attribute = attr, - Filter = filter - }; - - var model = selector.ToModel(); - - Assert.Equal(name, model.Name); - Assert.Equal(count, model.Count); - Assert.Equal((int)clause, model.Clause); - Assert.Equal(attr, model.Attribute); - Assert.Equal(filter, model.Filter); - } - - [Theory] - [InlineData("", "", 1, "")] - [InlineData("name", "key", 1, "val")] - [InlineData("longlonglonglonglonglonglonglonglonglonglonglonglong name", "longlonglonglonglonglonglonglonglonglonglonglonglong key", 10, "longlonglonglonglonglonglonglonglonglonglonglonglong val")] - public void FilterToMessageTest(string name, string key, int operation, string value) - { - FrostFsFilter filter = new (name, key, operation, value, []); - - var message = filter.ToMessage(); - - Assert.Equal(name, message.Name); - Assert.Equal(key, message.Key); - Assert.Equal(operation, (int)message.Op); - Assert.Equal(value, message.Value); - } - - [Theory] - [InlineData("", "", 1, "")] - [InlineData("name", "key", 2, "val")] - [InlineData("longlonglonglonglonglonglonglonglonglonglonglonglong name", "longlonglonglonglonglonglonglonglonglonglonglonglong key", 10, "longlonglonglonglonglonglonglonglonglonglonglonglong val")] - public void SubFilterToMessageTest(string name, string key, int operation, string value) - { - FrostFsFilter subFilter = new(name, key, operation, value, []); - - FrostFsFilter filter = new("name", "key", 1, "value", [subFilter]); - - var message = filter.ToMessage(); - - Assert.Single(message.Filters); - - var grpcFilter = message.Filters[0]; - Assert.Equal(name, grpcFilter.Name); - Assert.Equal(key, grpcFilter.Key); - Assert.Equal(operation, (int)grpcFilter.Op); - Assert.Equal(value, grpcFilter.Value); - } - - [Fact] - public void SubFiltersToMessageTest() - { - string[] names = ["", "name1", "some pretty long name for name test"]; - string[] keys = ["", "key1", "some pretty long key for name test"]; - int[] operations = [1, 2, 10]; - string[] values = ["", "val1", "some pretty long value for name test"]; - - var subFilter = new FrostFsFilter[3]; - - for (int i = 0; i < 3; i++) - { - subFilter[i] = new FrostFsFilter(names[i], keys[i], operations[i], values[i], []); - } - - FrostFsFilter filter = new("name", "key", 1, "value", subFilter); - - var message = filter.ToMessage(); - - Assert.Equal(3, message.Filters.Count); - - for (int i = 0; i < 3; i++) - { - var grpcFilter = message.Filters[i]; - Assert.Equal(names[i], grpcFilter.Name); - Assert.Equal(keys[i], grpcFilter.Key); - Assert.Equal(operations[i], (int)grpcFilter.Op); - Assert.Equal(values[i], grpcFilter.Value); - } - } - - [Theory] - [InlineData("", "", Operation.Unspecified, "")] - [InlineData("name", "key", Operation.Unspecified, "val")] - [InlineData("name", "key", Operation.And, "val")] - [InlineData("name", "key", Operation.Eq, "val")] - [InlineData("name", "key", Operation.Le, "val")] - [InlineData("name", "key", Operation.Like, "val")] - [InlineData("name", "key", Operation.Ge, "val")] - [InlineData("name", "key", Operation.Gt, "val")] - [InlineData("name", "key", Operation.Lt, "val")] - [InlineData("name", "key", Operation.Ne, "val")] - [InlineData("name", "key", Operation.Not, "val")] - [InlineData("name", "key", Operation.Or, "val")] - [InlineData("longlonglonglonglonglonglonglonglonglonglonglonglong name", "longlonglonglonglonglonglonglonglonglonglonglonglong key", Operation.Like, "longlonglonglonglonglonglonglonglonglonglonglonglong val")] - public void FrostFsFilterToModelTest(string name, string key, Operation operation, string value) - { - Filter filter = new() - { - Name = name, - Key = key, - Op = operation, - Value = value - }; - - var model = filter.ToModel(); - - Assert.Equal(name, model.Name); - Assert.Equal(key, model.Key); - Assert.Equal((int)operation, model.Operation); - Assert.Equal(value, model.Value); - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs b/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs index 9833ae1..fdf7ffe 100644 --- a/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs @@ -248,12 +248,12 @@ public class FilterDto Key ?? string.Empty, (int)Op, Value ?? string.Empty, - Filters != null ? [.. Filters.Select(f => f.Filter)] : []); + Filters != null ? Filters.Select(f => f.Filter).ToArray() : []); } public class ReplicaDto { - public uint Count { get; set; } + public int Count { get; set; } public string? Selector { get; set; } } diff --git a/src/FrostFS.SDK.Tests/Unit/SessionTests.cs b/src/FrostFS.SDK.Tests/Unit/SessionTests.cs index be2e072..b2f1b78 100644 --- a/src/FrostFS.SDK.Tests/Unit/SessionTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/SessionTests.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using FrostFS.SDK.Client; using FrostFS.SDK.Client.Mappers.GRPC; +using FrostFS.SDK.Cryptography; namespace FrostFS.SDK.Tests.Unit; @@ -37,7 +38,7 @@ public class SessionTest : SessionTestsBase Assert.NotNull(result); Assert.NotEqual(Guid.Empty, result.Id); - Assert.Equal(Mocker.SessionId, result.Id.ToByteArray(true)); + Assert.Equal(Mocker.SessionId, result.Id.ToBytes()); Assert.Equal(Mocker.SessionKey, result.SessionKey.ToArray()); //Assert.Equal(OwnerId.ToMessage(), result.Token.Body.OwnerId); diff --git a/src/FrostFS.SDK.Tests/Unit/SignatureTests.cs b/src/FrostFS.SDK.Tests/Unit/SignatureTests.cs deleted file mode 100644 index 5062319..0000000 --- a/src/FrostFS.SDK.Tests/Unit/SignatureTests.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System.Text; -using FrostFS.SDK.Client; -using FrostFS.SDK.Client.Mappers.GRPC; - -namespace FrostFS.SDK.Tests.Unit; - -public class SignatureTests -{ - [Theory] - [InlineData(Refs.SignatureScheme.EcdsaSha512)] - [InlineData(Refs.SignatureScheme.EcdsaRfc6979Sha256)] - [InlineData(Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect)] - - public void SignatureToMessageTest(Refs.SignatureScheme scheme) - { - var key = Encoding.UTF8.GetBytes("datafortest"); - var sign = Encoding.UTF8.GetBytes("signdatafortest"); - - var frostFsScheme = scheme switch - { - Refs.SignatureScheme.EcdsaRfc6979Sha256 => SignatureScheme.EcdsaRfc6979Sha256, - Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect => SignatureScheme.EcdsaRfc6979Sha256WalletConnect, - Refs.SignatureScheme.EcdsaSha512 => SignatureScheme.EcdsaSha512 - }; - - FrostFsSignature signature = new() - { - Key = key, - Scheme = frostFsScheme, - Sign = sign - }; - - var result = signature.ToMessage(); - - Assert.Equal(scheme, result.Scheme); - Assert.Equal(sign, result.Sign.ToByteArray()); - Assert.Equal(key, result.Key.ToByteArray()); - } -} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/WalletTests.cs b/src/FrostFS.SDK.Tests/Unit/WalletTests.cs deleted file mode 100644 index 2acd77e..0000000 --- a/src/FrostFS.SDK.Tests/Unit/WalletTests.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Text; -using FrostFS.SDK.Client; - -namespace FrostFS.SDK.Tests.Unit; - -[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -public class WalletTest : SessionTestsBase -{ - [Fact] - public void TestWallet() - { - var password = Encoding.UTF8.GetBytes(""); - - var d = Directory.GetCurrentDirectory(); - var path = ".\\..\\..\\..\\TestData\\wallet.json"; - Assert.True(File.Exists(path)); - - var content = File.ReadAllText(path); - - var wif = WalletTools.GetWifFromWallet(content, password); - - Assert.Equal("KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK", wif); - } -} \ No newline at end of file