Compare commits
12 commits
Author | SHA1 | Date | |
---|---|---|---|
0816be732a | |||
eebba7665b | |||
20586d8ada | |||
f099edb17b | |||
764b669295 | |||
c88eea1f82 | |||
5f451c8818 | |||
42f4507638 | |||
b390778201 | |||
30af614558 | |||
45e73a6f8e | |||
87fe8db674 |
116 changed files with 2578 additions and 4092 deletions
|
@ -1,5 +1,4 @@
|
||||||
on:
|
on:
|
||||||
push:
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
@ -16,7 +15,7 @@ jobs:
|
||||||
# `dotnet build` implies and replaces `dotnet pack` thanks to `GeneratePackageOnBuild`
|
# `dotnet build` implies and replaces `dotnet pack` thanks to `GeneratePackageOnBuild`
|
||||||
run: dotnet build
|
run: dotnet build
|
||||||
|
|
||||||
- name: Publish NuGet packages
|
- name: Publish unsigned NuGet packages
|
||||||
run: |-
|
run: |-
|
||||||
dotnet nuget add source \
|
dotnet nuget add source \
|
||||||
--name "$NUGET_REGISTRY" \
|
--name "$NUGET_REGISTRY" \
|
||||||
|
|
5
.gitignore
vendored
5
.gitignore
vendored
|
@ -32,5 +32,8 @@ antlr-*.jar
|
||||||
|
|
||||||
# binary
|
# binary
|
||||||
bin/
|
bin/
|
||||||
release/
|
|
||||||
obj/
|
obj/
|
||||||
|
|
||||||
|
# Repository signing keys
|
||||||
|
release/maintainer.*
|
||||||
|
release/ca.*
|
||||||
|
|
58
Makefile
Normal file
58
Makefile
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
DOTNET?=dotnet
|
||||||
|
DOCKER?=docker
|
||||||
|
|
||||||
|
NUGET_REGISTRY?=TrueCloudLab
|
||||||
|
NUGET_REGISTRY_URL?=https://git.frostfs.info/api/packages/TrueCloudLab/nuget/index.json
|
||||||
|
NUGET_REGISTRY_USER?=
|
||||||
|
NUGET_REGISTRY_PASSWORD?=
|
||||||
|
|
||||||
|
NUPKG=find -iname '*.nupkg' | grep . | xargs -d'\n' -t -r -n1
|
||||||
|
RFC3161_TSA?=http://timestamp.digicert.com
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: build
|
||||||
|
build:
|
||||||
|
$(DOTNET) build
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: sign
|
||||||
|
sign: export NUGET_CERT_REVOCATION_MODE=offline
|
||||||
|
sign: release/maintainer.pfx
|
||||||
|
$(NUPKG) $(DOTNET) nuget sign --overwrite --certificate-path $< --timestamper "$(RFC3161_TSA)"
|
||||||
|
@rm -v "$<" # maintainer.pfx is not password protected and must be ephemeral
|
||||||
|
$(NUPKG) $(DOTNET) nuget verify
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: publish
|
||||||
|
publish:
|
||||||
|
$(NUPKG) $(DOTNET) nuget verify
|
||||||
|
$(NUPKG) $(DOTNET) nuget push --source "$(NUGET_REGISTRY)"
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: nuget-registry
|
||||||
|
nuget-registry:
|
||||||
|
ifeq (,$(NUGET_REGISTRY_USER))
|
||||||
|
$(error NUGET_REGISTRY_USER not set)
|
||||||
|
endif
|
||||||
|
ifeq (,$(NUGET_REGISTRY_PASSWORD))
|
||||||
|
$(error NUGET_REGISTRY_PASSWORD not set)
|
||||||
|
endif
|
||||||
|
$(DOTNET) nuget add source \
|
||||||
|
--name "$(NUGET_REGISTRY)" \
|
||||||
|
--username "$(NUGET_REGISTRY_USER)" \
|
||||||
|
--password "$(NUGET_REGISTRY_PASSWORD)" \
|
||||||
|
--store-password-in-clear-text \
|
||||||
|
"$(NUGET_REGISTRY_URL)"
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
|
clean:
|
||||||
|
-$(NUPKG) rm -v
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: container
|
||||||
|
container:
|
||||||
|
$(DOCKER) run --pull=always --rm -it -v "$$PWD:/src" -w /src git.frostfs.info/truecloudlab/env:dotnet-8.0
|
||||||
|
|
||||||
|
|
||||||
|
include release/codesign.mk
|
BIN
keyfile.snk
Normal file
BIN
keyfile.snk
Normal file
Binary file not shown.
82
release/README.md
Normal file
82
release/README.md
Normal file
|
@ -0,0 +1,82 @@
|
||||||
|
# Release process
|
||||||
|
|
||||||
|
## Preparing release
|
||||||
|
|
||||||
|
_TBD_
|
||||||
|
|
||||||
|
## Trusting TrueCloudLab code signing CA certificate
|
||||||
|
|
||||||
|
Verifying signatures (and signing) TrueCloudLab packages requires adding
|
||||||
|
[TrueCloudLab Code Signing CA](ca.cert) to the list of trusted roots.
|
||||||
|
|
||||||
|
On Linux this can be done by appending [release/ca.cert](ca.cert) to one of:
|
||||||
|
|
||||||
|
- `/etc/pki/ca-trust/extracted/pem/objsign-ca-bundle.pem`: compatible with
|
||||||
|
[update-ca-trust] and originally proposed in [.NET design docs]
|
||||||
|
- `…/dotnet/sdk/X.Y.ZZZ/trustedroots/codesignctl.pem`: [fallback] codesigning certificate trust list for .NET
|
||||||
|
|
||||||
|
[update-ca-trust]: https://www.linux.org/docs/man8/update-ca-trust.html
|
||||||
|
[.NET design docs]: https://github.com/dotnet/designs/blob/main/accepted/2021/signed-package-verification/re-enable-signed-package-verification-technical.md#linux
|
||||||
|
[fallback]: https://github.com/dotnet/sdk/blob/11150c0ec9020625308edeec555a8b78dbfb2aa5/src/Layout/redist/trustedroots/README.md
|
||||||
|
|
||||||
|
## Signing Nuget packages
|
||||||
|
|
||||||
|
Repository maintainer places `maintainer.cert` and `maintainer.key` (see below
|
||||||
|
regarding obtaining these files) into `release/` directory and then
|
||||||
|
executes:
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ make build sign
|
||||||
|
```
|
||||||
|
|
||||||
|
## Uploading packages to Nuget registry
|
||||||
|
|
||||||
|
**IMPORTANT: the following steps upload all `*.nupkg` files located under
|
||||||
|
`src/`. Maintainer MUST make sure that no unnecessary package versions will be
|
||||||
|
uploaded to the registry.**
|
||||||
|
|
||||||
|
Configure registry credentials (once per machine):
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ make nuget-registry NUGET_REGISTRY_USER=username NUGET_REGISTRY_PASSWORD=token
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish all locally built packages (implicitly clear existing `*.nupkg` and
|
||||||
|
rebuild current version only):
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ make clean build sign publish
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## Obtaining release signing certificate
|
||||||
|
|
||||||
|
Repository maintainer owns and keeps safe the release signing key
|
||||||
|
(`maintainer.key`). Private key should never leave maintainer's machine and
|
||||||
|
should be considered a highly sensitive secret.
|
||||||
|
|
||||||
|
- Generating new maintainer key and the corresponding CSR:
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ make maintainer.csr
|
||||||
|
...lines skipped...
|
||||||
|
Enter PEM pass phrase:
|
||||||
|
Verifying - Enter PEM pass phrase:
|
||||||
|
-----
|
||||||
|
IMPORTANT: Keep maintainer.key private!
|
||||||
|
|
||||||
|
Certificate signing request is ready.
|
||||||
|
Send maintainer.csr to CA administrator to obtain the certificate.
|
||||||
|
```
|
||||||
|
|
||||||
|
Resulting CSR (`maintainer.csr`) does not contain any sensitive
|
||||||
|
cryptographic material and may be passed to CA administrator through regular
|
||||||
|
communication channels.
|
||||||
|
|
||||||
|
- CA administrator then issues the certificate (`make maintainer.cert`) and
|
||||||
|
sends it back to the maintainer to be used in combination with
|
||||||
|
`maintainer.key`
|
||||||
|
|
||||||
|
This procedure should be repeated once per machine per `maintainer.cert`
|
||||||
|
lifetime (1 year) - typically just once per year since we expect the
|
||||||
|
maintainer to use only a single computer to sign releases.
|
34
release/ca.cert
Normal file
34
release/ca.cert
Normal file
|
@ -0,0 +1,34 @@
|
||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIF2zCCA8OgAwIBAgIUa9xC/RgvFtUG/xeR016nn0B4K0YwDQYJKoZIhvcNAQEL
|
||||||
|
BQAwdTELMAkGA1UEBhMCUlUxFTATBgNVBAoMDFRydWVDbG91ZExhYjEVMBMGA1UE
|
||||||
|
CwwMVHJ1ZUNsb3VkTGFiMTgwNgYDVQQDDC9UcnVlQ2xvdWRMYWIgQ29kZSBTaWdu
|
||||||
|
aW5nIENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yNTA0MTAxNTI2MTFaFw0zNTA0
|
||||||
|
MDgxNTI2MTFaMHUxCzAJBgNVBAYTAlJVMRUwEwYDVQQKDAxUcnVlQ2xvdWRMYWIx
|
||||||
|
FTATBgNVBAsMDFRydWVDbG91ZExhYjE4MDYGA1UEAwwvVHJ1ZUNsb3VkTGFiIENv
|
||||||
|
ZGUgU2lnbmluZyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEB
|
||||||
|
AQUAA4ICDwAwggIKAoICAQCyANB4cjf+ZEAFx9RiUYXCAOPMV+jyqgcVbhzh2YKc
|
||||||
|
9SlvGKRlc00Ar1RlFcrycdkIrTKmhhobiWsFp7UgphwLqRTCb5NB6qfUoWhnfiD9
|
||||||
|
m0OBgeVX5wivVaibRI9PSTbFDcIhYUiNvwFJ6GduH/9zOxf1BvuL7LMaoyhIDcg/
|
||||||
|
XVLuekE2lnX83zsedv0v/2jyyMY9Ct6N2BXzyHSAzSdYYg0F9Qu9fIMAPjoKhWPc
|
||||||
|
PnotqaACjb1DScLUr3E/o2W1FfprTT2Pip/0AXxO4wixl4QWh9HeOKV22KRcCHo6
|
||||||
|
3wNdg5q1ZVGTNBW0+yoB4jsSG8/JM+2Ujhc1ZnYH10armvGq/0Oc2YQE00960Wy8
|
||||||
|
t0drCFWJUO1XHNeBxkkupmj7N1TAPbixtfiGZJhECOWOJwyMpcKixlt5P0cNH4N/
|
||||||
|
p3vjyrGQxGLBIkgV/QgjfGkpTHKT1/H40YK6DliWJc01KfNTqn0K+/TIyF0n26kD
|
||||||
|
BWYVlvDh5P1+V9DGuD2zeXB3PstoifD/Pd7D8wuqpm17noFE19MLp94xv03q9nEa
|
||||||
|
jRMEd2J2buTLvMh5BBVH0Sm38QAHpSIZ9O3dSLvvjlALbVtwmcsNE9fgxiue3vTB
|
||||||
|
iXNW8wWs+/DMYwbWyBoCwORxVOdOyc1JLn7qAAEUBweilPVXpWuzMLdUsifPiqrV
|
||||||
|
dQIDAQABo2MwYTAdBgNVHQ4EFgQUEz4y/RvQMmbUFvf5JbGe/0PZR90wHwYDVR0j
|
||||||
|
BBgwFoAUEz4y/RvQMmbUFvf5JbGe/0PZR90wDwYDVR0TAQH/BAUwAwEB/zAOBgNV
|
||||||
|
HQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAF79W9hMGnrKUWLDOcoeXDEn
|
||||||
|
+gZxd5rjeF0tNEQGbWKGTJAGQdprOkzWQ47PewpFeS+ePpjOglBAt7cV2ZnOugAT
|
||||||
|
Brx31vYpNAvYnUCYn+/IUqD8S/U4uErVS9zG9QjirZWo56eJP62vnScKuApCQCbA
|
||||||
|
pp0zrIyJ+2lQKzlMOENRqzYYA/UTOqYTtnW6x2D8goVqCmDXpgpxEp5XliFjJSr6
|
||||||
|
dOjiopNWMvaV3R/Bnd4i41taM7M6HpIV+gbXmEKEFS0ejVfzT8z1pTigN7GBqbxf
|
||||||
|
nXD03eLUIsbMDv4ZQPrheN7nKnjRUn8kxz0SSK1m2YDrXW51m8fOs6aTvwC/cNe+
|
||||||
|
FJMqQMF32i4IXVfUbyUJi+JMvawqm2wEY46vrh7siprY6rXsAzCKJo07i6jvUwnT
|
||||||
|
TXMldSCPgTEqzT2JBzzr0tRfuPKsv0/NqflHvwfuMRCpcZ7jJZ700iN92xXkiQHP
|
||||||
|
DmCZOILXcNclAth3nAnyY4XE5a8myv8bwYaPdJdIFlV+BoU/8mClDeA8ck4rDy12
|
||||||
|
T5YChKew2oiL4j4B6v9/yrDjD1IT0gv4BWyPhb/n390BCEXt8g9auNcT0s6O8kEc
|
||||||
|
VUDVc1519ocMCuWVuqUK9o2w0zu50/pBn4hVLfT3QyW8sqtlRKghOWtqZzigvCWF
|
||||||
|
VjATeO5F/Z7OSDebHUGv
|
||||||
|
-----END CERTIFICATE-----
|
74
release/codesign.mk
Normal file
74
release/codesign.mk
Normal file
|
@ -0,0 +1,74 @@
|
||||||
|
PKI_ROLE?=maintainer
|
||||||
|
PKI_DIR?=release
|
||||||
|
|
||||||
|
# Note: Only RSA signatures are supported (NU3013)
|
||||||
|
# https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3013)
|
||||||
|
|
||||||
|
|
||||||
|
ifeq ($(PKI_ROLE),maintainer)
|
||||||
|
.PHONY: maintainer.csr
|
||||||
|
maintainer.csr: $(PKI_DIR)/maintainer.csr
|
||||||
|
$(PKI_DIR)/maintainer.csr: KEY=$(patsubst %.csr,%.key,$@)
|
||||||
|
$(PKI_DIR)/maintainer.csr:
|
||||||
|
openssl req \
|
||||||
|
-new \
|
||||||
|
-newkey rsa:4096 \
|
||||||
|
-keyout $(KEY) \
|
||||||
|
-out $@ \
|
||||||
|
-sha256 \
|
||||||
|
-addext keyUsage=critical,digitalSignature \
|
||||||
|
-addext extendedKeyUsage=critical,codeSigning,msCodeCom \
|
||||||
|
-subj "/C=RU/O=TrueCloudLab/OU=TrueCloudLab/CN=frostfs-sdk-csharp Release Team"
|
||||||
|
@echo "IMPORTANT: Keep $(KEY) private!\n"
|
||||||
|
@echo "Certificate signing request is ready.\nSend $@ to CA administrator to obtain the certificate."
|
||||||
|
|
||||||
|
$(PKI_DIR)/maintainer.pfx: $(PKI_DIR)/maintainer.cert $(PKI_DIR)/maintainer.key $(PKI_DIR)/ca.cert
|
||||||
|
openssl verify \
|
||||||
|
-CAfile $(PKI_DIR)/ca.cert \
|
||||||
|
$(PKI_DIR)/maintainer.cert
|
||||||
|
openssl pkcs12 \
|
||||||
|
-export \
|
||||||
|
-out $@ \
|
||||||
|
-inkey $(PKI_DIR)/maintainer.key \
|
||||||
|
-in $(PKI_DIR)/maintainer.cert \
|
||||||
|
-CAfile $(PKI_DIR)/ca.cert \
|
||||||
|
-chain \
|
||||||
|
-passout pass:
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
ifeq ($(PKI_ROLE),ca)
|
||||||
|
.PHONY: maintainer.cert
|
||||||
|
maintainer.cert: $(PKI_DIR)/maintainer.cert
|
||||||
|
$(PKI_DIR)/maintainer.cert: CSR=$(patsubst %.cert,%.csr,$@)
|
||||||
|
$(PKI_DIR)/maintainer.cert: $(PKI_DIR)/ca.key $(PKI_DIR)/ca.cert
|
||||||
|
openssl req -noout -text -in $(CSR)
|
||||||
|
@read -p "Review the CSR above. Press Enter to continue, Ctrl+C to cancel
" -r null
|
||||||
|
openssl x509 \
|
||||||
|
-req \
|
||||||
|
-days 365 \
|
||||||
|
-in $(CSR) \
|
||||||
|
-copy_extensions copy \
|
||||||
|
-ext keyUsage,extendedKeyUsage \
|
||||||
|
-CA $(PKI_DIR)/ca.cert \
|
||||||
|
-CAkey $(PKI_DIR)/ca.key \
|
||||||
|
-CAcreateserial \
|
||||||
|
-out $@
|
||||||
|
echo >> $@
|
||||||
|
cat $(PKI_DIR)/ca.cert >> $@
|
||||||
|
openssl x509 -noout -text -in $@ -fingerprint -sha256
|
||||||
|
@echo "Certificate is ready.\nSend $@ back to maintainer."
|
||||||
|
|
||||||
|
$(PKI_DIR)/ca.key: CERT=$(patsubst %.key,%.cert,$@)
|
||||||
|
$(PKI_DIR)/ca.key:
|
||||||
|
openssl req \
|
||||||
|
-x509 \
|
||||||
|
-newkey rsa:4096 \
|
||||||
|
-keyout $@ \
|
||||||
|
-out $(CERT) \
|
||||||
|
-sha256 \
|
||||||
|
-days 3650 \
|
||||||
|
-addext keyUsage=critical,keyCertSign \
|
||||||
|
-subj "/C=RU/O=TrueCloudLab/OU=TrueCloudLab/CN=TrueCloudLab Code Signing Certificate Authority"
|
||||||
|
@echo "IMPORTANT: Keep $@ private!\n"
|
||||||
|
endif
|
|
@ -8,7 +8,7 @@ public struct Actions(bool inverted, string[] names) : System.IEquatable<Actions
|
||||||
|
|
||||||
public override readonly bool Equals(object obj)
|
public override readonly bool Equals(object obj)
|
||||||
{
|
{
|
||||||
if (obj == null || obj is not Actions)
|
if (obj == null || obj is not Actions)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
return Equals((Actions)obj);
|
return Equals((Actions)obj);
|
||||||
|
|
|
@ -12,7 +12,7 @@ public struct Condition : System.IEquatable<Condition>
|
||||||
|
|
||||||
public override bool Equals(object obj)
|
public override bool Equals(object obj)
|
||||||
{
|
{
|
||||||
if (obj == null || obj is not Condition)
|
if (obj == null || obj is not Condition)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
return Equals((Condition)obj);
|
return Equals((Condition)obj);
|
||||||
|
|
|
@ -8,7 +8,7 @@ public struct Resources(bool inverted, string[] names) : System.IEquatable<Resou
|
||||||
|
|
||||||
public override readonly bool Equals(object obj)
|
public override readonly bool Equals(object obj)
|
||||||
{
|
{
|
||||||
if (obj == null || obj is not Resources)
|
if (obj == null || obj is not Resources)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
return Equals((Resources)obj);
|
return Equals((Resources)obj);
|
||||||
|
|
|
@ -1,7 +1,4 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Runtime.CompilerServices;
|
|
||||||
|
|
||||||
[assembly: InternalsVisibleTo("FrostFS.SDK.Tests")]
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
|
|
|
@ -1,8 +1,14 @@
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
[assembly: AssemblyCompany("FrostFS.SDK.Client")]
|
[assembly: AssemblyCompany("TrueCloudLab")]
|
||||||
[assembly: AssemblyFileVersion("1.0.2.0")]
|
[assembly: InternalsVisibleTo("FrostFS.SDK.Tests, PublicKey=" +
|
||||||
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
|
"002400000480000094000000060200000024000052534131000400000100010089b992fb14ebcf"+
|
||||||
|
"4b85b4b1a3af1897290c52ff85a106035c47dc5604cbaa58ae3180f5c9b8523fee5dd1bb9ea9cf"+
|
||||||
|
"e15ab287e6239c98d5dfa91615bd77485d523a3a3f65a4e5028454cedd5ac4d9eca6da18b81985"+
|
||||||
|
"ac6905d33cc64b5a2587050c16f67b71ef8889dbd3c90ef7cc0b06bbbe09886601d195f5db179a"+
|
||||||
|
"3c2a25b1")]
|
||||||
|
[assembly: AssemblyFileVersion("1.0.7.0")]
|
||||||
[assembly: AssemblyProduct("FrostFS.SDK.Client")]
|
[assembly: AssemblyProduct("FrostFS.SDK.Client")]
|
||||||
[assembly: AssemblyTitle("FrostFS.SDK.Client")]
|
[assembly: AssemblyTitle("FrostFS.SDK.Client")]
|
||||||
[assembly: AssemblyVersion("1.0.3")]
|
[assembly: AssemblyVersion("1.0.7")]
|
||||||
|
|
|
@ -6,17 +6,8 @@ internal static class Caches
|
||||||
{
|
{
|
||||||
private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions
|
private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions
|
||||||
{
|
{
|
||||||
// TODO: get from options?
|
|
||||||
SizeLimit = 256
|
SizeLimit = 256
|
||||||
});
|
});
|
||||||
|
|
||||||
private static readonly IMemoryCache _containersCache = new MemoryCache(new MemoryCacheOptions
|
|
||||||
{
|
|
||||||
// TODO: get from options?
|
|
||||||
SizeLimit = 1024
|
|
||||||
});
|
|
||||||
|
|
||||||
internal static IMemoryCache Owners => _ownersCache;
|
internal static IMemoryCache Owners => _ownersCache;
|
||||||
|
|
||||||
internal static IMemoryCache Containers => _containersCache;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,7 +11,7 @@ public class FrostFsResponseException : FrostFsException
|
||||||
}
|
}
|
||||||
|
|
||||||
public FrostFsResponseException(FrostFsResponseStatus status)
|
public FrostFsResponseException(FrostFsResponseStatus status)
|
||||||
: base(status != null ? status.Message != null ? "" : "" : "")
|
: base(status != null ? status.Message != null ? "" : "" : "")
|
||||||
{
|
{
|
||||||
Status = status;
|
Status = status;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,14 +1,17 @@
|
||||||
using System;
|
using System;
|
||||||
|
using System.Security.Cryptography;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
|
||||||
namespace FrostFS.SDK.Cryptography;
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
public static class FrostFsExtensions
|
public static class FrostFsExtensions
|
||||||
{
|
{
|
||||||
public static ByteString Sha256(this IMessage data)
|
public static byte[] Sha256(this IMessage data)
|
||||||
{
|
{
|
||||||
return ByteString.CopyFrom(data.ToByteArray().Sha256());
|
using var sha256 = SHA256.Create();
|
||||||
|
using HashStream stream = new(sha256);
|
||||||
|
data.WriteTo(stream);
|
||||||
|
return stream.Hash();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Guid ToUuid(this ByteString id)
|
public static Guid ToUuid(this ByteString id)
|
||||||
|
@ -16,9 +19,18 @@ public static class FrostFsExtensions
|
||||||
if (id == null)
|
if (id == null)
|
||||||
throw new ArgumentNullException(nameof(id));
|
throw new ArgumentNullException(nameof(id));
|
||||||
|
|
||||||
var orderedBytes = GetGuidBytesDirectOrder(id.Span);
|
return new Guid(
|
||||||
|
(id[0] << 24) + (id[1] << 16) + (id[2] << 8) + id[3],
|
||||||
return new Guid(orderedBytes);
|
(short)((id[4] << 8) + id[5]),
|
||||||
|
(short)((id[6] << 8) + id[7]),
|
||||||
|
id[8],
|
||||||
|
id[9],
|
||||||
|
id[10],
|
||||||
|
id[11],
|
||||||
|
id[12],
|
||||||
|
id[13],
|
||||||
|
id[14],
|
||||||
|
id[15]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -26,37 +38,25 @@ public static class FrostFsExtensions
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="id"></param>
|
/// <param name="id"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static byte[] ToBytes(this Guid id)
|
public unsafe static void ToBytes(this Guid id, Span<byte> span)
|
||||||
{
|
{
|
||||||
var bytes = id.ToByteArray();
|
var pGuid = (byte*)&id;
|
||||||
|
|
||||||
var orderedBytes = GetGuidBytesDirectOrder(bytes);
|
span[0] = pGuid[3];
|
||||||
|
span[1] = pGuid[2];
|
||||||
return orderedBytes;
|
span[2] = pGuid[1];
|
||||||
}
|
span[3] = pGuid[0];
|
||||||
|
span[4] = pGuid[5];
|
||||||
private static byte[] GetGuidBytesDirectOrder(ReadOnlySpan<byte> source)
|
span[5] = pGuid[4];
|
||||||
{
|
span[6] = pGuid[7];
|
||||||
if (source.Length != 16)
|
span[7] = pGuid[6];
|
||||||
throw new ArgumentException("Wrong uuid binary format");
|
span[8] = pGuid[8];
|
||||||
|
span[9] = pGuid[9];
|
||||||
return [
|
span[10] = pGuid[10];
|
||||||
source[3],
|
span[11] = pGuid[11];
|
||||||
source[2],
|
span[12] = pGuid[12];
|
||||||
source[1],
|
span[13] = pGuid[13];
|
||||||
source[0],
|
span[14] = pGuid[14];
|
||||||
source[5],
|
span[15] = pGuid[15];
|
||||||
source[4],
|
|
||||||
source[7],
|
|
||||||
source[6],
|
|
||||||
source[8],
|
|
||||||
source[9],
|
|
||||||
source[10],
|
|
||||||
source[11],
|
|
||||||
source[12],
|
|
||||||
source[13],
|
|
||||||
source[14],
|
|
||||||
source[15]
|
|
||||||
];
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,7 @@
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
|
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
|
||||||
<PackageId>FrostFS.SDK.Client</PackageId>
|
<PackageId>FrostFS.SDK.Client</PackageId>
|
||||||
<Version>1.0.3</Version>
|
<Version>1.0.7</Version>
|
||||||
<Description>
|
<Description>
|
||||||
C# SDK for FrostFS gRPC native protocol
|
C# SDK for FrostFS gRPC native protocol
|
||||||
</Description>
|
</Description>
|
||||||
|
@ -31,10 +31,12 @@
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
||||||
|
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
|
||||||
|
<SignAssembly>True</SignAssembly>
|
||||||
|
<AssemblyOriginatorKeyFile>.\\..\\..\\keyfile.snk</AssemblyOriginatorKeyFile>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
|
|
||||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
|
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
|
||||||
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
|
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
|
||||||
<PrivateAssets>all</PrivateAssets>
|
<PrivateAssets>all</PrivateAssets>
|
||||||
|
@ -45,6 +47,7 @@
|
||||||
<PackageReference Include="Microsoft.Extensions.Options" Version="7.0.1" />
|
<PackageReference Include="Microsoft.Extensions.Options" Version="7.0.1" />
|
||||||
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="7.0.0" />
|
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="7.0.0" />
|
||||||
<PackageReference Include="System.Runtime.Caching" Version="7.0.0" />
|
<PackageReference Include="System.Runtime.Caching" Version="7.0.0" />
|
||||||
|
<PackageReference Include="System.Text.Json" Version="7.0.1" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
@ -162,24 +162,6 @@ public class FrostFSClient : IFrostFSClient
|
||||||
Callback = settings.Value.Callback,
|
Callback = settings.Value.Callback,
|
||||||
Interceptors = settings.Value.Interceptors
|
Interceptors = settings.Value.Interceptors
|
||||||
};
|
};
|
||||||
|
|
||||||
// TODO: define timeout logic
|
|
||||||
// CheckFrostFsVersionSupport(new Context { Timeout = TimeSpan.FromSeconds(20) });
|
|
||||||
}
|
|
||||||
|
|
||||||
internal FrostFSClient(WrapperPrm prm, SessionCache cache)
|
|
||||||
{
|
|
||||||
ClientCtx = new ClientContext(
|
|
||||||
client: this,
|
|
||||||
key: new ClientKey(prm.Key),
|
|
||||||
owner: FrostFsOwner.FromKey(prm.Key!),
|
|
||||||
channel: prm.GrpcChannelFactory(prm.Address),
|
|
||||||
version: new FrostFsVersion(2, 13))
|
|
||||||
{
|
|
||||||
SessionCache = cache,
|
|
||||||
Interceptors = prm.Interceptors,
|
|
||||||
Callback = prm.Callback
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#region ApeManagerImplementation
|
#region ApeManagerImplementation
|
||||||
|
@ -272,7 +254,8 @@ public class FrostFSClient : IFrostFSClient
|
||||||
|
|
||||||
public Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
|
public Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
|
||||||
{
|
{
|
||||||
return GetObjectService().PutClientCutObjectAsync(args, ctx);
|
return GetObjectService().PutClientCutSingleObjectAsync(args, ctx);
|
||||||
|
// return GetObjectService().PutClientCutObjectAsync(args, ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<FrostFsObjectId> PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx)
|
public Task<FrostFsObjectId> PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx)
|
||||||
|
@ -448,17 +431,4 @@ public class FrostFSClient : IFrostFSClient
|
||||||
|
|
||||||
return ObjectServiceProvider;
|
return ObjectServiceProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<string?> Dial(CallContext ctx)
|
|
||||||
{
|
|
||||||
var service = GetAccouningService();
|
|
||||||
_ = await service.GetBallance(ctx).ConfigureAwait(false);
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool RestartIfUnhealthy(CallContext ctx)
|
|
||||||
{
|
|
||||||
throw new NotImplementedException();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,6 +64,4 @@ public interface IFrostFSClient
|
||||||
#region Account
|
#region Account
|
||||||
Task<Accounting.Decimal> GetBalanceAsync(CallContext ctx);
|
Task<Accounting.Decimal> GetBalanceAsync(CallContext ctx);
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
public Task<string?> Dial(CallContext ctx);
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,16 +5,10 @@ using FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
|
||||||
using Microsoft.Extensions.Caching.Memory;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client.Mappers.GRPC;
|
namespace FrostFS.SDK.Client.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ContainerIdMapper
|
public static class ContainerIdMapper
|
||||||
{
|
{
|
||||||
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
|
|
||||||
.SetSlidingExpiration(TimeSpan.FromHours(1))
|
|
||||||
.SetSize(1);
|
|
||||||
|
|
||||||
public static ContainerID ToMessage(this FrostFsContainerId model)
|
public static ContainerID ToMessage(this FrostFsContainerId model)
|
||||||
{
|
{
|
||||||
if (model is null)
|
if (model is null)
|
||||||
|
@ -24,15 +18,11 @@ public static class ContainerIdMapper
|
||||||
|
|
||||||
var containerId = model.GetValue() ?? throw new ArgumentNullException(nameof(model));
|
var containerId = model.GetValue() ?? throw new ArgumentNullException(nameof(model));
|
||||||
|
|
||||||
if (!Caches.Containers.TryGetValue(containerId, out ContainerID? message))
|
var message = new ContainerID
|
||||||
{
|
{
|
||||||
message = new ContainerID
|
Value = UnsafeByteOperations.UnsafeWrap(Base58.Decode(containerId))
|
||||||
{
|
};
|
||||||
Value = ByteString.CopyFrom(Base58.Decode(containerId))
|
|
||||||
};
|
|
||||||
|
|
||||||
Caches.Containers.Set(containerId, message, _oneHourExpiration);
|
|
||||||
}
|
|
||||||
return message!;
|
return message!;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,8 +16,8 @@ public static class MetaHeaderMapper
|
||||||
return new RequestMetaHeader
|
return new RequestMetaHeader
|
||||||
{
|
{
|
||||||
Version = metaHeader.Version.ToMessage(),
|
Version = metaHeader.Version.ToMessage(),
|
||||||
Epoch = (uint)metaHeader.Epoch,
|
Epoch = metaHeader.Epoch,
|
||||||
Ttl = (uint)metaHeader.Ttl
|
Ttl = metaHeader.Ttl
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -11,8 +11,10 @@ public static class PolicyMapper
|
||||||
{
|
{
|
||||||
return new Replica
|
return new Replica
|
||||||
{
|
{
|
||||||
Count = (uint)replica.Count,
|
Count = replica.Count,
|
||||||
Selector = replica.Selector
|
Selector = replica.Selector,
|
||||||
|
EcDataCount = replica.EcDataCount,
|
||||||
|
EcParityCount = replica.EcParityCount
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -23,7 +25,11 @@ public static class PolicyMapper
|
||||||
throw new ArgumentNullException(nameof(replica));
|
throw new ArgumentNullException(nameof(replica));
|
||||||
}
|
}
|
||||||
|
|
||||||
return new FrostFsReplica((int)replica.Count, replica.Selector);
|
return new FrostFsReplica(replica.Count, replica.Selector)
|
||||||
|
{
|
||||||
|
EcDataCount = replica.EcDataCount,
|
||||||
|
EcParityCount = replica.EcParityCount
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Selector ToMessage(this FrostFsSelector selector)
|
public static Selector ToMessage(this FrostFsSelector selector)
|
||||||
|
|
|
@ -24,25 +24,14 @@ public static class ObjectHeaderMapper
|
||||||
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
|
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
|
||||||
};
|
};
|
||||||
|
|
||||||
FrostFsSplit? split = null;
|
FrostFsSplit? split = header!.Split != null
|
||||||
|
? header.Split.ToModel()
|
||||||
if (header.Split != null)
|
: null;
|
||||||
{
|
|
||||||
var children = header.Split.Children.Count != 0 ? new ReadOnlyCollection<FrostFsObjectId>(
|
|
||||||
header.Split.Children.Select(x => x.ToModel()).ToList()) : null;
|
|
||||||
|
|
||||||
split = new FrostFsSplit(new SplitId(header.Split.SplitId.ToUuid()),
|
|
||||||
header.Split.Previous?.ToModel(),
|
|
||||||
header.Split.Parent?.ToModel(),
|
|
||||||
header.Split.ParentHeader?.ToModel(),
|
|
||||||
null,
|
|
||||||
children);
|
|
||||||
}
|
|
||||||
|
|
||||||
var model = new FrostFsObjectHeader(
|
var model = new FrostFsObjectHeader(
|
||||||
new FrostFsContainerId(Base58.Encode(header.ContainerId.Value.Span)),
|
new FrostFsContainerId(Base58.Encode(header.ContainerId.Value.Span)),
|
||||||
objTypeName,
|
objTypeName,
|
||||||
header.Attributes.Select(attribute => attribute.ToModel()).ToArray(),
|
[.. header.Attributes.Select(attribute => attribute.ToModel())],
|
||||||
split,
|
split,
|
||||||
header.OwnerId.ToModel(),
|
header.OwnerId.ToModel(),
|
||||||
header.Version.ToModel())
|
header.Version.ToModel())
|
||||||
|
@ -52,4 +41,23 @@ public static class ObjectHeaderMapper
|
||||||
|
|
||||||
return model;
|
return model;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static FrostFsSplit ToModel(this Header.Types.Split split)
|
||||||
|
{
|
||||||
|
if (split is null)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(split));
|
||||||
|
}
|
||||||
|
|
||||||
|
var children = split!.Children.Count != 0
|
||||||
|
? new ReadOnlyCollection<FrostFsObjectId>([.. split.Children.Select(x => x.ToModel())])
|
||||||
|
: null;
|
||||||
|
|
||||||
|
return new FrostFsSplit(new SplitId(split.SplitId.ToUuid()),
|
||||||
|
split.Previous?.ToModel(),
|
||||||
|
split.Parent?.ToModel(),
|
||||||
|
split.ParentHeader?.ToModel(),
|
||||||
|
null,
|
||||||
|
children);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,7 @@ public static class ObjectIdMapper
|
||||||
|
|
||||||
return new ObjectID
|
return new ObjectID
|
||||||
{
|
{
|
||||||
Value = ByteString.CopyFrom(objectId.ToHash())
|
Value = UnsafeByteOperations.UnsafeWrap(objectId.ToHash())
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -26,7 +26,7 @@ public static class OwnerIdMapper
|
||||||
{
|
{
|
||||||
message = new OwnerID
|
message = new OwnerID
|
||||||
{
|
{
|
||||||
Value = ByteString.CopyFrom(model.ToHash())
|
Value = UnsafeByteOperations.UnsafeWrap(model.ToHash())
|
||||||
};
|
};
|
||||||
|
|
||||||
Caches.Owners.Set(model, message, _oneHourExpiration);
|
Caches.Owners.Set(model, message, _oneHourExpiration);
|
||||||
|
|
|
@ -1,28 +0,0 @@
|
||||||
using System;
|
|
||||||
|
|
||||||
using Google.Protobuf;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public static class SessionMapper
|
|
||||||
{
|
|
||||||
public static byte[] Serialize(this Session.SessionToken token)
|
|
||||||
{
|
|
||||||
if (token is null)
|
|
||||||
{
|
|
||||||
throw new ArgumentNullException(nameof(token));
|
|
||||||
}
|
|
||||||
|
|
||||||
byte[] bytes = new byte[token.CalculateSize()];
|
|
||||||
using CodedOutputStream stream = new(bytes);
|
|
||||||
token.WriteTo(stream);
|
|
||||||
|
|
||||||
return bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Session.SessionToken Deserialize(this Session.SessionToken token, byte[] bytes)
|
|
||||||
{
|
|
||||||
token.MergeFrom(bytes);
|
|
||||||
return token;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -23,9 +23,9 @@ public static class SignatureMapper
|
||||||
|
|
||||||
return new Refs.Signature
|
return new Refs.Signature
|
||||||
{
|
{
|
||||||
Key = ByteString.CopyFrom(signature.Key),
|
Key = UnsafeByteOperations.UnsafeWrap(signature.Key),
|
||||||
Scheme = scheme,
|
Scheme = scheme,
|
||||||
Sign = ByteString.CopyFrom(signature.Sign)
|
Sign = UnsafeByteOperations.UnsafeWrap(signature.Sign)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,7 @@ public static class VersionMapper
|
||||||
throw new System.ArgumentNullException(nameof(model));
|
throw new System.ArgumentNullException(nameof(model));
|
||||||
}
|
}
|
||||||
|
|
||||||
var key = model.Major << 16 + model.Minor;
|
var key = (int)model.Major << 16 + (int)model.Minor;
|
||||||
|
|
||||||
if (!_cacheMessages.ContainsKey(key))
|
if (!_cacheMessages.ContainsKey(key))
|
||||||
{
|
{
|
||||||
|
@ -28,8 +28,8 @@ public static class VersionMapper
|
||||||
_spinlock.Enter(ref lockTaken);
|
_spinlock.Enter(ref lockTaken);
|
||||||
var message = new Version
|
var message = new Version
|
||||||
{
|
{
|
||||||
Major = (uint)model.Major,
|
Major = model.Major,
|
||||||
Minor = (uint)model.Minor
|
Minor = model.Minor
|
||||||
};
|
};
|
||||||
|
|
||||||
_cacheMessages.Add(key, message);
|
_cacheMessages.Add(key, message);
|
||||||
|
@ -64,7 +64,7 @@ public static class VersionMapper
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_spinlock.Enter(ref lockTaken);
|
_spinlock.Enter(ref lockTaken);
|
||||||
var model = new FrostFsVersion((int)message.Major, (int)message.Minor);
|
var model = new FrostFsVersion(message.Major, message.Minor);
|
||||||
|
|
||||||
_cacheModels.Add(key, model);
|
_cacheModels.Add(key, model);
|
||||||
return model;
|
return model;
|
||||||
|
|
|
@ -91,10 +91,13 @@ public class FrostFsContainerInfo
|
||||||
throw new ArgumentNullException("PlacementPolicy is null");
|
throw new ArgumentNullException("PlacementPolicy is null");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Span<byte> nonce = stackalloc byte[16];
|
||||||
|
Nonce.ToBytes(nonce);
|
||||||
|
|
||||||
this.container = new Container.Container()
|
this.container = new Container.Container()
|
||||||
{
|
{
|
||||||
PlacementPolicy = PlacementPolicy.Value.GetPolicy(),
|
PlacementPolicy = PlacementPolicy.Value.GetPolicy(),
|
||||||
Nonce = ByteString.CopyFrom(Nonce.ToBytes()),
|
Nonce = ByteString.CopyFrom(nonce),
|
||||||
OwnerId = Owner?.OwnerID,
|
OwnerId = Owner?.OwnerID,
|
||||||
Version = Version?.VersionID
|
Version = Version?.VersionID
|
||||||
};
|
};
|
||||||
|
|
|
@ -11,7 +11,7 @@ public class CheckSum
|
||||||
|
|
||||||
public static CheckSum CreateCheckSum(byte[] content)
|
public static CheckSum CreateCheckSum(byte[] content)
|
||||||
{
|
{
|
||||||
return new CheckSum { hash = content.Sha256() };
|
return new CheckSum { hash = DataHasher.Sha256(content.AsSpan()) };
|
||||||
}
|
}
|
||||||
|
|
||||||
public override string ToString()
|
public override string ToString()
|
||||||
|
|
|
@ -4,12 +4,12 @@ namespace FrostFS.SDK;
|
||||||
|
|
||||||
public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
||||||
{
|
{
|
||||||
public int Count { get; set; }
|
public uint Count { get; set; }
|
||||||
public string Selector { get; set; }
|
public string Selector { get; set; }
|
||||||
public uint EcDataCount { get; set; }
|
public uint EcDataCount { get; set; }
|
||||||
public uint EcParityCount { get; set; }
|
public uint EcParityCount { get; set; }
|
||||||
|
|
||||||
public FrostFsReplica(int count, string? selector = null)
|
public FrostFsReplica(uint count, string? selector = null)
|
||||||
{
|
{
|
||||||
selector ??= string.Empty;
|
selector ??= string.Empty;
|
||||||
|
|
||||||
|
@ -31,12 +31,12 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
||||||
|
|
||||||
public readonly uint CountNodes()
|
public readonly uint CountNodes()
|
||||||
{
|
{
|
||||||
return Count != 0 ? (uint)Count : EcDataCount + EcParityCount;
|
return Count != 0 ? Count : EcDataCount + EcParityCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
public override readonly int GetHashCode()
|
public override readonly int GetHashCode()
|
||||||
{
|
{
|
||||||
return (Count + Selector.GetHashCode()) ^ (int)EcDataCount ^ (int)EcParityCount;
|
return Count.GetHashCode() ^ Selector.GetHashCode() ^ (int)EcDataCount ^ (int)EcParityCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static bool operator ==(FrostFsReplica left, FrostFsReplica right)
|
public static bool operator ==(FrostFsReplica left, FrostFsReplica right)
|
||||||
|
|
|
@ -3,12 +3,12 @@ using FrostFS.SDK.Client.Mappers.GRPC;
|
||||||
|
|
||||||
namespace FrostFS.SDK;
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
public class FrostFsVersion(int major, int minor)
|
public class FrostFsVersion(uint major, uint minor)
|
||||||
{
|
{
|
||||||
private Version? version;
|
private Version? version;
|
||||||
|
|
||||||
public int Major { get; set; } = major;
|
public uint Major { get; set; } = major;
|
||||||
public int Minor { get; set; } = minor;
|
public uint Minor { get; set; } = minor;
|
||||||
|
|
||||||
internal Version VersionID
|
internal Version VersionID
|
||||||
{
|
{
|
||||||
|
|
|
@ -353,7 +353,7 @@ internal struct Context
|
||||||
|
|
||||||
var start = hasPrefix ? likeWildcard.Length : 0;
|
var start = hasPrefix ? likeWildcard.Length : 0;
|
||||||
var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length;
|
var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length;
|
||||||
var str = f.Value.Substring(start, end-start);
|
var str = f.Value.Substring(start, end - start);
|
||||||
|
|
||||||
if (hasPrefix && hasSuffix)
|
if (hasPrefix && hasSuffix)
|
||||||
return nodeInfo.Attributes[f.Key].Contains(str);
|
return nodeInfo.Attributes[f.Key].Contains(str);
|
||||||
|
|
|
@ -4,10 +4,6 @@ namespace FrostFS.SDK;
|
||||||
|
|
||||||
public class FrostFsObject
|
public class FrostFsObject
|
||||||
{
|
{
|
||||||
// private byte[]? _payloadBytes;
|
|
||||||
// private ReadOnlyMemory<byte> _payloadMemory;
|
|
||||||
// private bool _isInitPayloadMemory;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates new instance from <c>ObjectHeader</c>
|
/// Creates new instance from <c>ObjectHeader</c>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
@ -49,25 +45,6 @@ public class FrostFsObject
|
||||||
|
|
||||||
public ReadOnlyMemory<byte> SingleObjectPayload { get; set; }
|
public ReadOnlyMemory<byte> SingleObjectPayload { get; set; }
|
||||||
|
|
||||||
//public ReadOnlyMemory<byte> SingleObjectPayloadMemory
|
|
||||||
//{
|
|
||||||
// get
|
|
||||||
// {
|
|
||||||
// if (!_isInitPayloadMemory)
|
|
||||||
// {
|
|
||||||
// _payloadMemory = _payloadBytes.AsMemory();
|
|
||||||
// _isInitPayloadMemory = true;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// return _payloadMemory;
|
|
||||||
// }
|
|
||||||
// set
|
|
||||||
// {
|
|
||||||
// _payloadMemory = value;
|
|
||||||
// _isInitPayloadMemory = true;
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic
|
/// Provide SHA256 hash of the payload. If null, the hash is calculated by internal logic
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
|
@ -1,4 +1,7 @@
|
||||||
using System.Collections.ObjectModel;
|
using System.Collections.ObjectModel;
|
||||||
|
using System.Linq;
|
||||||
|
using FrostFS.Object;
|
||||||
|
using FrostFS.SDK.Client.Mappers.GRPC;
|
||||||
|
|
||||||
namespace FrostFS.SDK;
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
|
@ -9,6 +12,8 @@ public class FrostFsSplit(SplitId splitId,
|
||||||
FrostFsSignature? parentSignature = null,
|
FrostFsSignature? parentSignature = null,
|
||||||
ReadOnlyCollection<FrostFsObjectId>? children = null)
|
ReadOnlyCollection<FrostFsObjectId>? children = null)
|
||||||
{
|
{
|
||||||
|
private Header.Types.Split? _split;
|
||||||
|
|
||||||
public FrostFsSplit() : this(new SplitId())
|
public FrostFsSplit() : this(new SplitId())
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
@ -24,4 +29,25 @@ public class FrostFsSplit(SplitId splitId,
|
||||||
public FrostFsObjectHeader? ParentHeader { get; set; } = parentHeader;
|
public FrostFsObjectHeader? ParentHeader { get; set; } = parentHeader;
|
||||||
|
|
||||||
public ReadOnlyCollection<FrostFsObjectId>? Children { get; } = children;
|
public ReadOnlyCollection<FrostFsObjectId>? Children { get; } = children;
|
||||||
|
|
||||||
|
public Header.Types.Split GetSplit()
|
||||||
|
{
|
||||||
|
if (_split == null)
|
||||||
|
{
|
||||||
|
_split = new Header.Types.Split
|
||||||
|
{
|
||||||
|
SplitId = SplitId?.GetSplitId(),
|
||||||
|
Parent = Parent?.ToMessage(),
|
||||||
|
ParentHeader = ParentHeader?.GetHeader(),
|
||||||
|
ParentSignature = ParentSignature?.ToMessage()
|
||||||
|
};
|
||||||
|
|
||||||
|
if (Children != null)
|
||||||
|
{
|
||||||
|
_split.Children.AddRange(Children.Select(x => x.ToMessage()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return _split;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
|
public class PartUploadedEventArgs(ObjectPartInfo part) : EventArgs
|
||||||
|
{
|
||||||
|
public ObjectPartInfo Part { get; } = part;
|
||||||
|
}
|
|
@ -47,16 +47,11 @@ public class SplitId
|
||||||
return this.id.ToString();
|
return this.id.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public byte[]? ToBinary()
|
|
||||||
{
|
|
||||||
if (this.id == Guid.Empty)
|
|
||||||
return null;
|
|
||||||
|
|
||||||
return this.id.ToBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
public ByteString? GetSplitId()
|
public ByteString? GetSplitId()
|
||||||
{
|
{
|
||||||
return this.message ??= ByteString.CopyFrom(ToBinary());
|
Span<byte> span = stackalloc byte[16];
|
||||||
|
id.ToBytes(span);
|
||||||
|
|
||||||
|
return this.message ??= ByteString.CopyFrom(span);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
36
src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs
Normal file
36
src/FrostFS.SDK.Client/Models/Object/UploadInfo.cs
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
|
public readonly struct ObjectPartInfo(long offset, int length, FrostFsObjectId objectId) : System.IEquatable<ObjectPartInfo>
|
||||||
|
{
|
||||||
|
public long Offset { get; } = offset;
|
||||||
|
public int Length { get; } = length;
|
||||||
|
public FrostFsObjectId ObjectId { get; } = objectId;
|
||||||
|
|
||||||
|
public override bool Equals(object obj)
|
||||||
|
{
|
||||||
|
if (obj == null || obj is not ObjectPartInfo)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
return Equals((ObjectPartInfo)obj);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override int GetHashCode()
|
||||||
|
{
|
||||||
|
return ((int)(Offset >> 32)) ^ (int)Offset ^ Length ^ ObjectId.Value.GetHashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator ==(ObjectPartInfo left, ObjectPartInfo right)
|
||||||
|
{
|
||||||
|
return left.Equals(right);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator !=(ObjectPartInfo left, ObjectPartInfo right)
|
||||||
|
{
|
||||||
|
return !(left == right);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool Equals(ObjectPartInfo other)
|
||||||
|
{
|
||||||
|
return GetHashCode() == other.GetHashCode();
|
||||||
|
}
|
||||||
|
}
|
48
src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs
Normal file
48
src/FrostFS.SDK.Client/Models/Object/UploadProgressInfo.cs
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.ObjectModel;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
|
public class UploadProgressInfo
|
||||||
|
{
|
||||||
|
private List<ObjectPartInfo> _parts;
|
||||||
|
|
||||||
|
public UploadProgressInfo(Guid splitId, int capacity = 8)
|
||||||
|
{
|
||||||
|
_parts = new List<ObjectPartInfo>(capacity);
|
||||||
|
SplitId = new SplitId(splitId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public UploadProgressInfo(Guid splitId, Collection<ObjectPartInfo> parts)
|
||||||
|
{
|
||||||
|
_parts = [.. parts];
|
||||||
|
SplitId = new SplitId(splitId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public event EventHandler<PartUploadedEventArgs>? Notify;
|
||||||
|
|
||||||
|
public SplitId SplitId { get; }
|
||||||
|
|
||||||
|
internal void AddPart(ObjectPartInfo part)
|
||||||
|
{
|
||||||
|
_parts.Add(part);
|
||||||
|
Notify?.Invoke(this, new PartUploadedEventArgs(part));
|
||||||
|
}
|
||||||
|
|
||||||
|
public ObjectPartInfo GetPart(int index)
|
||||||
|
{
|
||||||
|
return _parts[index];
|
||||||
|
}
|
||||||
|
|
||||||
|
public ObjectPartInfo GetLast()
|
||||||
|
{
|
||||||
|
return _parts.LastOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
public ReadOnlyCollection<ObjectPartInfo> GetParts()
|
||||||
|
{
|
||||||
|
return new ReadOnlyCollection<ObjectPartInfo>(_parts);
|
||||||
|
}
|
||||||
|
}
|
|
@ -3,6 +3,7 @@ namespace FrostFS.SDK;
|
||||||
public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = null, string? details = null)
|
public class FrostFsResponseStatus(FrostFsStatusCode code, string? message = null, string? details = null)
|
||||||
{
|
{
|
||||||
public FrostFsStatusCode Code { get; set; } = code;
|
public FrostFsStatusCode Code { get; set; } = code;
|
||||||
|
|
||||||
public string Message { get; set; } = message ?? string.Empty;
|
public string Message { get; set; } = message ?? string.Empty;
|
||||||
|
|
||||||
public string Details { get; set; } = details ?? string.Empty;
|
public string Details { get; set; } = details ?? string.Empty;
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
namespace FrostFS.SDK;
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
public class MetaHeader(FrostFsVersion version, int epoch, int ttl)
|
public class MetaHeader(FrostFsVersion version, ulong epoch, uint ttl)
|
||||||
{
|
{
|
||||||
public FrostFsVersion Version { get; set; } = version;
|
public FrostFsVersion Version { get; set; } = version;
|
||||||
public int Epoch { get; set; } = epoch;
|
public ulong Epoch { get; set; } = epoch;
|
||||||
public int Ttl { get; set; } = ttl;
|
public uint Ttl { get; set; } = ttl;
|
||||||
|
|
||||||
public static MetaHeader Default()
|
public static MetaHeader Default()
|
||||||
{
|
{
|
||||||
|
|
|
@ -8,7 +8,8 @@ public readonly struct PrmObjectClientCutPut(
|
||||||
int bufferMaxSize = 0,
|
int bufferMaxSize = 0,
|
||||||
FrostFsSessionToken? sessionToken = null,
|
FrostFsSessionToken? sessionToken = null,
|
||||||
byte[]? customBuffer = null,
|
byte[]? customBuffer = null,
|
||||||
string[]? xheaders = null) : PrmObjectPutBase, System.IEquatable<PrmObjectClientCutPut>
|
string[]? xheaders = null,
|
||||||
|
UploadProgressInfo? progress = null) : PrmObjectPutBase, System.IEquatable<PrmObjectClientCutPut>
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Need to provide values like <c>ContainerId</c> and <c>ObjectType</c> to create and object.
|
/// Need to provide values like <c>ContainerId</c> and <c>ObjectType</c> to create and object.
|
||||||
|
@ -41,6 +42,8 @@ public readonly struct PrmObjectClientCutPut(
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string[] XHeaders { get; } = xheaders ?? [];
|
public string[] XHeaders { get; } = xheaders ?? [];
|
||||||
|
|
||||||
|
public UploadProgressInfo? Progress { get; } = progress;
|
||||||
|
|
||||||
internal PutObjectContext PutObjectContext { get; } = new();
|
internal PutObjectContext PutObjectContext { get; } = new();
|
||||||
|
|
||||||
public override readonly bool Equals(object obj)
|
public override readonly bool Equals(object obj)
|
||||||
|
|
|
@ -1,163 +0,0 @@
|
||||||
using System;
|
|
||||||
using System.Threading;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
// clientStatusMonitor count error rate and other statistics for connection.
|
|
||||||
public class ClientStatusMonitor : IClientStatus
|
|
||||||
{
|
|
||||||
private static readonly MethodIndex[] MethodIndexes =
|
|
||||||
[
|
|
||||||
MethodIndex.methodBalanceGet,
|
|
||||||
MethodIndex.methodContainerPut,
|
|
||||||
MethodIndex.methodContainerGet,
|
|
||||||
MethodIndex.methodContainerList,
|
|
||||||
MethodIndex.methodContainerDelete,
|
|
||||||
MethodIndex.methodEndpointInfo,
|
|
||||||
MethodIndex.methodNetworkInfo,
|
|
||||||
MethodIndex.methodNetMapSnapshot,
|
|
||||||
MethodIndex.methodObjectPut,
|
|
||||||
MethodIndex.methodObjectDelete,
|
|
||||||
MethodIndex.methodObjectGet,
|
|
||||||
MethodIndex.methodObjectHead,
|
|
||||||
MethodIndex.methodObjectRange,
|
|
||||||
MethodIndex.methodObjectPatch,
|
|
||||||
MethodIndex.methodSessionCreate,
|
|
||||||
MethodIndex.methodAPEManagerAddChain,
|
|
||||||
MethodIndex.methodAPEManagerRemoveChain,
|
|
||||||
MethodIndex.methodAPEManagerListChains
|
|
||||||
];
|
|
||||||
|
|
||||||
public static string GetMethodName(MethodIndex index)
|
|
||||||
{
|
|
||||||
return index switch
|
|
||||||
{
|
|
||||||
MethodIndex.methodBalanceGet => "BalanceGet",
|
|
||||||
MethodIndex.methodContainerPut => "ContainerPut",
|
|
||||||
MethodIndex.methodContainerGet => "ContainerGet",
|
|
||||||
MethodIndex.methodContainerList => "ContainerList",
|
|
||||||
MethodIndex.methodContainerDelete => "ContainerDelete",
|
|
||||||
MethodIndex.methodEndpointInfo => "EndpointInfo",
|
|
||||||
MethodIndex.methodNetworkInfo => "NetworkInfo",
|
|
||||||
MethodIndex.methodNetMapSnapshot => "NetMapSnapshot",
|
|
||||||
MethodIndex.methodObjectPut => "ObjectPut",
|
|
||||||
MethodIndex.methodObjectDelete => "ObjectDelete",
|
|
||||||
MethodIndex.methodObjectGet => "ObjectGet",
|
|
||||||
MethodIndex.methodObjectHead => "ObjectHead",
|
|
||||||
MethodIndex.methodObjectRange => "ObjectRange",
|
|
||||||
MethodIndex.methodObjectPatch => "ObjectPatch",
|
|
||||||
MethodIndex.methodSessionCreate => "SessionCreate",
|
|
||||||
MethodIndex.methodAPEManagerAddChain => "APEManagerAddChain",
|
|
||||||
MethodIndex.methodAPEManagerRemoveChain => "APEManagerRemoveChain",
|
|
||||||
MethodIndex.methodAPEManagerListChains => "APEManagerListChains",
|
|
||||||
_ => throw new ArgumentException("Unknown method", nameof(index)),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private readonly object _lock = new();
|
|
||||||
|
|
||||||
private readonly ILogger? logger;
|
|
||||||
private int healthy;
|
|
||||||
|
|
||||||
public ClientStatusMonitor(ILogger? logger, string address)
|
|
||||||
{
|
|
||||||
this.logger = logger;
|
|
||||||
healthy = (int)HealthyStatus.Healthy;
|
|
||||||
|
|
||||||
Address = address;
|
|
||||||
Methods = new MethodStatus[MethodIndexes.Length];
|
|
||||||
|
|
||||||
for (int i = 0; i < MethodIndexes.Length; i++)
|
|
||||||
{
|
|
||||||
Methods[i] = new MethodStatus(GetMethodName(MethodIndexes[i]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Address { get; }
|
|
||||||
|
|
||||||
internal uint ErrorThreshold { get; set; }
|
|
||||||
|
|
||||||
public uint CurrentErrorCount { get; set; }
|
|
||||||
|
|
||||||
public ulong OverallErrorCount { get; set; }
|
|
||||||
|
|
||||||
public MethodStatus[] Methods { get; private set; }
|
|
||||||
|
|
||||||
public bool IsHealthy()
|
|
||||||
{
|
|
||||||
var res = Interlocked.CompareExchange(ref healthy, -1, -1) == (int)HealthyStatus.Healthy;
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool IsDialed()
|
|
||||||
{
|
|
||||||
return Interlocked.CompareExchange(ref healthy, -1, -1) != (int)HealthyStatus.UnhealthyOnDial;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void SetHealthy()
|
|
||||||
{
|
|
||||||
Interlocked.Exchange(ref healthy, (int)HealthyStatus.Healthy);
|
|
||||||
}
|
|
||||||
public void SetUnhealthy()
|
|
||||||
{
|
|
||||||
Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnRequest);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void SetUnhealthyOnDial()
|
|
||||||
{
|
|
||||||
Interlocked.Exchange(ref healthy, (int)HealthyStatus.UnhealthyOnDial);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void IncErrorRate()
|
|
||||||
{
|
|
||||||
bool thresholdReached;
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
CurrentErrorCount++;
|
|
||||||
OverallErrorCount++;
|
|
||||||
|
|
||||||
thresholdReached = CurrentErrorCount >= ErrorThreshold;
|
|
||||||
|
|
||||||
if (thresholdReached)
|
|
||||||
{
|
|
||||||
SetUnhealthy();
|
|
||||||
CurrentErrorCount = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (thresholdReached && logger != null)
|
|
||||||
{
|
|
||||||
FrostFsMessages.ErrorЕhresholdReached(logger, Address, ErrorThreshold);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public uint GetCurrentErrorRate()
|
|
||||||
{
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
return CurrentErrorCount;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public ulong GetOverallErrorRate()
|
|
||||||
{
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
return OverallErrorCount;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public StatusSnapshot[] MethodsStatus()
|
|
||||||
{
|
|
||||||
var result = new StatusSnapshot[Methods.Length];
|
|
||||||
|
|
||||||
for (int i = 0; i < result.Length; i++)
|
|
||||||
{
|
|
||||||
result[i] = Methods[i].Snapshot!;
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,135 +0,0 @@
|
||||||
using System;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
using Grpc.Core;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
// clientWrapper is used by default, alternative implementations are intended for testing purposes only.
|
|
||||||
public class ClientWrapper : ClientStatusMonitor
|
|
||||||
{
|
|
||||||
private readonly object _lock = new();
|
|
||||||
private SessionCache sessionCache;
|
|
||||||
|
|
||||||
internal ClientWrapper(WrapperPrm wrapperPrm, Pool pool) : base(wrapperPrm.Logger, wrapperPrm.Address)
|
|
||||||
{
|
|
||||||
WrapperPrm = wrapperPrm;
|
|
||||||
ErrorThreshold = wrapperPrm.ErrorThreshold;
|
|
||||||
|
|
||||||
sessionCache = pool.SessionCache;
|
|
||||||
Client = new FrostFSClient(WrapperPrm, sessionCache);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal FrostFSClient? Client { get; private set; }
|
|
||||||
|
|
||||||
internal WrapperPrm WrapperPrm { get; }
|
|
||||||
|
|
||||||
internal FrostFSClient? GetClient()
|
|
||||||
{
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
if (IsHealthy())
|
|
||||||
{
|
|
||||||
return Client;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// dial establishes a connection to the server from the FrostFS network.
|
|
||||||
// Returns an error describing failure reason. If failed, the client
|
|
||||||
// SHOULD NOT be used.
|
|
||||||
internal async Task Dial(CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = GetClient() ?? throw new FrostFsInvalidObjectException("pool client unhealthy");
|
|
||||||
|
|
||||||
await client.Dial(ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void HandleError(Exception ex)
|
|
||||||
{
|
|
||||||
if (ex is FrostFsResponseException responseException && responseException.Status != null)
|
|
||||||
{
|
|
||||||
switch (responseException.Status.Code)
|
|
||||||
{
|
|
||||||
case FrostFsStatusCode.Internal:
|
|
||||||
case FrostFsStatusCode.WrongMagicNumber:
|
|
||||||
case FrostFsStatusCode.SignatureVerificationFailure:
|
|
||||||
case FrostFsStatusCode.NodeUnderMaintenance:
|
|
||||||
IncErrorRate();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IncErrorRate();
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task ScheduleGracefulClose()
|
|
||||||
{
|
|
||||||
if (Client == null)
|
|
||||||
return;
|
|
||||||
|
|
||||||
await Task.Delay((int)WrapperPrm.GracefulCloseOnSwitchTimeout).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
// restartIfUnhealthy checks healthy status of client and recreate it if status is unhealthy.
|
|
||||||
// Indicating if status was changed by this function call and returns error that caused unhealthy status.
|
|
||||||
internal async Task<bool> RestartIfUnhealthy(CallContext ctx)
|
|
||||||
{
|
|
||||||
bool wasHealthy;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var response = await Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (RpcException)
|
|
||||||
{
|
|
||||||
wasHealthy = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// if connection is dialed before, to avoid routine/connection leak,
|
|
||||||
// pool has to close it and then initialize once again.
|
|
||||||
if (IsDialed())
|
|
||||||
{
|
|
||||||
await ScheduleGracefulClose().ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
FrostFSClient? client = new(WrapperPrm, sessionCache);
|
|
||||||
|
|
||||||
var error = await client.Dial(ctx).ConfigureAwait(false);
|
|
||||||
if (!string.IsNullOrEmpty(error))
|
|
||||||
{
|
|
||||||
SetUnhealthyOnDial();
|
|
||||||
return wasHealthy;
|
|
||||||
}
|
|
||||||
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
Client = client;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var res = await Client.GetNodeInfoAsync(ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
catch (FrostFsException)
|
|
||||||
{
|
|
||||||
SetUnhealthy();
|
|
||||||
|
|
||||||
return wasHealthy;
|
|
||||||
}
|
|
||||||
|
|
||||||
SetHealthy();
|
|
||||||
return !wasHealthy;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void IncRequests(ulong elapsed, MethodIndex method)
|
|
||||||
{
|
|
||||||
var methodStat = Methods[(int)method];
|
|
||||||
|
|
||||||
methodStat.IncRequests(elapsed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,18 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
// values for healthy status of clientStatusMonitor.
|
|
||||||
public enum HealthyStatus
|
|
||||||
{
|
|
||||||
// statusUnhealthyOnDial is set when dialing to the endpoint is failed,
|
|
||||||
// so there is no connection to the endpoint, and pool should not close it
|
|
||||||
// before re-establishing connection once again.
|
|
||||||
UnhealthyOnDial,
|
|
||||||
|
|
||||||
// statusUnhealthyOnRequest is set when communication after dialing to the
|
|
||||||
// endpoint is failed due to immediate or accumulated errors, connection is
|
|
||||||
// available and pool should close it before re-establishing connection once again.
|
|
||||||
UnhealthyOnRequest,
|
|
||||||
|
|
||||||
// statusHealthy is set when connection is ready to be used by the pool.
|
|
||||||
Healthy
|
|
||||||
}
|
|
|
@ -1,28 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public interface IClientStatus
|
|
||||||
{
|
|
||||||
// isHealthy checks if the connection can handle requests.
|
|
||||||
bool IsHealthy();
|
|
||||||
|
|
||||||
// isDialed checks if the connection was created.
|
|
||||||
bool IsDialed();
|
|
||||||
|
|
||||||
// setUnhealthy marks client as unhealthy.
|
|
||||||
void SetUnhealthy();
|
|
||||||
|
|
||||||
// address return address of endpoint.
|
|
||||||
string Address { get; }
|
|
||||||
|
|
||||||
// currentErrorRate returns current errors rate.
|
|
||||||
// After specific threshold connection is considered as unhealthy.
|
|
||||||
// Pool.startRebalance routine can make this connection healthy again.
|
|
||||||
uint GetCurrentErrorRate();
|
|
||||||
|
|
||||||
// overallErrorRate returns the number of all happened errors.
|
|
||||||
ulong GetOverallErrorRate();
|
|
||||||
|
|
||||||
// methodsStatus returns statistic for all used methods.
|
|
||||||
StatusSnapshot[] MethodsStatus();
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,45 +0,0 @@
|
||||||
using System;
|
|
||||||
using System.Collections.ObjectModel;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using Grpc.Core;
|
|
||||||
using Grpc.Core.Interceptors;
|
|
||||||
using Grpc.Net.Client;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
// InitParameters contains values used to initialize connection Pool.
|
|
||||||
public class InitParameters(Func<string, ChannelBase> grpcChannelFactory)
|
|
||||||
{
|
|
||||||
public ECDsa? Key { get; set; }
|
|
||||||
|
|
||||||
public ulong NodeDialTimeout { get; set; }
|
|
||||||
|
|
||||||
public ulong NodeStreamTimeout { get; set; }
|
|
||||||
|
|
||||||
public ulong HealthcheckTimeout { get; set; }
|
|
||||||
|
|
||||||
public ulong ClientRebalanceInterval { get; set; }
|
|
||||||
|
|
||||||
public ulong SessionExpirationDuration { get; set; }
|
|
||||||
|
|
||||||
public uint ErrorThreshold { get; set; }
|
|
||||||
|
|
||||||
public NodeParam[]? NodeParams { get; set; }
|
|
||||||
|
|
||||||
public GrpcChannelOptions[]? DialOptions { get; set; }
|
|
||||||
|
|
||||||
public Func<string, ClientWrapper>? ClientBuilder { get; set; }
|
|
||||||
|
|
||||||
public ulong GracefulCloseOnSwitchTimeout { get; set; }
|
|
||||||
|
|
||||||
public ILogger? Logger { get; set; }
|
|
||||||
|
|
||||||
public Action<CallStatistics>? Callback { get; set; }
|
|
||||||
|
|
||||||
public Collection<Interceptor> Interceptors { get; } = [];
|
|
||||||
|
|
||||||
public Func<string, ChannelBase> GrpcChannelFactory { get; set; } = grpcChannelFactory;
|
|
||||||
}
|
|
|
@ -1,47 +0,0 @@
|
||||||
using FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
internal sealed class InnerPool
|
|
||||||
{
|
|
||||||
private readonly object _lock = new();
|
|
||||||
|
|
||||||
internal InnerPool(Sampler sampler, ClientWrapper[] clients)
|
|
||||||
{
|
|
||||||
Clients = clients;
|
|
||||||
Sampler = sampler;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal Sampler Sampler { get; set; }
|
|
||||||
|
|
||||||
internal ClientWrapper[] Clients { get; }
|
|
||||||
|
|
||||||
internal ClientWrapper? Connection()
|
|
||||||
{
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
if (Clients.Length == 1)
|
|
||||||
{
|
|
||||||
var client = Clients[0];
|
|
||||||
if (client.IsHealthy())
|
|
||||||
{
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
var attempts = 3 * Clients.Length;
|
|
||||||
|
|
||||||
for (int i = 0; i < attempts; i++)
|
|
||||||
{
|
|
||||||
int index = Sampler.Next();
|
|
||||||
|
|
||||||
if (Clients[index].IsHealthy())
|
|
||||||
{
|
|
||||||
return Clients[index];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public enum MethodIndex
|
|
||||||
{
|
|
||||||
methodBalanceGet,
|
|
||||||
methodContainerPut,
|
|
||||||
methodContainerGet,
|
|
||||||
methodContainerList,
|
|
||||||
methodContainerDelete,
|
|
||||||
methodEndpointInfo,
|
|
||||||
methodNetworkInfo,
|
|
||||||
methodNetMapSnapshot,
|
|
||||||
methodObjectPut,
|
|
||||||
methodObjectDelete,
|
|
||||||
methodObjectGet,
|
|
||||||
methodObjectHead,
|
|
||||||
methodObjectRange,
|
|
||||||
methodObjectPatch,
|
|
||||||
methodSessionCreate,
|
|
||||||
methodAPEManagerAddChain,
|
|
||||||
methodAPEManagerRemoveChain,
|
|
||||||
methodAPEManagerListChains,
|
|
||||||
methodLast
|
|
||||||
}
|
|
|
@ -1,19 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public class MethodStatus(string name)
|
|
||||||
{
|
|
||||||
private readonly object _lock = new();
|
|
||||||
|
|
||||||
public string Name { get; } = name;
|
|
||||||
|
|
||||||
public StatusSnapshot Snapshot { get; set; } = new StatusSnapshot();
|
|
||||||
|
|
||||||
internal void IncRequests(ulong elapsed)
|
|
||||||
{
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
Snapshot.AllTime += elapsed;
|
|
||||||
Snapshot.AllRequests++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,12 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
// NodeParam groups parameters of remote node.
|
|
||||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>")]
|
|
||||||
public readonly struct NodeParam(int priority, string address, float weight)
|
|
||||||
{
|
|
||||||
public int Priority { get; } = priority;
|
|
||||||
|
|
||||||
public string Address { get; } = address;
|
|
||||||
|
|
||||||
public float Weight { get; } = weight;
|
|
||||||
}
|
|
|
@ -1,12 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public class NodeStatistic
|
|
||||||
{
|
|
||||||
public string? Address { get; internal set; }
|
|
||||||
|
|
||||||
public StatusSnapshot[]? Methods { get; internal set; }
|
|
||||||
|
|
||||||
public ulong OverallErrors { get; internal set; }
|
|
||||||
|
|
||||||
public uint CurrentErrors { get; internal set; }
|
|
||||||
}
|
|
|
@ -1,12 +0,0 @@
|
||||||
using System.Collections.ObjectModel;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public class NodesParam(int priority)
|
|
||||||
{
|
|
||||||
public int Priority { get; } = priority;
|
|
||||||
|
|
||||||
public Collection<string> Addresses { get; } = [];
|
|
||||||
|
|
||||||
public Collection<double> Weights { get; } = [];
|
|
||||||
}
|
|
|
@ -1,677 +0,0 @@
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
using FrostFS.Refs;
|
|
||||||
using FrostFS.SDK.Client.Interfaces;
|
|
||||||
using FrostFS.SDK.Client.Mappers.GRPC;
|
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
|
|
||||||
using Grpc.Core;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public partial class Pool : IFrostFSClient
|
|
||||||
{
|
|
||||||
const int defaultSessionTokenExpirationDuration = 100; // in epochs
|
|
||||||
|
|
||||||
const int defaultErrorThreshold = 100;
|
|
||||||
|
|
||||||
const int defaultGracefulCloseOnSwitchTimeout = 10; //Seconds;
|
|
||||||
const int defaultRebalanceInterval = 15; //Seconds;
|
|
||||||
const int defaultHealthcheckTimeout = 4; //Seconds;
|
|
||||||
const int defaultDialTimeout = 5; //Seconds;
|
|
||||||
const int defaultStreamTimeout = 10; //Seconds;
|
|
||||||
|
|
||||||
private readonly object _lock = new();
|
|
||||||
|
|
||||||
private InnerPool[]? InnerPools { get; set; }
|
|
||||||
|
|
||||||
private ClientKey Key { get; set; }
|
|
||||||
|
|
||||||
private OwnerID? _ownerId;
|
|
||||||
|
|
||||||
private FrostFsVersion _version;
|
|
||||||
|
|
||||||
private FrostFsOwner? _owner;
|
|
||||||
|
|
||||||
private FrostFsOwner Owner
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
_owner ??= new FrostFsOwner(Key.ECDsaKey.PublicKey().PublicKeyToAddress());
|
|
||||||
return _owner;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private OwnerID OwnerId
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (_ownerId == null)
|
|
||||||
{
|
|
||||||
_owner = Key.Owner;
|
|
||||||
_ownerId = _owner.ToMessage();
|
|
||||||
}
|
|
||||||
return _ownerId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal CancellationTokenSource CancellationTokenSource { get; } = new CancellationTokenSource();
|
|
||||||
|
|
||||||
internal SessionCache SessionCache { get; set; }
|
|
||||||
|
|
||||||
private ulong SessionTokenDuration { get; set; }
|
|
||||||
|
|
||||||
private RebalanceParameters RebalanceParams { get; set; }
|
|
||||||
|
|
||||||
private Func<string, ClientWrapper> ClientBuilder;
|
|
||||||
|
|
||||||
private bool disposedValue;
|
|
||||||
|
|
||||||
private ILogger? logger { get; set; }
|
|
||||||
|
|
||||||
private ulong MaxObjectSize { get; set; }
|
|
||||||
|
|
||||||
public IClientStatus? ClientStatus { get; }
|
|
||||||
|
|
||||||
// NewPool creates connection pool using parameters.
|
|
||||||
public Pool(InitParameters options)
|
|
||||||
{
|
|
||||||
if (options is null)
|
|
||||||
{
|
|
||||||
throw new ArgumentNullException(nameof(options));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.Key == null)
|
|
||||||
{
|
|
||||||
throw new ArgumentException($"Missed required parameter {nameof(options.Key)}");
|
|
||||||
}
|
|
||||||
|
|
||||||
_version = new FrostFsVersion(2, 13);
|
|
||||||
|
|
||||||
var nodesParams = AdjustNodeParams(options.NodeParams);
|
|
||||||
|
|
||||||
var cache = new SessionCache(options.SessionExpirationDuration);
|
|
||||||
|
|
||||||
FillDefaultInitParams(options, this);
|
|
||||||
|
|
||||||
Key = new ClientKey(options.Key);
|
|
||||||
|
|
||||||
SessionCache = cache;
|
|
||||||
logger = options.Logger;
|
|
||||||
SessionTokenDuration = options.SessionExpirationDuration;
|
|
||||||
|
|
||||||
RebalanceParams = new RebalanceParameters(
|
|
||||||
nodesParams.ToArray(),
|
|
||||||
options.HealthcheckTimeout,
|
|
||||||
options.ClientRebalanceInterval,
|
|
||||||
options.SessionExpirationDuration);
|
|
||||||
|
|
||||||
ClientBuilder = options.ClientBuilder!;
|
|
||||||
|
|
||||||
// ClientContext.PoolErrorHandler = client.HandleError;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dial establishes a connection to the servers from the FrostFS network.
|
|
||||||
// It also starts a routine that checks the health of the nodes and
|
|
||||||
// updates the weights of the nodes for balancing.
|
|
||||||
// Returns an error describing failure reason.
|
|
||||||
//
|
|
||||||
// If failed, the Pool SHOULD NOT be used.
|
|
||||||
//
|
|
||||||
// See also InitParameters.SetClientRebalanceInterval.
|
|
||||||
public async Task<string?> Dial(CallContext ctx)
|
|
||||||
{
|
|
||||||
var inner = new InnerPool[RebalanceParams.NodesParams.Length];
|
|
||||||
|
|
||||||
bool atLeastOneHealthy = false;
|
|
||||||
int i = 0;
|
|
||||||
foreach (var nodeParams in RebalanceParams.NodesParams)
|
|
||||||
{
|
|
||||||
var wrappers = new ClientWrapper[nodeParams.Weights.Count];
|
|
||||||
|
|
||||||
for (int j = 0; j < nodeParams.Addresses.Count; j++)
|
|
||||||
{
|
|
||||||
ClientWrapper? wrapper = null;
|
|
||||||
bool dialed = false;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
wrapper = wrappers[j] = ClientBuilder(nodeParams.Addresses[j]);
|
|
||||||
|
|
||||||
await wrapper.Dial(ctx).ConfigureAwait(false);
|
|
||||||
dialed = true;
|
|
||||||
|
|
||||||
var token = await InitSessionForDuration(ctx, wrapper, RebalanceParams.SessionExpirationDuration, Key.ECDsaKey, false)
|
|
||||||
.ConfigureAwait(false);
|
|
||||||
|
|
||||||
var key = FormCacheKey(nodeParams.Addresses[j], Key.PublicKey);
|
|
||||||
SessionCache.SetValue(key, token);
|
|
||||||
|
|
||||||
atLeastOneHealthy = true;
|
|
||||||
}
|
|
||||||
catch (RpcException ex)
|
|
||||||
{
|
|
||||||
if (!dialed)
|
|
||||||
wrapper!.SetUnhealthyOnDial();
|
|
||||||
else
|
|
||||||
wrapper!.SetUnhealthy();
|
|
||||||
|
|
||||||
if (logger != null)
|
|
||||||
{
|
|
||||||
FrostFsMessages.SessionCreationError(logger, wrapper!.WrapperPrm.Address, ex.Message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (FrostFsInvalidObjectException)
|
|
||||||
{
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var sampler = new Sampler(nodeParams.Weights.ToArray());
|
|
||||||
|
|
||||||
inner[i] = new InnerPool(sampler, wrappers);
|
|
||||||
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!atLeastOneHealthy)
|
|
||||||
return "At least one node must be healthy";
|
|
||||||
|
|
||||||
InnerPools = inner;
|
|
||||||
|
|
||||||
var res = await GetNetworkSettingsAsync(default).ConfigureAwait(false);
|
|
||||||
|
|
||||||
MaxObjectSize = res.MaxObjectSize;
|
|
||||||
|
|
||||||
StartRebalance(ctx);
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IEnumerable<NodesParam> AdjustNodeParams(NodeParam[]? nodeParams)
|
|
||||||
{
|
|
||||||
if (nodeParams == null || nodeParams.Length == 0)
|
|
||||||
{
|
|
||||||
throw new ArgumentException("No FrostFS peers configured");
|
|
||||||
}
|
|
||||||
|
|
||||||
Dictionary<int, NodesParam> nodesParamsDict = new(nodeParams.Length);
|
|
||||||
foreach (var nodeParam in nodeParams)
|
|
||||||
{
|
|
||||||
if (!nodesParamsDict.TryGetValue(nodeParam.Priority, out var nodes))
|
|
||||||
{
|
|
||||||
nodes = new NodesParam(nodeParam.Priority);
|
|
||||||
nodesParamsDict[nodeParam.Priority] = nodes;
|
|
||||||
}
|
|
||||||
|
|
||||||
nodes.Addresses.Add(nodeParam.Address);
|
|
||||||
nodes.Weights.Add(nodeParam.Weight);
|
|
||||||
}
|
|
||||||
|
|
||||||
var nodesParams = new List<NodesParam>(nodesParamsDict.Count);
|
|
||||||
|
|
||||||
foreach (var key in nodesParamsDict.Keys)
|
|
||||||
{
|
|
||||||
var nodes = nodesParamsDict[key];
|
|
||||||
var newWeights = AdjustWeights([.. nodes.Weights]);
|
|
||||||
nodes.Weights.Clear();
|
|
||||||
foreach (var weight in newWeights)
|
|
||||||
{
|
|
||||||
nodes.Weights.Add(weight);
|
|
||||||
}
|
|
||||||
|
|
||||||
nodesParams.Add(nodes);
|
|
||||||
}
|
|
||||||
|
|
||||||
return nodesParams.OrderBy(n => n.Priority);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static double[] AdjustWeights(double[] weights)
|
|
||||||
{
|
|
||||||
var adjusted = new double[weights.Length];
|
|
||||||
|
|
||||||
var sum = weights.Sum();
|
|
||||||
|
|
||||||
if (sum > 0)
|
|
||||||
{
|
|
||||||
for (int i = 0; i < weights.Length; i++)
|
|
||||||
{
|
|
||||||
adjusted[i] = weights[i] / sum;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return adjusted;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void FillDefaultInitParams(InitParameters parameters, Pool pool)
|
|
||||||
{
|
|
||||||
if (parameters.SessionExpirationDuration == 0)
|
|
||||||
parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration;
|
|
||||||
|
|
||||||
if (parameters.ErrorThreshold == 0)
|
|
||||||
parameters.ErrorThreshold = defaultErrorThreshold;
|
|
||||||
|
|
||||||
if (parameters.ClientRebalanceInterval <= 0)
|
|
||||||
parameters.ClientRebalanceInterval = defaultRebalanceInterval;
|
|
||||||
|
|
||||||
if (parameters.GracefulCloseOnSwitchTimeout <= 0)
|
|
||||||
parameters.GracefulCloseOnSwitchTimeout = defaultGracefulCloseOnSwitchTimeout;
|
|
||||||
|
|
||||||
if (parameters.HealthcheckTimeout <= 0)
|
|
||||||
parameters.HealthcheckTimeout = defaultHealthcheckTimeout;
|
|
||||||
|
|
||||||
if (parameters.NodeDialTimeout <= 0)
|
|
||||||
parameters.NodeDialTimeout = defaultDialTimeout;
|
|
||||||
|
|
||||||
if (parameters.NodeStreamTimeout <= 0)
|
|
||||||
parameters.NodeStreamTimeout = defaultStreamTimeout;
|
|
||||||
|
|
||||||
if (parameters.SessionExpirationDuration == 0)
|
|
||||||
parameters.SessionExpirationDuration = defaultSessionTokenExpirationDuration;
|
|
||||||
|
|
||||||
parameters.ClientBuilder ??= new Func<string, ClientWrapper>((address) =>
|
|
||||||
{
|
|
||||||
var wrapperPrm = new WrapperPrm()
|
|
||||||
{
|
|
||||||
Address = address,
|
|
||||||
Key = parameters.Key!,
|
|
||||||
Logger = parameters.Logger,
|
|
||||||
DialTimeout = parameters.NodeDialTimeout,
|
|
||||||
StreamTimeout = parameters.NodeStreamTimeout,
|
|
||||||
ErrorThreshold = parameters.ErrorThreshold,
|
|
||||||
GracefulCloseOnSwitchTimeout = parameters.GracefulCloseOnSwitchTimeout,
|
|
||||||
Callback = parameters.Callback,
|
|
||||||
Interceptors = parameters.Interceptors,
|
|
||||||
GrpcChannelFactory = parameters.GrpcChannelFactory
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
return new ClientWrapper(wrapperPrm, pool);
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private ClientWrapper Connection()
|
|
||||||
{
|
|
||||||
foreach (var pool in InnerPools!)
|
|
||||||
{
|
|
||||||
var client = pool.Connection();
|
|
||||||
if (client != null)
|
|
||||||
{
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new FrostFsException("Cannot find alive client");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task<FrostFsSessionToken> InitSessionForDuration(CallContext ctx, ClientWrapper cw, ulong duration, ECDsa key, bool clientCut)
|
|
||||||
{
|
|
||||||
var client = cw.Client;
|
|
||||||
var networkInfo = await client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
|
|
||||||
|
|
||||||
var epoch = networkInfo.Epoch;
|
|
||||||
|
|
||||||
ulong exp = ulong.MaxValue - epoch < duration
|
|
||||||
? ulong.MaxValue
|
|
||||||
: epoch + duration;
|
|
||||||
|
|
||||||
var prmSessionCreate = new PrmSessionCreate(exp);
|
|
||||||
|
|
||||||
return await client.CreateSessionAsync(prmSessionCreate, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static string FormCacheKey(string address, string key)
|
|
||||||
{
|
|
||||||
return $"{address}{key}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Close()
|
|
||||||
{
|
|
||||||
CancellationTokenSource.Cancel();
|
|
||||||
|
|
||||||
//if (InnerPools != null)
|
|
||||||
//{
|
|
||||||
// // close all clients
|
|
||||||
// foreach (var innerPool in InnerPools)
|
|
||||||
// foreach (var client in innerPool.Clients)
|
|
||||||
// if (client.IsDialed())
|
|
||||||
// client.Client?.Close();
|
|
||||||
//}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startRebalance runs loop to monitor connection healthy status.
|
|
||||||
internal void StartRebalance(CallContext ctx)
|
|
||||||
{
|
|
||||||
var buffers = new double[RebalanceParams.NodesParams.Length][];
|
|
||||||
|
|
||||||
for (int i = 0; i < RebalanceParams.NodesParams.Length; i++)
|
|
||||||
{
|
|
||||||
var parameters = RebalanceParams.NodesParams[i];
|
|
||||||
buffers[i] = new double[parameters.Weights.Count];
|
|
||||||
|
|
||||||
Task.Run(async () =>
|
|
||||||
{
|
|
||||||
await Task.Delay((int)RebalanceParams.ClientRebalanceInterval).ConfigureAwait(false);
|
|
||||||
UpdateNodesHealth(ctx, buffers);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void UpdateNodesHealth(CallContext ctx, double[][] buffers)
|
|
||||||
{
|
|
||||||
var tasks = new Task[InnerPools!.Length];
|
|
||||||
|
|
||||||
for (int i = 0; i < InnerPools.Length; i++)
|
|
||||||
{
|
|
||||||
var bufferWeights = buffers[i];
|
|
||||||
|
|
||||||
tasks[i] = Task.Run(() => UpdateInnerNodesHealth(ctx, i, bufferWeights));
|
|
||||||
}
|
|
||||||
|
|
||||||
Task.WaitAll(tasks);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async ValueTask UpdateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights)
|
|
||||||
{
|
|
||||||
if (poolIndex > InnerPools!.Length - 1)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var pool = InnerPools[poolIndex];
|
|
||||||
|
|
||||||
var options = RebalanceParams;
|
|
||||||
|
|
||||||
int healthyChanged = 0;
|
|
||||||
|
|
||||||
var tasks = new Task[pool.Clients.Length];
|
|
||||||
|
|
||||||
for (int j = 0; j < pool.Clients.Length; j++)
|
|
||||||
{
|
|
||||||
var client = pool.Clients[j];
|
|
||||||
var healthy = false;
|
|
||||||
string? error = null;
|
|
||||||
var changed = false;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
// check timeout settings
|
|
||||||
changed = await client!.RestartIfUnhealthy(ctx).ConfigureAwait(false);
|
|
||||||
healthy = true;
|
|
||||||
bufferWeights[j] = options.NodesParams[poolIndex].Weights[j];
|
|
||||||
}
|
|
||||||
// TODO: specify
|
|
||||||
catch (FrostFsException e)
|
|
||||||
{
|
|
||||||
error = e.Message;
|
|
||||||
bufferWeights[j] = 0;
|
|
||||||
|
|
||||||
SessionCache.DeleteByPrefix(client.Address);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (changed)
|
|
||||||
{
|
|
||||||
if (!string.IsNullOrEmpty(error))
|
|
||||||
{
|
|
||||||
if (logger != null)
|
|
||||||
{
|
|
||||||
FrostFsMessages.HealthChanged(logger, client.Address, healthy, error!);
|
|
||||||
}
|
|
||||||
|
|
||||||
Interlocked.Exchange(ref healthyChanged, 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await Task.WhenAll(tasks).ConfigureAwait(false);
|
|
||||||
|
|
||||||
if (Interlocked.CompareExchange(ref healthyChanged, -1, -1) == 1)
|
|
||||||
{
|
|
||||||
var probabilities = AdjustWeights(bufferWeights);
|
|
||||||
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
pool.Sampler = new Sampler(probabilities);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// TODO: remove
|
|
||||||
private bool CheckSessionTokenErr(Exception error, string address)
|
|
||||||
{
|
|
||||||
if (error == null)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error is SessionNotFoundException || error is SessionExpiredException)
|
|
||||||
{
|
|
||||||
this.SessionCache.DeleteByPrefix(address);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Statistic Statistic()
|
|
||||||
{
|
|
||||||
if (InnerPools == null)
|
|
||||||
{
|
|
||||||
throw new FrostFsInvalidObjectException(nameof(Pool));
|
|
||||||
}
|
|
||||||
|
|
||||||
var statistics = new Statistic();
|
|
||||||
|
|
||||||
foreach (var inner in InnerPools)
|
|
||||||
{
|
|
||||||
int valueIndex = 0;
|
|
||||||
var nodes = new string[inner.Clients.Length];
|
|
||||||
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
foreach (var client in inner.Clients)
|
|
||||||
{
|
|
||||||
if (client.IsHealthy())
|
|
||||||
{
|
|
||||||
nodes[valueIndex] = client.Address;
|
|
||||||
}
|
|
||||||
|
|
||||||
var node = new NodeStatistic
|
|
||||||
{
|
|
||||||
Address = client.Address,
|
|
||||||
Methods = client.MethodsStatus(),
|
|
||||||
OverallErrors = client.GetOverallErrorRate(),
|
|
||||||
CurrentErrors = client.GetCurrentErrorRate()
|
|
||||||
};
|
|
||||||
|
|
||||||
statistics.Nodes.Add(node);
|
|
||||||
|
|
||||||
valueIndex++;
|
|
||||||
|
|
||||||
statistics.OverallErrors += node.OverallErrors;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (statistics.CurrentNodes == null || statistics.CurrentNodes.Length == 0)
|
|
||||||
{
|
|
||||||
statistics.CurrentNodes = nodes;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return statistics;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsNetmapSnapshot> GetNetmapSnapshotAsync(CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
|
|
||||||
return await client.Client!.GetNetmapSnapshotAsync(ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsNodeInfo> GetNodeInfoAsync(CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetNodeInfoAsync(ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<NetworkSettings> GetNetworkSettingsAsync(CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsSessionToken> CreateSessionAsync(PrmSessionCreate args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.CreateSessionAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<ReadOnlyMemory<byte>> AddChainAsync(PrmApeChainAdd args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.AddChainAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task RemoveChainAsync(PrmApeChainRemove args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
await client.Client!.RemoveChainAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsChain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.ListChainAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsContainerInfo> GetContainerAsync(PrmContainerGet args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetContainerAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IAsyncEnumerable<FrostFsContainerId> ListContainersAsync(PrmContainerGetAll args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return client.Client!.ListContainersAsync(args, ctx);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Obsolete("Use PutContainerAsync method")]
|
|
||||||
public async Task<FrostFsContainerId> CreateContainerAsync(PrmContainerCreate args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsContainerId> PutContainerAsync(PrmContainerCreate args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PutContainerAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task DeleteContainerAsync(PrmContainerDelete args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
await client.Client!.DeleteContainerAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsHeaderResult> GetObjectHeadAsync(PrmObjectHeadGet args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetObjectHeadAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsObject> GetObjectAsync(PrmObjectGet args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<IObjectWriter> PutObjectAsync(PrmObjectPut args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PutObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PutClientCutObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsObjectId> PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PutSingleObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsObjectId> PatchObjectAsync(PrmObjectPatch args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<RangeReader> GetRangeAsync(PrmRangeGet args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetRangeAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<ReadOnlyMemory<byte>[]> GetRangeHashAsync(PrmRangeHashGet args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetRangeHashAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<FrostFsObjectId> PatchAsync(PrmObjectPatch args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.PatchObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task DeleteObjectAsync(PrmObjectDelete args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
await client.Client!.DeleteObjectAsync(args, ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IAsyncEnumerable<FrostFsObjectId> SearchObjectsAsync(PrmObjectSearch args, CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return client.Client!.SearchObjectsAsync(args, ctx);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<Accounting.Decimal> GetBalanceAsync(CallContext ctx)
|
|
||||||
{
|
|
||||||
var client = Connection();
|
|
||||||
return await client.Client!.GetBalanceAsync(ctx).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected virtual void Dispose(bool disposing)
|
|
||||||
{
|
|
||||||
if (!disposedValue)
|
|
||||||
{
|
|
||||||
if (disposing)
|
|
||||||
{
|
|
||||||
Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
disposedValue = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose()
|
|
||||||
{
|
|
||||||
Dispose(disposing: true);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,16 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public class RebalanceParameters(
|
|
||||||
NodesParam[] nodesParams,
|
|
||||||
ulong nodeRequestTimeout,
|
|
||||||
ulong clientRebalanceInterval,
|
|
||||||
ulong sessionExpirationDuration)
|
|
||||||
{
|
|
||||||
public NodesParam[] NodesParams { get; set; } = nodesParams;
|
|
||||||
|
|
||||||
public ulong NodeRequestTimeout { get; set; } = nodeRequestTimeout;
|
|
||||||
|
|
||||||
public ulong ClientRebalanceInterval { get; set; } = clientRebalanceInterval;
|
|
||||||
|
|
||||||
public ulong SessionExpirationDuration { get; set; } = sessionExpirationDuration;
|
|
||||||
}
|
|
|
@ -1,14 +0,0 @@
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
// RequestInfo groups info about pool request.
|
|
||||||
struct RequestInfo
|
|
||||||
{
|
|
||||||
public string Address { get; set; }
|
|
||||||
|
|
||||||
public MethodIndex MethodIndex { get; set; }
|
|
||||||
|
|
||||||
public TimeSpan Elapsed { get; set; }
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,85 +0,0 @@
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
internal sealed class Sampler
|
|
||||||
{
|
|
||||||
private readonly object _lock = new();
|
|
||||||
|
|
||||||
private Random random = new();
|
|
||||||
|
|
||||||
internal double[] Probabilities { get; set; }
|
|
||||||
internal int[] Alias { get; set; }
|
|
||||||
|
|
||||||
internal Sampler(double[] probabilities)
|
|
||||||
{
|
|
||||||
var small = new WorkList();
|
|
||||||
var large = new WorkList();
|
|
||||||
|
|
||||||
var n = probabilities.Length;
|
|
||||||
|
|
||||||
// sampler.randomGenerator = rand.New(source)
|
|
||||||
Probabilities = new double[n];
|
|
||||||
Alias = new int[n];
|
|
||||||
|
|
||||||
// Compute scaled probabilities.
|
|
||||||
var p = new double[n];
|
|
||||||
|
|
||||||
for (int i = 0; i < n; i++)
|
|
||||||
{
|
|
||||||
p[i] = probabilities[i] * n;
|
|
||||||
if (p[i] < 1)
|
|
||||||
small.Add(i);
|
|
||||||
else
|
|
||||||
large.Add(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
while (small.Length > 0 && large.Length > 0)
|
|
||||||
{
|
|
||||||
var l = small.Remove();
|
|
||||||
var g = large.Remove();
|
|
||||||
|
|
||||||
Probabilities[l] = p[l];
|
|
||||||
Alias[l] = g;
|
|
||||||
|
|
||||||
p[g] = p[g] + p[l] - 1;
|
|
||||||
|
|
||||||
if (p[g] < 1)
|
|
||||||
small.Add(g);
|
|
||||||
else
|
|
||||||
large.Add(g);
|
|
||||||
}
|
|
||||||
|
|
||||||
while (large.Length > 0)
|
|
||||||
{
|
|
||||||
var g = large.Remove();
|
|
||||||
Probabilities[g] = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
while (small.Length > 0)
|
|
||||||
{
|
|
||||||
var l = small.Remove();
|
|
||||||
probabilities[l] = 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal int Next()
|
|
||||||
{
|
|
||||||
var n = Alias.Length;
|
|
||||||
|
|
||||||
int i;
|
|
||||||
double f;
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
i = random.Next(0, n - 1);
|
|
||||||
f = random.NextDouble();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (f < Probabilities[i])
|
|
||||||
{
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Alias[i];
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,12 +0,0 @@
|
||||||
using System.Collections.ObjectModel;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public sealed class Statistic
|
|
||||||
{
|
|
||||||
public ulong OverallErrors { get; internal set; }
|
|
||||||
|
|
||||||
public Collection<NodeStatistic> Nodes { get; } = [];
|
|
||||||
|
|
||||||
public string[]? CurrentNodes { get; internal set; }
|
|
||||||
}
|
|
|
@ -1,8 +0,0 @@
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
public class StatusSnapshot()
|
|
||||||
{
|
|
||||||
public ulong AllTime { get; internal set; }
|
|
||||||
|
|
||||||
public ulong AllRequests { get; internal set; }
|
|
||||||
}
|
|
|
@ -1,26 +0,0 @@
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
internal sealed class WorkList
|
|
||||||
{
|
|
||||||
private readonly List<int> elements = [];
|
|
||||||
|
|
||||||
internal int Length
|
|
||||||
{
|
|
||||||
get { return elements.Count; }
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Add(int element)
|
|
||||||
{
|
|
||||||
elements.Add(element);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal int Remove()
|
|
||||||
{
|
|
||||||
int last = elements.LastOrDefault();
|
|
||||||
elements.RemoveAt(elements.Count - 1);
|
|
||||||
return last;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,39 +0,0 @@
|
||||||
using System;
|
|
||||||
using System.Collections.ObjectModel;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using Grpc.Core;
|
|
||||||
using Grpc.Core.Interceptors;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
|
||||||
|
|
||||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>")]
|
|
||||||
public struct WrapperPrm
|
|
||||||
{
|
|
||||||
internal ILogger? Logger { get; set; }
|
|
||||||
|
|
||||||
internal string Address { get; set; }
|
|
||||||
|
|
||||||
internal ECDsa Key { get; set; }
|
|
||||||
|
|
||||||
internal ulong DialTimeout { get; set; }
|
|
||||||
|
|
||||||
internal ulong StreamTimeout { get; set; }
|
|
||||||
|
|
||||||
internal uint ErrorThreshold { get; set; }
|
|
||||||
|
|
||||||
internal Action ResponseInfoCallback { get; set; }
|
|
||||||
|
|
||||||
internal Action PoolRequestInfoCallback { get; set; }
|
|
||||||
|
|
||||||
internal Func<string, ChannelBase> GrpcChannelFactory { get; set; }
|
|
||||||
|
|
||||||
internal ulong GracefulCloseOnSwitchTimeout { get; set; }
|
|
||||||
|
|
||||||
internal Action<CallStatistics>? Callback { get; set; }
|
|
||||||
|
|
||||||
internal Collection<Interceptor>? Interceptors { get; set; }
|
|
||||||
}
|
|
||||||
|
|
|
@ -20,8 +20,6 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
|
||||||
{
|
{
|
||||||
var binary = RuleSerializer.Serialize(args.Chain);
|
var binary = RuleSerializer.Serialize(args.Chain);
|
||||||
|
|
||||||
var base64 = Convert.ToBase64String(binary);
|
|
||||||
|
|
||||||
AddChainRequest request = new()
|
AddChainRequest request = new()
|
||||||
{
|
{
|
||||||
Body = new()
|
Body = new()
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Security.Cryptography;
|
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
using FrostFS.Container;
|
using FrostFS.Container;
|
||||||
|
@ -83,7 +82,7 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
|
||||||
Body = new PutRequest.Types.Body
|
Body = new PutRequest.Types.Body
|
||||||
{
|
{
|
||||||
Container = grpcContainer,
|
Container = grpcContainer,
|
||||||
Signature = ClientContext.Key.ECDsaKey.SignRFC6979(grpcContainer)
|
Signature = ClientContext.Key.SignRFC6979(grpcContainer)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -113,8 +112,8 @@ internal sealed class ContainerServiceProvider(ContainerService.ContainerService
|
||||||
{
|
{
|
||||||
Body = new DeleteRequest.Types.Body
|
Body = new DeleteRequest.Types.Body
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToMessage(),
|
ContainerId = args.ContainerId.GetContainerID(),
|
||||||
Signature = ClientContext.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value)
|
Signature = ClientContext.Key.SignRFC6979(args.ContainerId.GetContainerID().Value)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -96,7 +96,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToMessage(),
|
ContainerId = args.ContainerId.GetContainerID(),
|
||||||
ObjectId = args.ObjectId.ToMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -124,7 +124,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToMessage(),
|
ContainerId = args.ContainerId.GetContainerID(),
|
||||||
ObjectId = args.ObjectId.ToMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
},
|
},
|
||||||
Range = new Object.Range
|
Range = new Object.Range
|
||||||
|
@ -159,7 +159,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToMessage(),
|
ContainerId = args.ContainerId.GetContainerID(),
|
||||||
ObjectId = args.ObjectId.ToMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
},
|
},
|
||||||
Type = ChecksumType.Sha256,
|
Type = ChecksumType.Sha256,
|
||||||
|
@ -204,7 +204,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToMessage(),
|
ContainerId = args.ContainerId.GetContainerID(),
|
||||||
ObjectId = args.ObjectId.ToMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -231,7 +231,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
{
|
{
|
||||||
Body = new SearchRequest.Types.Body
|
Body = new SearchRequest.Types.Body
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToMessage(),
|
ContainerId = args.ContainerId.GetContainerID(),
|
||||||
Version = 1 // TODO: clarify this param
|
Version = 1 // TODO: clarify this param
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -295,87 +295,88 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
internal async Task<FrostFsObjectId> PatchObjectAsync(PrmObjectPatch args, CallContext ctx)
|
internal async Task<FrostFsObjectId> PatchObjectAsync(PrmObjectPatch args, CallContext ctx)
|
||||||
{
|
{
|
||||||
var chunkSize = args.MaxChunkLength;
|
var chunkSize = args.MaxChunkLength;
|
||||||
Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null");
|
|
||||||
|
|
||||||
var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken);
|
var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken);
|
||||||
|
|
||||||
byte[]? chunkBuffer = null;
|
var address = new Address
|
||||||
try
|
|
||||||
{
|
{
|
||||||
// common
|
ObjectId = args.Address.ObjectId,
|
||||||
chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
|
ContainerId = args.Address.ContainerId
|
||||||
|
};
|
||||||
|
|
||||||
bool isFirstChunk = true;
|
if (args.Payload != null && args.Payload.Length > 0)
|
||||||
ulong currentPos = args.Range.Offset;
|
{
|
||||||
|
byte[]? chunkBuffer = null;
|
||||||
var address = new Address
|
try
|
||||||
{
|
{
|
||||||
ObjectId = args.Address.ObjectId,
|
chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
|
||||||
ContainerId = args.Address.ContainerId
|
|
||||||
};
|
|
||||||
|
|
||||||
while (true)
|
bool isFirstChunk = true;
|
||||||
{
|
ulong currentPos = args.Range.Offset;
|
||||||
var bytesCount = await payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false);
|
|
||||||
|
|
||||||
if (bytesCount == 0)
|
while (true)
|
||||||
{
|
{
|
||||||
break;
|
var bytesCount = await args.Payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false);
|
||||||
}
|
|
||||||
|
|
||||||
var request = new PatchRequest()
|
if (bytesCount == 0)
|
||||||
{
|
|
||||||
Body = new()
|
|
||||||
{
|
{
|
||||||
Address = address,
|
break;
|
||||||
Patch = new PatchRequest.Types.Body.Types.Patch
|
}
|
||||||
|
|
||||||
|
PatchRequest request;
|
||||||
|
|
||||||
|
if (isFirstChunk)
|
||||||
|
{
|
||||||
|
request = await CreateFirstRequest(args, ctx, address).ConfigureAwait(false);
|
||||||
|
|
||||||
|
request.Body.Patch = new PatchRequest.Types.Body.Types.Patch
|
||||||
{
|
{
|
||||||
Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)),
|
Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)),
|
||||||
SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount }
|
SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount }
|
||||||
}
|
};
|
||||||
|
|
||||||
|
isFirstChunk = false;
|
||||||
}
|
}
|
||||||
};
|
else
|
||||||
|
|
||||||
if (isFirstChunk)
|
|
||||||
{
|
|
||||||
var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false);
|
|
||||||
|
|
||||||
var protoToken = sessionToken.CreateObjectTokenContext(
|
|
||||||
address,
|
|
||||||
ObjectSessionContext.Types.Verb.Patch,
|
|
||||||
ClientContext.Key);
|
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders, protoToken);
|
|
||||||
|
|
||||||
if (args.NewAttributes != null && args.NewAttributes.Length > 0)
|
|
||||||
{
|
{
|
||||||
foreach (var attr in args.NewAttributes)
|
request = new PatchRequest()
|
||||||
{
|
{
|
||||||
request.Body.NewAttributes.Add(attr.ToMessage());
|
Body = new()
|
||||||
request.Body.ReplaceAttributes = args.ReplaceAttributes;
|
{
|
||||||
}
|
Address = address,
|
||||||
|
Patch = new PatchRequest.Types.Body.Types.Patch
|
||||||
|
{
|
||||||
|
Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)),
|
||||||
|
SourceRange = new Range { Offset = currentPos, Length = (ulong)bytesCount }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
request.AddMetaHeader(args.XHeaders);
|
||||||
}
|
}
|
||||||
|
|
||||||
isFirstChunk = false;
|
request.Sign(ClientContext.Key);
|
||||||
|
|
||||||
|
await call.RequestStream.WriteAsync(request).ConfigureAwait(false);
|
||||||
|
|
||||||
|
currentPos += (ulong)bytesCount;
|
||||||
}
|
}
|
||||||
else
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (chunkBuffer != null)
|
||||||
{
|
{
|
||||||
request.AddMetaHeader(args.XHeaders);
|
ArrayPool<byte>.Shared.Return(chunkBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
request.Sign(ClientContext.Key);
|
|
||||||
|
|
||||||
await call.RequestStream.WriteAsync(request).ConfigureAwait(false);
|
|
||||||
|
|
||||||
currentPos += (ulong)bytesCount;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
else if (args.NewAttributes != null && args.NewAttributes.Length > 0)
|
||||||
{
|
{
|
||||||
if (chunkBuffer != null)
|
PatchRequest request = await CreateFirstRequest(args, ctx, address).ConfigureAwait(false);
|
||||||
{
|
|
||||||
ArrayPool<byte>.Shared.Return(chunkBuffer);
|
request.Sign(ClientContext.Key);
|
||||||
}
|
|
||||||
|
await call.RequestStream.WriteAsync(request).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
await call.RequestStream.CompleteAsync().ConfigureAwait(false);
|
await call.RequestStream.CompleteAsync().ConfigureAwait(false);
|
||||||
|
@ -384,95 +385,289 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
Verifier.CheckResponse(response);
|
Verifier.CheckResponse(response);
|
||||||
|
|
||||||
return response.Body.ObjectId.ToModel();
|
return response.Body.ObjectId.ToModel();
|
||||||
|
|
||||||
|
async Task<PatchRequest> CreateFirstRequest(PrmObjectPatch args, CallContext ctx, Address address)
|
||||||
|
{
|
||||||
|
var body = new PatchRequest.Types.Body() { Address = address };
|
||||||
|
|
||||||
|
if (args.NewAttributes != null)
|
||||||
|
{
|
||||||
|
body.ReplaceAttributes = args.ReplaceAttributes;
|
||||||
|
|
||||||
|
foreach (var attr in args.NewAttributes!)
|
||||||
|
{
|
||||||
|
body.NewAttributes.Add(attr.ToMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var request = new PatchRequest() { Body = body };
|
||||||
|
|
||||||
|
var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false);
|
||||||
|
|
||||||
|
var protoToken = sessionToken.CreateObjectTokenContext(
|
||||||
|
address,
|
||||||
|
ObjectSessionContext.Types.Verb.Patch,
|
||||||
|
ClientContext.Key);
|
||||||
|
|
||||||
|
request.AddMetaHeader(args.XHeaders, protoToken);
|
||||||
|
return request;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal async Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
|
internal async Task<FrostFsObjectId> PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
|
||||||
{
|
{
|
||||||
var stream = args.Payload!;
|
if (args.Payload == null)
|
||||||
var header = args.Header!;
|
throw new ArgumentException(nameof(args.Payload));
|
||||||
|
|
||||||
if (header.PayloadLength > 0)
|
if (args.Header == null)
|
||||||
args.PutObjectContext.FullLength = header.PayloadLength;
|
throw new ArgumentException(nameof(args.Header));
|
||||||
else if (stream.CanSeek)
|
|
||||||
args.PutObjectContext.FullLength = (ulong)stream.Length;
|
|
||||||
else
|
|
||||||
throw new ArgumentException("The stream does not have a length and payload length is not defined");
|
|
||||||
|
|
||||||
if (args.PutObjectContext.FullLength == 0)
|
|
||||||
throw new ArgumentException("The stream has zero length");
|
|
||||||
|
|
||||||
var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
|
var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
|
||||||
var partSize = (int)networkSettings.MaxObjectSize;
|
int partSize = (int)networkSettings.MaxObjectSize;
|
||||||
|
|
||||||
var restBytes = args.PutObjectContext.FullLength;
|
int chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
|
||||||
|
|
||||||
var objectSize = (int)Math.Min((ulong)partSize, restBytes);
|
ulong fullLength;
|
||||||
|
|
||||||
// define collection capacity
|
// Information about the uploaded parts.
|
||||||
var objectsCount = (int)(restBytes / (ulong)objectSize) + ((restBytes % (ulong)objectSize) > 0 ? 1 : 0);
|
var progressInfo = args.Progress; //
|
||||||
|
var offset = 0L;
|
||||||
|
|
||||||
// if the object fits one part, it can be loaded as non-complex object
|
if (progressInfo != null && progressInfo.GetParts().Count > 0)
|
||||||
if (objectsCount == 1)
|
{
|
||||||
|
if (!args.Payload.CanSeek)
|
||||||
|
{
|
||||||
|
throw new FrostFsException("Cannot resume client cut upload for this stream. Seek must be supported.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastPart = progressInfo.GetLast();
|
||||||
|
args.Payload.Position = lastPart.Offset + lastPart.Length;
|
||||||
|
fullLength = (ulong)(args.Payload.Length - args.Payload.Position);
|
||||||
|
offset = args.Payload.Position;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (args.Header.PayloadLength > 0)
|
||||||
|
fullLength = args.Header.PayloadLength;
|
||||||
|
else if (args.Payload.CanSeek)
|
||||||
|
fullLength = (ulong)args.Payload.Length;
|
||||||
|
else
|
||||||
|
throw new ArgumentException("The stream does not have a length and payload length is not defined");
|
||||||
|
}
|
||||||
|
|
||||||
|
//define collection capacity
|
||||||
|
var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0;
|
||||||
|
var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0;
|
||||||
|
|
||||||
|
progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount);
|
||||||
|
|
||||||
|
var remain = fullLength;
|
||||||
|
|
||||||
|
byte[]? buffer = null;
|
||||||
|
bool isRentBuffer = false;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (args.CustomBuffer != null)
|
||||||
|
{
|
||||||
|
if (args.CustomBuffer.Length < chunkSize)
|
||||||
|
throw new ArgumentException($"Buffer size is too small. At least {chunkSize} required");
|
||||||
|
|
||||||
|
buffer = args.CustomBuffer;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
buffer = ArrayPool<byte>.Shared.Rent(chunkSize);
|
||||||
|
isRentBuffer = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FrostFsObjectId? resultObjectId = null;
|
||||||
|
FrostFsObjectHeader? parentHeader = null;
|
||||||
|
|
||||||
|
while (remain > 0)
|
||||||
|
{
|
||||||
|
var bytesToWrite = Math.Min((ulong)partSize, remain);
|
||||||
|
var isLastPart = remain <= (ulong)partSize;
|
||||||
|
|
||||||
|
// When the last part of the object is uploaded, all metadata for the object must be added
|
||||||
|
if (isLastPart && objectsCount > 1)
|
||||||
|
{
|
||||||
|
parentHeader = new FrostFsObjectHeader(args.Header.ContainerId, FrostFsObjectType.Regular)
|
||||||
|
{
|
||||||
|
Attributes = args.Header.Attributes,
|
||||||
|
PayloadLength = fullLength
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter
|
||||||
|
var header = objectsCount == 1 ? args.Header : new FrostFsObjectHeader(
|
||||||
|
args.Header.ContainerId,
|
||||||
|
FrostFsObjectType.Regular,
|
||||||
|
[],
|
||||||
|
new FrostFsSplit(progressInfo.SplitId, progressInfo.GetLast().ObjectId, parentHeader: parentHeader));
|
||||||
|
|
||||||
|
var prm = new PrmObjectPut(header);
|
||||||
|
using var stream = await PutStreamObjectAsync(prm, ctx).ConfigureAwait(false);
|
||||||
|
var uploaded = 0;
|
||||||
|
|
||||||
|
// If an error occurs while uploading a part of the object, there is no need to re-upload the parts
|
||||||
|
// that were successfully uploaded before. It is sufficient to re-upload only the failed part
|
||||||
|
|
||||||
|
var thisPartRest = (int)Math.Min((ulong)partSize, remain);
|
||||||
|
while (thisPartRest > 0)
|
||||||
|
{
|
||||||
|
var nextChunkSize = Math.Min(thisPartRest, chunkSize);
|
||||||
|
var size = await args.Payload.ReadAsync(buffer, 0, nextChunkSize).ConfigureAwait(false);
|
||||||
|
|
||||||
|
if (size == 0)
|
||||||
|
break;
|
||||||
|
|
||||||
|
await stream.WriteAsync(buffer.AsMemory(0, size)).ConfigureAwait(false);
|
||||||
|
uploaded += size;
|
||||||
|
thisPartRest -= size;
|
||||||
|
}
|
||||||
|
|
||||||
|
var objectId = await stream.CompleteAsync().ConfigureAwait(false);
|
||||||
|
var part = new ObjectPartInfo(offset, uploaded, objectId);
|
||||||
|
offset += uploaded;
|
||||||
|
progressInfo.AddPart(part);
|
||||||
|
|
||||||
|
remain -= bytesToWrite;
|
||||||
|
|
||||||
|
if (isLastPart)
|
||||||
|
{
|
||||||
|
if (objectsCount == 1)
|
||||||
|
{
|
||||||
|
return progressInfo.GetPart(0).ObjectId;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parentHeader == null) continue;
|
||||||
|
|
||||||
|
// Once all parts of the object are uploaded, they must be linked into a single entity
|
||||||
|
var linkObject = new FrostFsLinkObject(header.ContainerId, progressInfo.SplitId, parentHeader,
|
||||||
|
[.. progressInfo.GetParts().Select(p => p.ObjectId)]);
|
||||||
|
|
||||||
|
await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false);
|
||||||
|
|
||||||
|
// Retrieve the ID of the linked object
|
||||||
|
resultObjectId = FrostFsObjectId.FromHash(prm.Header!.GetHeader().Split!.Parent.Value.Span);
|
||||||
|
return resultObjectId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new FrostFsException("Unexpected error: cannot send object");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (isRentBuffer && buffer != null)
|
||||||
|
{
|
||||||
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal async Task<FrostFsObjectId> PutClientCutSingleObjectAsync(PrmObjectClientCutPut args, CallContext ctx)
|
||||||
|
{
|
||||||
|
if (args.Payload == null)
|
||||||
|
throw new ArgumentException(nameof(args.Payload));
|
||||||
|
|
||||||
|
if (args.Header == null)
|
||||||
|
throw new ArgumentException(nameof(args.Header));
|
||||||
|
|
||||||
|
var networkSettings = await ClientContext.Client.GetNetworkSettingsAsync(ctx).ConfigureAwait(false);
|
||||||
|
int partSize = (int)networkSettings.MaxObjectSize;
|
||||||
|
|
||||||
|
int chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
|
||||||
|
|
||||||
|
ulong fullLength;
|
||||||
|
|
||||||
|
// Information about the uploaded parts.
|
||||||
|
var progressInfo = args.Progress; //
|
||||||
|
var offset = 0L;
|
||||||
|
|
||||||
|
if (progressInfo != null && progressInfo.GetParts().Count > 0)
|
||||||
|
{
|
||||||
|
if (!args.Payload.CanSeek)
|
||||||
|
{
|
||||||
|
throw new FrostFsException("Cannot resume client cut upload for this stream. Seek must be supported.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastPart = progressInfo.GetLast();
|
||||||
|
args.Payload.Position = lastPart.Offset + lastPart.Length;
|
||||||
|
fullLength = (ulong)(args.Payload.Length - args.Payload.Position);
|
||||||
|
offset = args.Payload.Position;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (args.Header.PayloadLength > 0)
|
||||||
|
fullLength = args.Header.PayloadLength;
|
||||||
|
else if (args.Payload.CanSeek)
|
||||||
|
fullLength = (ulong)args.Payload.Length;
|
||||||
|
else
|
||||||
|
throw new ArgumentException("The stream does not have a length and payload length is not defined");
|
||||||
|
}
|
||||||
|
|
||||||
|
//define collection capacity
|
||||||
|
var restPart = (fullLength % (ulong)partSize) > 0 ? 1 : 0;
|
||||||
|
var objectsCount = fullLength > 0 ? (int)(fullLength / (ulong)partSize) + restPart : 0;
|
||||||
|
|
||||||
|
progressInfo ??= new UploadProgressInfo(Guid.NewGuid(), objectsCount);
|
||||||
|
|
||||||
|
// if the object fits one part, it can be loaded as non-complex object, but if it is not upload resuming
|
||||||
|
if (objectsCount == 1 && progressInfo.GetLast().Length == 0)
|
||||||
{
|
{
|
||||||
args.PutObjectContext.MaxObjectSizeCache = partSize;
|
args.PutObjectContext.MaxObjectSizeCache = partSize;
|
||||||
|
args.PutObjectContext.FullLength = fullLength;
|
||||||
var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false);
|
var singlePartResult = await PutMultipartStreamObjectAsync(args, default).ConfigureAwait(false);
|
||||||
return singlePartResult.ObjectId;
|
return singlePartResult.ObjectId;
|
||||||
}
|
}
|
||||||
|
|
||||||
List<FrostFsObjectId> parts = new(objectsCount);
|
var remain = fullLength;
|
||||||
|
|
||||||
SplitId splitId = new();
|
|
||||||
|
|
||||||
// keep attributes for the large object
|
|
||||||
var attributes = args.Header!.Attributes.ToArray();
|
|
||||||
header.Attributes = null;
|
|
||||||
|
|
||||||
var remain = args.PutObjectContext.FullLength;
|
|
||||||
|
|
||||||
FrostFsObjectHeader? parentHeader = null;
|
|
||||||
|
|
||||||
var lastIndex = objectsCount - 1;
|
|
||||||
|
|
||||||
bool rentBuffer = false;
|
|
||||||
byte[]? buffer = null;
|
byte[]? buffer = null;
|
||||||
|
bool isRentBuffer = false;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
for (int i = 0; i < objectsCount; i++)
|
if (args.CustomBuffer != null)
|
||||||
{
|
{
|
||||||
if (args.CustomBuffer != null)
|
if (args.CustomBuffer.Length < partSize)
|
||||||
{
|
{
|
||||||
if (args.CustomBuffer.Length < partSize)
|
throw new ArgumentException($"Buffer size is too small. A buffer with capacity {partSize} is required");
|
||||||
{
|
|
||||||
throw new ArgumentException($"Buffer size is too small. A buffer with capacity {partSize} is required");
|
|
||||||
}
|
|
||||||
|
|
||||||
buffer = args.CustomBuffer;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
buffer = ArrayPool<byte>.Shared.Rent(partSize);
|
|
||||||
rentBuffer = true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
buffer = args.CustomBuffer;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
buffer = ArrayPool<byte>.Shared.Rent(partSize);
|
||||||
|
isRentBuffer = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FrostFsObjectHeader? parentHeader = null;
|
||||||
|
|
||||||
|
for (int i = 0; i < objectsCount;)
|
||||||
|
{
|
||||||
|
i++;
|
||||||
var bytesToWrite = Math.Min((ulong)partSize, remain);
|
var bytesToWrite = Math.Min((ulong)partSize, remain);
|
||||||
|
|
||||||
var size = await stream.ReadAsync(buffer, 0, (int)bytesToWrite).ConfigureAwait(false);
|
var size = await args.Payload.ReadAsync(buffer, 0, (int)bytesToWrite).ConfigureAwait(false);
|
||||||
|
|
||||||
if (i == lastIndex)
|
if (i == objectsCount)
|
||||||
{
|
{
|
||||||
parentHeader = new FrostFsObjectHeader(header.ContainerId, FrostFsObjectType.Regular, attributes)
|
parentHeader = new FrostFsObjectHeader(args.Header.ContainerId, FrostFsObjectType.Regular)
|
||||||
{
|
{
|
||||||
PayloadLength = args.PutObjectContext.FullLength
|
PayloadLength = args.PutObjectContext.FullLength,
|
||||||
|
Attributes = args.Header.Attributes
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter
|
// Uploading the next part of the object. Note: the request must contain a non-null SplitId parameter
|
||||||
var partHeader = new FrostFsObjectHeader(
|
var partHeader = new FrostFsObjectHeader(
|
||||||
header.ContainerId,
|
args.Header.ContainerId,
|
||||||
FrostFsObjectType.Regular,
|
FrostFsObjectType.Regular,
|
||||||
[],
|
[],
|
||||||
new FrostFsSplit(splitId, parts.LastOrDefault(),
|
new FrostFsSplit(progressInfo.SplitId, progressInfo.GetLast().ObjectId,
|
||||||
parentHeader: parentHeader))
|
parentHeader: parentHeader))
|
||||||
{
|
{
|
||||||
PayloadLength = (ulong)size
|
PayloadLength = (ulong)size
|
||||||
|
@ -485,15 +680,20 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
|
|
||||||
var prm = new PrmSingleObjectPut(obj);
|
var prm = new PrmSingleObjectPut(obj);
|
||||||
|
|
||||||
var objId = await PutSingleObjectAsync(prm, ctx).ConfigureAwait(false);
|
var objectId = await PutSingleObjectAsync(prm, ctx).ConfigureAwait(false);
|
||||||
|
|
||||||
parts.Add(objId);
|
var part = new ObjectPartInfo(offset, size, objectId);
|
||||||
|
progressInfo.AddPart(part);
|
||||||
|
|
||||||
if (i < lastIndex)
|
offset += size;
|
||||||
|
|
||||||
|
if (i < objectsCount)
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// Once all parts of the object are uploaded, they must be linked into a single entity
|
// Once all parts of the object are uploaded, they must be linked into a single entity
|
||||||
var linkObject = new FrostFsLinkObject(header.ContainerId, splitId, parentHeader!, parts);
|
var linkObject = new FrostFsLinkObject(args.Header.ContainerId, progressInfo.SplitId, parentHeader!, [.. progressInfo.GetParts().Select(p => p.ObjectId)]);
|
||||||
|
|
||||||
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false);
|
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject), ctx).ConfigureAwait(false);
|
||||||
|
|
||||||
|
@ -505,7 +705,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
if (rentBuffer && buffer != null)
|
if (isRentBuffer && buffer != null)
|
||||||
{
|
{
|
||||||
ArrayPool<byte>.Shared.Return(buffer);
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
using System;
|
using System.Collections.Concurrent;
|
||||||
using System.Collections.Concurrent;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
|
@ -36,15 +35,4 @@ internal sealed class SessionCache(ulong sessionExpirationDuration)
|
||||||
_cache[key] = value;
|
_cache[key] = value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal void DeleteByPrefix(string prefix)
|
|
||||||
{
|
|
||||||
foreach (var key in _cache.Keys)
|
|
||||||
{
|
|
||||||
if (key.StartsWith(prefix, StringComparison.Ordinal))
|
|
||||||
{
|
|
||||||
_cache.TryRemove(key, out var _);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
|
@ -39,7 +39,7 @@ public class ClientContext(FrostFSClient client, ClientKey key, FrostFsOwner own
|
||||||
{
|
{
|
||||||
if (sessionKey == null && Key != null && Address != null)
|
if (sessionKey == null && Key != null && Address != null)
|
||||||
{
|
{
|
||||||
sessionKey = Pool.FormCacheKey(Address, Key.PublicKey);
|
sessionKey = $"{Address}{Key}";
|
||||||
}
|
}
|
||||||
|
|
||||||
return sessionKey;
|
return sessionKey;
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using System.Security.Cryptography;
|
||||||
using FrostFS.Object;
|
using FrostFS.Object;
|
||||||
using FrostFS.Refs;
|
using FrostFS.Refs;
|
||||||
using FrostFS.SDK.Client.Mappers.GRPC;
|
using FrostFS.SDK.Client.Mappers.GRPC;
|
||||||
|
@ -44,7 +44,11 @@ public static class ObjectTools
|
||||||
if (header.Split != null)
|
if (header.Split != null)
|
||||||
SetSplitValues(grpcHeader, header.Split, owner, version, key);
|
SetSplitValues(grpcHeader, header.Split, owner, version, key);
|
||||||
|
|
||||||
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
|
using var sha256 = SHA256.Create();
|
||||||
|
using HashStream stream = new(sha256);
|
||||||
|
grpcHeader.WriteTo(stream);
|
||||||
|
|
||||||
|
return new FrostFsObjectId(Base58.Encode(stream.Hash()));
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx)
|
internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx)
|
||||||
|
@ -75,7 +79,7 @@ public static class ObjectTools
|
||||||
var obj = new Object.Object
|
var obj = new Object.Object
|
||||||
{
|
{
|
||||||
Header = grpcHeader,
|
Header = grpcHeader,
|
||||||
ObjectId = new ObjectID { Value = grpcHeader.Sha256() },
|
ObjectId = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcHeader.Sha256()) },
|
||||||
Payload = UnsafeByteOperations.UnsafeWrap(@object.SingleObjectPayload)
|
Payload = UnsafeByteOperations.UnsafeWrap(@object.SingleObjectPayload)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -117,9 +121,9 @@ public static class ObjectTools
|
||||||
|
|
||||||
if (split.ParentHeader is not null)
|
if (split.ParentHeader is not null)
|
||||||
{
|
{
|
||||||
var grpcParentHeader = CreateHeader(split.ParentHeader, Array.Empty<byte>().Sha256(), owner, version);
|
var grpcParentHeader = CreateHeader(split.ParentHeader, DataHasher.Sha256([]), owner, version);
|
||||||
|
|
||||||
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
|
grpcHeader.Split.Parent = new ObjectID { Value = UnsafeByteOperations.UnsafeWrap(grpcParentHeader.Sha256()) };
|
||||||
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
||||||
grpcHeader.Split.ParentSignature = new Signature
|
grpcHeader.Split.ParentSignature = new Signature
|
||||||
{
|
{
|
||||||
|
@ -147,21 +151,12 @@ public static class ObjectTools
|
||||||
return grpcHeader;
|
return grpcHeader;
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static Checksum Sha256Checksum(byte[] data)
|
|
||||||
{
|
|
||||||
return new Checksum
|
|
||||||
{
|
|
||||||
Type = ChecksumType.Sha256,
|
|
||||||
Sum = ByteString.CopyFrom(data.Sha256())
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static Checksum Sha256Checksum(ReadOnlyMemory<byte> data)
|
internal static Checksum Sha256Checksum(ReadOnlyMemory<byte> data)
|
||||||
{
|
{
|
||||||
return new Checksum
|
return new Checksum
|
||||||
{
|
{
|
||||||
Type = ChecksumType.Sha256,
|
Type = ChecksumType.Sha256,
|
||||||
Sum = ByteString.CopyFrom(data.Sha256())
|
Sum = UnsafeByteOperations.UnsafeWrap(DataHasher.Sha256(data))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -33,6 +33,7 @@ public static class RequestSigner
|
||||||
var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N);
|
var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N);
|
||||||
var privateKey = new ECPrivateKeyParameters(new BigInteger(1, key.PrivateKey()), ecParameters);
|
var privateKey = new ECPrivateKeyParameters(new BigInteger(1, key.PrivateKey()), ecParameters);
|
||||||
var signer = new ECDsaSigner(new HMacDsaKCalculator(digest));
|
var signer = new ECDsaSigner(new HMacDsaKCalculator(digest));
|
||||||
|
|
||||||
var hash = new byte[digest.GetDigestSize()];
|
var hash = new byte[digest.GetDigestSize()];
|
||||||
|
|
||||||
digest.BlockUpdate(data, 0, data.Length);
|
digest.BlockUpdate(data, 0, data.Length);
|
||||||
|
@ -54,21 +55,21 @@ public static class RequestSigner
|
||||||
return ByteString.CopyFrom(signature);
|
return ByteString.CopyFrom(signature);
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static SignatureRFC6979 SignRFC6979(this ECDsa key, IMessage message)
|
internal static SignatureRFC6979 SignRFC6979(this ClientKey key, IMessage message)
|
||||||
{
|
{
|
||||||
return new SignatureRFC6979
|
return new SignatureRFC6979
|
||||||
{
|
{
|
||||||
Key = ByteString.CopyFrom(key.PublicKey()),
|
Key = key.PublicKeyProto,
|
||||||
Sign = key.SignRFC6979(message.ToByteArray()),
|
Sign = key.ECDsaKey.SignRFC6979(message.ToByteArray()),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static SignatureRFC6979 SignRFC6979(this ECDsa key, ByteString data)
|
internal static SignatureRFC6979 SignRFC6979(this ClientKey key, ByteString data)
|
||||||
{
|
{
|
||||||
return new SignatureRFC6979
|
return new SignatureRFC6979
|
||||||
{
|
{
|
||||||
Key = ByteString.CopyFrom(key.PublicKey()),
|
Key = key.PublicKeyProto,
|
||||||
Sign = key.SignRFC6979(data.ToByteArray()),
|
Sign = key.ECDsaKey.SignRFC6979(data.ToByteArray()),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,7 +83,7 @@ public static class RequestSigner
|
||||||
Span<byte> result = stackalloc byte[65];
|
Span<byte> result = stackalloc byte[65];
|
||||||
result[0] = 0x04;
|
result[0] = 0x04;
|
||||||
|
|
||||||
key.SignHash(data.Sha512()).AsSpan().CopyTo(result.Slice(1));
|
key.SignHash(DataHasher.Sha512(data)).AsSpan().CopyTo(result.Slice(1));
|
||||||
|
|
||||||
return ByteString.CopyFrom(result);
|
return ByteString.CopyFrom(result);
|
||||||
}
|
}
|
||||||
|
@ -113,7 +114,8 @@ public static class RequestSigner
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
using HashStream stream = new();
|
using var sha512 = SHA512.Create();
|
||||||
|
using HashStream stream = new(sha512);
|
||||||
data.WriteTo(stream);
|
data.WriteTo(stream);
|
||||||
|
|
||||||
var sig = new Signature
|
var sig = new Signature
|
||||||
|
|
|
@ -9,61 +9,13 @@ using FrostFS.Session;
|
||||||
|
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
|
||||||
using Org.BouncyCastle.Asn1.Sec;
|
|
||||||
using Org.BouncyCastle.Crypto.Digests;
|
|
||||||
using Org.BouncyCastle.Crypto.Parameters;
|
|
||||||
using Org.BouncyCastle.Crypto.Signers;
|
|
||||||
using Org.BouncyCastle.Math;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Client;
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
public static class Verifier
|
public static class Verifier
|
||||||
{
|
{
|
||||||
public const int RFC6979SignatureSize = 64;
|
public const int RFC6979SignatureSize = 64;
|
||||||
|
|
||||||
private static BigInteger[] DecodeSignature(byte[] sig)
|
public static bool VerifyData(this ECDsa key, IMessage data, ByteString sig)
|
||||||
{
|
|
||||||
if (sig.Length != RFC6979SignatureSize)
|
|
||||||
throw new FormatException($"Wrong signature size, expect={RFC6979SignatureSize}, actual={sig.Length}");
|
|
||||||
|
|
||||||
var rs = new BigInteger[2];
|
|
||||||
rs[0] = new BigInteger(1, sig.AsSpan(0, 32).ToArray());
|
|
||||||
rs[1] = new BigInteger(1, sig.AsSpan(32).ToArray());
|
|
||||||
|
|
||||||
return rs;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool VerifyRFC6979(this byte[] publicKey, byte[] data, byte[] sig)
|
|
||||||
{
|
|
||||||
if (publicKey is null || data is null || sig is null)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
var rs = DecodeSignature(sig);
|
|
||||||
var digest = new Sha256Digest();
|
|
||||||
var signer = new ECDsaSigner(new HMacDsaKCalculator(digest));
|
|
||||||
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
|
|
||||||
var ecParameters = new ECDomainParameters(secp256R1.Curve, secp256R1.G, secp256R1.N);
|
|
||||||
var bcPublicKey = new ECPublicKeyParameters(secp256R1.Curve.DecodePoint(publicKey), ecParameters);
|
|
||||||
var hash = new byte[digest.GetDigestSize()];
|
|
||||||
|
|
||||||
digest.BlockUpdate(data, 0, data.Length);
|
|
||||||
digest.DoFinal(hash, 0);
|
|
||||||
signer.Init(false, bcPublicKey);
|
|
||||||
|
|
||||||
return signer.VerifySignature(hash, rs[0], rs[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool VerifyRFC6979(this SignatureRFC6979 signature, IMessage message)
|
|
||||||
{
|
|
||||||
if (signature is null)
|
|
||||||
{
|
|
||||||
throw new ArgumentNullException(nameof(signature));
|
|
||||||
}
|
|
||||||
|
|
||||||
return signature.Key.ToByteArray().VerifyRFC6979(message.ToByteArray(), signature.Sign.ToByteArray());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool VerifyData(this ECDsa key, ReadOnlyMemory<byte> data, byte[] sig)
|
|
||||||
{
|
{
|
||||||
if (key is null)
|
if (key is null)
|
||||||
throw new ArgumentNullException(nameof(key));
|
throw new ArgumentNullException(nameof(key));
|
||||||
|
@ -71,7 +23,18 @@ public static class Verifier
|
||||||
if (sig is null)
|
if (sig is null)
|
||||||
throw new ArgumentNullException(nameof(sig));
|
throw new ArgumentNullException(nameof(sig));
|
||||||
|
|
||||||
return key.VerifyHash(data.Sha512(), sig.AsSpan(1).ToArray());
|
var signature = sig.Span.Slice(1).ToArray();
|
||||||
|
using var sha = SHA512.Create();
|
||||||
|
|
||||||
|
if (data is null)
|
||||||
|
{
|
||||||
|
return key.VerifyHash(DataHasher.Sha512(new Span<byte>([])), signature);
|
||||||
|
}
|
||||||
|
|
||||||
|
using var stream = new HashStream(sha);
|
||||||
|
data.WriteTo(stream);
|
||||||
|
|
||||||
|
return key.VerifyHash(stream.Hash(), signature);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static bool VerifyMessagePart(this Signature sig, IMessage data)
|
public static bool VerifyMessagePart(this Signature sig, IMessage data)
|
||||||
|
@ -80,9 +43,8 @@ public static class Verifier
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
using var key = sig.Key.ToByteArray().LoadPublicKey();
|
using var key = sig.Key.ToByteArray().LoadPublicKey();
|
||||||
var data2Verify = data is null ? [] : data.ToByteArray();
|
|
||||||
|
|
||||||
return key.VerifyData(data2Verify, sig.Sign.ToByteArray());
|
return key.VerifyData(data, sig.Sign);
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static bool VerifyMatryoskaLevel(IMessage body, IMetaHeader meta, IVerificationHeader verification)
|
internal static bool VerifyMatryoskaLevel(IMessage body, IMetaHeader meta, IVerificationHeader verification)
|
||||||
|
|
31
src/FrostFS.SDK.Client/Tools/WalletTools.cs
Normal file
31
src/FrostFS.SDK.Client/Tools/WalletTools.cs
Normal file
|
@ -0,0 +1,31 @@
|
||||||
|
using System.Text.Json;
|
||||||
|
using System;
|
||||||
|
using FrostFS.SDK.Client.Wallets;
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client;
|
||||||
|
|
||||||
|
public static class WalletTools
|
||||||
|
{
|
||||||
|
public static string GetWifFromWallet(string walletJsonText, byte[] password, int accountIndex = 0)
|
||||||
|
{
|
||||||
|
var wallet = JsonSerializer.Deserialize<Wallet>(walletJsonText) ?? throw new ArgumentException("Wrong wallet format");
|
||||||
|
|
||||||
|
if (wallet.Accounts == null || wallet.Accounts.Length < accountIndex + 1)
|
||||||
|
{
|
||||||
|
throw new ArgumentException("Wrong wallet content");
|
||||||
|
}
|
||||||
|
|
||||||
|
var encryptedKey = wallet.Accounts[accountIndex].Key;
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(encryptedKey))
|
||||||
|
{
|
||||||
|
throw new ArgumentException("Cannot get encrypted WIF");
|
||||||
|
}
|
||||||
|
|
||||||
|
var privateKey = CryptoWallet.GetKeyFromEncodedWif(password, encryptedKey!);
|
||||||
|
var wif = privateKey.GetWIFFromPrivateKey();
|
||||||
|
|
||||||
|
return wif;
|
||||||
|
}
|
||||||
|
}
|
24
src/FrostFS.SDK.Client/Wallets/Account.cs
Normal file
24
src/FrostFS.SDK.Client/Wallets/Account.cs
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Wallets;
|
||||||
|
|
||||||
|
public class Account
|
||||||
|
{
|
||||||
|
[JsonPropertyName("address")]
|
||||||
|
public string? Address { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("key")]
|
||||||
|
public string? Key { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("label")]
|
||||||
|
public string? Label { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("contract")]
|
||||||
|
public Contract? Contract { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("lock")]
|
||||||
|
public bool Lock { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("isDefault")]
|
||||||
|
public bool IsDefault { get; set; }
|
||||||
|
}
|
15
src/FrostFS.SDK.Client/Wallets/Contract.cs
Normal file
15
src/FrostFS.SDK.Client/Wallets/Contract.cs
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Wallets;
|
||||||
|
|
||||||
|
public class Contract
|
||||||
|
{
|
||||||
|
[JsonPropertyName("script")]
|
||||||
|
public string? Script { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("parameters")]
|
||||||
|
public Parameter[]? Parameters { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("deployed")]
|
||||||
|
public bool Deployed { get; set; }
|
||||||
|
}
|
6
src/FrostFS.SDK.Client/Wallets/Extra.cs
Normal file
6
src/FrostFS.SDK.Client/Wallets/Extra.cs
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
namespace FrostFS.SDK.Client.Wallets;
|
||||||
|
|
||||||
|
public class Extra
|
||||||
|
{
|
||||||
|
public string? Tokens { get; set; }
|
||||||
|
}
|
12
src/FrostFS.SDK.Client/Wallets/Parameter.cs
Normal file
12
src/FrostFS.SDK.Client/Wallets/Parameter.cs
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Wallets;
|
||||||
|
|
||||||
|
public class Parameter
|
||||||
|
{
|
||||||
|
[JsonPropertyName("name")]
|
||||||
|
public string? Name { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("type")]
|
||||||
|
public string? Type { get; set; }
|
||||||
|
}
|
15
src/FrostFS.SDK.Client/Wallets/ScryptValue.cs
Normal file
15
src/FrostFS.SDK.Client/Wallets/ScryptValue.cs
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Wallets;
|
||||||
|
|
||||||
|
public class ScryptValue
|
||||||
|
{
|
||||||
|
[JsonPropertyName("n")]
|
||||||
|
public int N { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("r")]
|
||||||
|
public int R { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("p")]
|
||||||
|
public int P { get; set; }
|
||||||
|
}
|
18
src/FrostFS.SDK.Client/Wallets/Wallet.cs
Normal file
18
src/FrostFS.SDK.Client/Wallets/Wallet.cs
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Wallets;
|
||||||
|
|
||||||
|
public class Wallet
|
||||||
|
{
|
||||||
|
[JsonPropertyName("version")]
|
||||||
|
public string? Version { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("accounts")]
|
||||||
|
public Account[]? Accounts { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("scrypt")]
|
||||||
|
public ScryptValue? Scrypt { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("extra")]
|
||||||
|
public Extra? Extra { get; set; }
|
||||||
|
}
|
|
@ -21,4 +21,18 @@ internal static class ArrayHelper
|
||||||
|
|
||||||
return dst;
|
return dst;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
|
public static void GetRevertedArray(ReadOnlySpan<byte> source, byte[] data)
|
||||||
|
{
|
||||||
|
if (source.Length != 0)
|
||||||
|
{
|
||||||
|
int i = 0;
|
||||||
|
int j = source.Length - 1;
|
||||||
|
while (i < source.Length)
|
||||||
|
{
|
||||||
|
data[i++] = source[j--];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -1,8 +1,7 @@
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
|
||||||
[assembly: AssemblyCompany("FrostFS.SDK.Cryptography")]
|
[assembly: AssemblyCompany("TrueCloudLab")]
|
||||||
[assembly: AssemblyFileVersion("1.0.2.0")]
|
[assembly: AssemblyFileVersion("1.0.7.0")]
|
||||||
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
|
|
||||||
[assembly: AssemblyProduct("FrostFS.SDK.Cryptography")]
|
[assembly: AssemblyProduct("FrostFS.SDK.Cryptography")]
|
||||||
[assembly: AssemblyTitle("FrostFS.SDK.Cryptography")]
|
[assembly: AssemblyTitle("FrostFS.SDK.Cryptography")]
|
||||||
[assembly: AssemblyVersion("1.0.3.0")]
|
[assembly: AssemblyVersion("1.0.7.0")]
|
||||||
|
|
|
@ -19,19 +19,20 @@ public static class Base58
|
||||||
if (buffer.Length < 4)
|
if (buffer.Length < 4)
|
||||||
throw new FormatException();
|
throw new FormatException();
|
||||||
|
|
||||||
var check = buffer.AsSpan(0, buffer.Length - 4).ToArray();
|
var check = buffer.AsSpan(0, buffer.Length - 4);
|
||||||
byte[] checksum = check.Sha256().Sha256();
|
byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(check).AsSpan());
|
||||||
|
|
||||||
if (!buffer.AsSpan(buffer.Length - 4).SequenceEqual(checksum.AsSpan(0, 4)))
|
if (!buffer.AsSpan(buffer.Length - 4).SequenceEqual(checksum.AsSpan(0, 4)))
|
||||||
throw new FormatException();
|
throw new FormatException();
|
||||||
|
|
||||||
|
var result = check.ToArray();
|
||||||
Array.Clear(buffer, 0, buffer.Length);
|
Array.Clear(buffer, 0, buffer.Length);
|
||||||
return check;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string Base58CheckEncode(this ReadOnlySpan<byte> data)
|
public static string Base58CheckEncode(this Span<byte> data)
|
||||||
{
|
{
|
||||||
byte[] checksum = data.ToArray().Sha256().Sha256();
|
byte[] checksum = DataHasher.Sha256(DataHasher.Sha256(data).AsSpan());
|
||||||
Span<byte> buffer = stackalloc byte[data.Length + 4];
|
Span<byte> buffer = stackalloc byte[data.Length + 4];
|
||||||
data.CopyTo(buffer);
|
data.CopyTo(buffer);
|
||||||
|
|
||||||
|
@ -59,10 +60,9 @@ public static class Base58
|
||||||
}
|
}
|
||||||
|
|
||||||
int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count();
|
int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count();
|
||||||
var leadingZeros = new byte[leadingZeroCount];
|
|
||||||
|
|
||||||
if (bi.IsZero)
|
if (bi.IsZero)
|
||||||
return leadingZeros;
|
return new byte[leadingZeroCount];
|
||||||
|
|
||||||
var bytesBigEndian = bi.ToByteArray().Reverse().ToArray();
|
var bytesBigEndian = bi.ToByteArray().Reverse().ToArray();
|
||||||
|
|
||||||
|
@ -73,12 +73,26 @@ public static class Base58
|
||||||
|
|
||||||
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();
|
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();
|
||||||
|
|
||||||
return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros);
|
var result = new byte[leadingZeroCount + bytesBigEndian.Length - firstNonZeroIndex];
|
||||||
|
|
||||||
|
int p = 0;
|
||||||
|
while (p < leadingZeroCount)
|
||||||
|
result[p++] = 0;
|
||||||
|
|
||||||
|
for (int j = firstNonZeroIndex; j < bytesBigEndian.Length; j++)
|
||||||
|
result[p++] = bytesBigEndian[j];
|
||||||
|
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string Encode(ReadOnlySpan<byte> input)
|
public static string Encode(ReadOnlySpan<byte> input)
|
||||||
{
|
{
|
||||||
var data = input.ToArray().Reverse().Concat(new byte[] { 0 }).ToArray();
|
var data = new byte[input.Length + 1];
|
||||||
|
|
||||||
|
ArrayHelper.GetRevertedArray(input, data);
|
||||||
|
|
||||||
|
data[input.Length] = 0;
|
||||||
|
|
||||||
BigInteger value = new(data);
|
BigInteger value = new(data);
|
||||||
|
|
||||||
// Encode BigInteger to Base58 string
|
// Encode BigInteger to Base58 string
|
||||||
|
|
121
src/FrostFS.SDK.Cryptography/DataHasher.cs
Normal file
121
src/FrostFS.SDK.Cryptography/DataHasher.cs
Normal file
|
@ -0,0 +1,121 @@
|
||||||
|
using System;
|
||||||
|
using System.Buffers;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
|
public static class DataHasher
|
||||||
|
{
|
||||||
|
private const int LargeBlockSize = 1024 * 1024;
|
||||||
|
private const int SmallBlockSize = 256;
|
||||||
|
|
||||||
|
public static byte[] Hash(this ReadOnlyMemory<byte> bytes, HashAlgorithm algorithm)
|
||||||
|
{
|
||||||
|
if (algorithm is null)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(algorithm));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bytes.Length == 0)
|
||||||
|
{
|
||||||
|
return algorithm.ComputeHash([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
int rest, pos = 0;
|
||||||
|
|
||||||
|
var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize;
|
||||||
|
|
||||||
|
byte[] buffer = ArrayPool<byte>.Shared.Rent(blockSize);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while ((rest = bytes.Length - pos) > 0)
|
||||||
|
{
|
||||||
|
var size = Math.Min(rest, blockSize);
|
||||||
|
|
||||||
|
bytes.Slice(pos, size).CopyTo(buffer);
|
||||||
|
|
||||||
|
algorithm.TransformBlock(buffer, 0, size, buffer, 0);
|
||||||
|
|
||||||
|
pos += size;
|
||||||
|
}
|
||||||
|
|
||||||
|
algorithm.TransformFinalBlock([], 0, 0);
|
||||||
|
return algorithm.Hash;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (buffer != null)
|
||||||
|
{
|
||||||
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] Hash(this ReadOnlySpan<byte> bytes, HashAlgorithm algorithm)
|
||||||
|
{
|
||||||
|
if (algorithm is null)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(algorithm));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bytes.Length == 0)
|
||||||
|
{
|
||||||
|
return algorithm.ComputeHash([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
int rest, pos = 0;
|
||||||
|
|
||||||
|
var blockSize = bytes.Length <= SmallBlockSize ? SmallBlockSize : LargeBlockSize;
|
||||||
|
|
||||||
|
byte[] buffer = ArrayPool<byte>.Shared.Rent(blockSize);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while ((rest = bytes.Length - pos) > 0)
|
||||||
|
{
|
||||||
|
var size = Math.Min(rest, blockSize);
|
||||||
|
|
||||||
|
bytes.Slice(pos, size).CopyTo(buffer);
|
||||||
|
|
||||||
|
algorithm.TransformBlock(buffer, 0, size, buffer, 0);
|
||||||
|
|
||||||
|
pos += size;
|
||||||
|
}
|
||||||
|
|
||||||
|
algorithm.TransformFinalBlock([], 0, 0);
|
||||||
|
return algorithm.Hash;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (buffer != null)
|
||||||
|
{
|
||||||
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] Sha256(ReadOnlyMemory<byte> value)
|
||||||
|
{
|
||||||
|
using SHA256 sha = SHA256.Create();
|
||||||
|
return Hash(value, sha);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] Sha256(ReadOnlySpan<byte> value)
|
||||||
|
{
|
||||||
|
using SHA256 sha = SHA256.Create();
|
||||||
|
return Hash(value, sha);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] Sha512(ReadOnlyMemory<byte> value)
|
||||||
|
{
|
||||||
|
using SHA512 sha = SHA512.Create();
|
||||||
|
return Hash(value, sha);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] Sha512(ReadOnlySpan<byte> value)
|
||||||
|
{
|
||||||
|
using SHA512 sha = SHA512.Create();
|
||||||
|
return Hash(value, sha);
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,20 +1,9 @@
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
using System.Threading;
|
|
||||||
using CommunityToolkit.HighPerformance;
|
|
||||||
using Org.BouncyCastle.Crypto.Digests;
|
using Org.BouncyCastle.Crypto.Digests;
|
||||||
|
|
||||||
namespace FrostFS.SDK.Cryptography;
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
public static class Extentions
|
public static class Extentions
|
||||||
{
|
{
|
||||||
private static readonly SHA256 _sha256 = SHA256.Create();
|
|
||||||
private static SpinLock _spinlockSha256;
|
|
||||||
|
|
||||||
private static readonly SHA512 _sha512 = SHA512.Create();
|
|
||||||
private static SpinLock _spinlockSha512;
|
|
||||||
|
|
||||||
internal static byte[] RIPEMD160(this byte[] value)
|
internal static byte[] RIPEMD160(this byte[] value)
|
||||||
{
|
{
|
||||||
var hash = new byte[20];
|
var hash = new byte[20];
|
||||||
|
@ -24,72 +13,4 @@ public static class Extentions
|
||||||
digest.DoFinal(hash, 0);
|
digest.DoFinal(hash, 0);
|
||||||
return hash;
|
return hash;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static byte[] Sha256(this byte[] value)
|
|
||||||
{
|
|
||||||
bool lockTaken = false;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
_spinlockSha256.Enter(ref lockTaken);
|
|
||||||
|
|
||||||
return _sha256.ComputeHash(value);
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
if (lockTaken)
|
|
||||||
{
|
|
||||||
_spinlockSha256.Exit(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] Sha256(this ReadOnlyMemory<byte> value)
|
|
||||||
{
|
|
||||||
bool lockTaken = false;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
_spinlockSha256.Enter(ref lockTaken);
|
|
||||||
|
|
||||||
return _sha256.ComputeHash(value.AsStream());
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
if (lockTaken)
|
|
||||||
{
|
|
||||||
_spinlockSha256.Exit(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] Sha512(this ReadOnlyMemory<byte> value)
|
|
||||||
{
|
|
||||||
bool lockTaken = false;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
_spinlockSha512.Enter(ref lockTaken);
|
|
||||||
|
|
||||||
return _sha512.ComputeHash(value.AsStream());
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
if (lockTaken)
|
|
||||||
_spinlockSha512.Exit(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] Sha512(this Stream stream)
|
|
||||||
{
|
|
||||||
bool lockTaken = false;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
_spinlockSha512.Enter(ref lockTaken);
|
|
||||||
|
|
||||||
return _sha512.ComputeHash(stream);
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
if (lockTaken)
|
|
||||||
_spinlockSha512.Exit(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
<LangVersion>12.0</LangVersion>
|
<LangVersion>12.0</LangVersion>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<PackageId>FrostFS.SDK.Cryptography</PackageId>
|
<PackageId>FrostFS.SDK.Cryptography</PackageId>
|
||||||
<Version>1.0.3</Version>
|
<Version>1.0.7</Version>
|
||||||
<Description>
|
<Description>
|
||||||
Cryptography tools for C# SDK
|
Cryptography tools for C# SDK
|
||||||
</Description>
|
</Description>
|
||||||
|
@ -26,16 +26,18 @@
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
||||||
|
<SignAssembly>True</SignAssembly>
|
||||||
|
<AssemblyOriginatorKeyFile>.\\..\\..\\keyfile.snk</AssemblyOriginatorKeyFile>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
|
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
|
||||||
<PackageReference Include="CommunityToolkit.HighPerformance" Version="7.1.2" />
|
|
||||||
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
|
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.4">
|
||||||
<PrivateAssets>all</PrivateAssets>
|
<PrivateAssets>all</PrivateAssets>
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||||
</PackageReference>
|
</PackageReference>
|
||||||
<PackageReference Include="System.Memory" Version="4.5.5" />
|
<PackageReference Include="System.Memory" Version="4.5.5" />
|
||||||
|
<PackageReference Include="System.Text.Json" Version="7.0.1" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
@ -3,11 +3,11 @@ using System.Security.Cryptography;
|
||||||
|
|
||||||
namespace FrostFS.SDK.Cryptography;
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
public sealed class HashStream() : Stream
|
public sealed class HashStream(HashAlgorithm algorithm) : Stream
|
||||||
{
|
{
|
||||||
private long position;
|
private long position;
|
||||||
|
|
||||||
private readonly SHA512 _hash = SHA512.Create();
|
private readonly HashAlgorithm _hash = algorithm;
|
||||||
|
|
||||||
public override bool CanRead => false;
|
public override bool CanRead => false;
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,7 @@ public static class KeyExtension
|
||||||
private const int UncompressedPublicKeyLength = 65;
|
private const int UncompressedPublicKeyLength = 65;
|
||||||
|
|
||||||
private static readonly uint CheckSigDescriptor =
|
private static readonly uint CheckSigDescriptor =
|
||||||
BinaryPrimitives.ReadUInt32LittleEndian(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").Sha256());
|
BinaryPrimitives.ReadUInt32LittleEndian(DataHasher.Sha256(Encoding.ASCII.GetBytes("System.Crypto.CheckSig").AsSpan()));
|
||||||
|
|
||||||
public static byte[] Compress(this byte[] publicKey)
|
public static byte[] Compress(this byte[] publicKey)
|
||||||
{
|
{
|
||||||
|
@ -60,10 +60,15 @@ public static class KeyExtension
|
||||||
$"expected length={CompressedPublicKeyLength}, actual={publicKey.Length}"
|
$"expected length={CompressedPublicKeyLength}, actual={publicKey.Length}"
|
||||||
);
|
);
|
||||||
|
|
||||||
var script = new byte[] { 0x0c, CompressedPublicKeyLength }; //PUSHDATA1 33
|
var signDescriptor = BitConverter.GetBytes(CheckSigDescriptor);
|
||||||
script = ArrayHelper.Concat(script, publicKey);
|
|
||||||
script = ArrayHelper.Concat(script, [0x41]); //SYSCALL
|
var script = new byte[3 + publicKey.Length + signDescriptor.Length];
|
||||||
script = ArrayHelper.Concat(script, BitConverter.GetBytes(CheckSigDescriptor)); //Neo_Crypto_CheckSig
|
|
||||||
|
script[0] = 0x0c;
|
||||||
|
script[1] = CompressedPublicKeyLength; //PUSHDATA1 33
|
||||||
|
Buffer.BlockCopy(publicKey, 0, script, 2, publicKey.Length);
|
||||||
|
script[publicKey.Length + 2] = 0x41; //SYSCALL
|
||||||
|
Buffer.BlockCopy(signDescriptor, 0, script, publicKey.Length + 3, signDescriptor.Length);
|
||||||
|
|
||||||
return script;
|
return script;
|
||||||
}
|
}
|
||||||
|
@ -74,7 +79,26 @@ public static class KeyExtension
|
||||||
throw new ArgumentNullException(nameof(publicKey));
|
throw new ArgumentNullException(nameof(publicKey));
|
||||||
|
|
||||||
var script = publicKey.CreateSignatureRedeemScript();
|
var script = publicKey.CreateSignatureRedeemScript();
|
||||||
return script.Sha256().RIPEMD160();
|
return DataHasher.Sha256(script.AsSpan()).RIPEMD160();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetWIFFromPrivateKey(this byte[] privateKey)
|
||||||
|
{
|
||||||
|
if (privateKey == null || privateKey.Length != 32)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(privateKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
Span<byte> wifSpan = stackalloc byte[34];
|
||||||
|
|
||||||
|
wifSpan[0] = 0x80;
|
||||||
|
wifSpan[33] = 0x01;
|
||||||
|
|
||||||
|
privateKey.AsSpan().CopyTo(wifSpan.Slice(1));
|
||||||
|
|
||||||
|
var wif = Base58.Base58CheckEncode(wifSpan);
|
||||||
|
|
||||||
|
return wif;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string ToAddress(this byte[] scriptHash, byte version)
|
private static string ToAddress(this byte[] scriptHash, byte version)
|
||||||
|
@ -102,11 +126,6 @@ public static class KeyExtension
|
||||||
return privateKey;
|
return privateKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string Address(this ECDsa key)
|
|
||||||
{
|
|
||||||
return key.PublicKey().PublicKeyToAddress();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static string PublicKeyToAddress(this byte[] publicKey)
|
public static string PublicKeyToAddress(this byte[] publicKey)
|
||||||
{
|
{
|
||||||
if (publicKey == null)
|
if (publicKey == null)
|
||||||
|
@ -132,10 +151,12 @@ public static class KeyExtension
|
||||||
var pos = 33 - param.Q.X.Length;
|
var pos = 33 - param.Q.X.Length;
|
||||||
|
|
||||||
param.Q.X.CopyTo(pubkey, pos);
|
param.Q.X.CopyTo(pubkey, pos);
|
||||||
if (new BigInteger(param.Q.Y.Reverse().Concat(new byte[] { 0x0 }).ToArray()).IsEven)
|
|
||||||
pubkey[0] = 0x2;
|
var y = new byte[33];
|
||||||
else
|
ArrayHelper.GetRevertedArray(param.Q.Y, y);
|
||||||
pubkey[0] = 0x3;
|
y[32] = 0;
|
||||||
|
|
||||||
|
pubkey[0] = new BigInteger(y).IsEven ? (byte)0x2 : (byte)0x3;
|
||||||
|
|
||||||
return pubkey;
|
return pubkey;
|
||||||
}
|
}
|
||||||
|
@ -152,7 +173,9 @@ public static class KeyExtension
|
||||||
{
|
{
|
||||||
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
|
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
|
||||||
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
|
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
|
||||||
.GetEncoded(false).Skip(1).ToArray();
|
.GetEncoded(false)
|
||||||
|
.Skip(1)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
var key = ECDsa.Create(new ECParameters
|
var key = ECDsa.Create(new ECParameters
|
||||||
{
|
{
|
||||||
|
|
98
src/FrostFS.SDK.Cryptography/WalletExtractor.cs
Normal file
98
src/FrostFS.SDK.Cryptography/WalletExtractor.cs
Normal file
|
@ -0,0 +1,98 @@
|
||||||
|
using System;
|
||||||
|
using System.Text;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using Org.BouncyCastle.Crypto.Generators;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
|
public static class CryptoWallet
|
||||||
|
{
|
||||||
|
private static int N = 16384;
|
||||||
|
private static int R = 8;
|
||||||
|
private static int P = 8;
|
||||||
|
|
||||||
|
private enum CipherAction
|
||||||
|
{
|
||||||
|
Encrypt,
|
||||||
|
Decrypt
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] GetKeyFromEncodedWif(byte[] password, string encryptedWIF)
|
||||||
|
{
|
||||||
|
var nep2Data = Base58.Base58CheckDecode(encryptedWIF);
|
||||||
|
|
||||||
|
if (nep2Data.Length == 39 && nep2Data[0] == 1 && nep2Data[1] == 66 && nep2Data[2] == 224)
|
||||||
|
{
|
||||||
|
var addressHash = nep2Data.AsSpan(3, 4).ToArray();
|
||||||
|
var derivedKey = SCrypt.Generate(password, addressHash, N, R, P, 64);
|
||||||
|
var derivedKeyHalf1 = derivedKey.Take(32).ToArray();
|
||||||
|
var derivedKeyHalf2 = derivedKey.Skip(32).ToArray();
|
||||||
|
var encrypted = nep2Data.AsSpan(7, 32).ToArray();
|
||||||
|
var decrypted = Aes(encrypted, derivedKeyHalf2, CipherAction.Decrypt);
|
||||||
|
var plainPrivateKey = XorRange(decrypted, derivedKeyHalf1, 0, decrypted.Length);
|
||||||
|
|
||||||
|
return plainPrivateKey;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw new ArgumentException("Not valid NEP2 prefix.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string EncryptWif(string plainText, ECDsa key)
|
||||||
|
{
|
||||||
|
var addressHash = GetAddressHash(key);
|
||||||
|
var derivedKey = SCrypt.Generate(Encoding.UTF8.GetBytes(plainText), addressHash, N, R, P, 64);
|
||||||
|
var derivedHalf1 = derivedKey.Take(32).ToArray();
|
||||||
|
var derivedHalf2 = derivedKey.Skip(32).ToArray();
|
||||||
|
var encryptedHalf1 = Aes(XorRange(key.PrivateKey(), derivedHalf1, 0, 16), derivedHalf2, CipherAction.Encrypt);
|
||||||
|
var encryptedHalf2 = Aes(XorRange(key.PrivateKey(), derivedHalf1, 16, 32), derivedHalf2, CipherAction.Encrypt);
|
||||||
|
var prefixes = new byte[] { 1, 66, 224 };
|
||||||
|
var concatenation = ArrayHelper.Concat([prefixes, addressHash, encryptedHalf1, encryptedHalf2]);
|
||||||
|
|
||||||
|
return Base58.Base58CheckEncode(concatenation);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] GetAddressHash(ECDsa key)
|
||||||
|
{
|
||||||
|
string address = key.PublicKey().PublicKeyToAddress();
|
||||||
|
|
||||||
|
using SHA256 sha256 = SHA256.Create();
|
||||||
|
byte[] addressHashed = sha256.ComputeHash(Encoding.UTF8.GetBytes(address));
|
||||||
|
return [.. addressHashed.Take(4)];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] XorRange(byte[] arr1, byte[] arr2, int from, int length)
|
||||||
|
{
|
||||||
|
byte[] result = new byte[length];
|
||||||
|
|
||||||
|
var j = 0;
|
||||||
|
for (var i = from; i < length; ++i)
|
||||||
|
{
|
||||||
|
result[j++] = (byte)(arr1[i] ^ arr2[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] Aes(byte[] data, byte[] key, CipherAction action)
|
||||||
|
{
|
||||||
|
using var aes = System.Security.Cryptography.Aes.Create();
|
||||||
|
aes.Key = key;
|
||||||
|
aes.Mode = CipherMode.ECB;
|
||||||
|
aes.Padding = PaddingMode.None;
|
||||||
|
|
||||||
|
if (action == CipherAction.Encrypt)
|
||||||
|
{
|
||||||
|
return aes.CreateEncryptor().TransformFinalBlock(data, 0, data.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (action == CipherAction.Decrypt)
|
||||||
|
{
|
||||||
|
return aes.CreateDecryptor().TransformFinalBlock(data, 0, data.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ArgumentException("Wrong cippher action", nameof(action));
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,8 +1,7 @@
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
|
||||||
[assembly: AssemblyCompany("FrostFS.SDK.Protos")]
|
[assembly: AssemblyCompany("TrueCloudLab")]
|
||||||
[assembly: AssemblyFileVersion("1.0.2.0")]
|
[assembly: AssemblyFileVersion("1.0.7.0")]
|
||||||
[assembly: AssemblyInformationalVersion("1.0.0+d6fe0344538a223303c9295452f0ad73681ca376")]
|
|
||||||
[assembly: AssemblyProduct("FrostFS.SDK.Protos")]
|
[assembly: AssemblyProduct("FrostFS.SDK.Protos")]
|
||||||
[assembly: AssemblyTitle("FrostFS.SDK.Protos")]
|
[assembly: AssemblyTitle("FrostFS.SDK.Protos")]
|
||||||
[assembly: AssemblyVersion("1.0.3.0")]
|
[assembly: AssemblyVersion("1.0.7.0")]
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
<LangVersion>12.0</LangVersion>
|
<LangVersion>12.0</LangVersion>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<PackageId>FrostFS.SDK.Protos</PackageId>
|
<PackageId>FrostFS.SDK.Protos</PackageId>
|
||||||
<Version>1.0.3</Version>
|
<Version>1.0.7</Version>
|
||||||
<Description>
|
<Description>
|
||||||
Protobuf client for C# SDK
|
Protobuf client for C# SDK
|
||||||
</Description>
|
</Description>
|
||||||
|
@ -13,19 +13,21 @@
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<EnableNETAnalyzers>true</EnableNETAnalyzers>
|
<EnableNETAnalyzers>true</EnableNETAnalyzers>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<_SkipUpgradeNetAnalyzersNuGetWarning>true</_SkipUpgradeNetAnalyzersNuGetWarning>
|
<_SkipUpgradeNetAnalyzersNuGetWarning>true</_SkipUpgradeNetAnalyzersNuGetWarning>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
||||||
|
<SignAssembly>True</SignAssembly>
|
||||||
|
<AssemblyOriginatorKeyFile>.\\..\\..\\keyfile.snk</AssemblyOriginatorKeyFile>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
@ -10,11 +10,16 @@
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<EnableNETAnalyzers>true</EnableNETAnalyzers>
|
<EnableNETAnalyzers>true</EnableNETAnalyzers>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<SignAssembly>True</SignAssembly>
|
||||||
|
<AssemblyOriginatorKeyFile>.\\..\\..\\keyfile.snk</AssemblyOriginatorKeyFile>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
@ -42,5 +47,4 @@
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
@ -3,7 +3,6 @@ using System.Security.Cryptography;
|
||||||
using FrostFS.Object;
|
using FrostFS.Object;
|
||||||
using FrostFS.SDK.Client;
|
using FrostFS.SDK.Client;
|
||||||
using FrostFS.SDK.Client.Mappers.GRPC;
|
using FrostFS.SDK.Client.Mappers.GRPC;
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
using FrostFS.Session;
|
using FrostFS.Session;
|
||||||
|
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
@ -43,7 +42,7 @@ public class AsyncStreamReaderMock(string key, FrostFsObjectHeader objectHeader)
|
||||||
Signature = new Refs.Signature
|
Signature = new Refs.Signature
|
||||||
{
|
{
|
||||||
Key = Key.PublicKeyProto,
|
Key = Key.PublicKeyProto,
|
||||||
Sign = Key.ECDsaKey. SignData(header.ToByteArray()),
|
Sign = Key.ECDsaKey.SignData(header.ToByteArray()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -1,5 +1,3 @@
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using FrostFS.Container;
|
using FrostFS.Container;
|
||||||
using FrostFS.Object;
|
using FrostFS.Object;
|
||||||
using FrostFS.SDK.Client;
|
using FrostFS.SDK.Client;
|
||||||
|
|
|
@ -23,13 +23,16 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
||||||
|
|
||||||
var grpcVersion = Version.ToMessage();
|
var grpcVersion = Version.ToMessage();
|
||||||
|
|
||||||
|
Span<byte> ContainerGuidSpan = stackalloc byte[16];
|
||||||
|
ContainerGuid.ToBytes(ContainerGuidSpan);
|
||||||
|
|
||||||
PutResponse putResponse = new()
|
PutResponse putResponse = new()
|
||||||
{
|
{
|
||||||
Body = new PutResponse.Types.Body
|
Body = new PutResponse.Types.Body
|
||||||
{
|
{
|
||||||
ContainerId = new ContainerID
|
ContainerId = new ContainerID
|
||||||
{
|
{
|
||||||
Value = ByteString.CopyFrom(ContainerGuid.ToBytes())
|
Value = ByteString.CopyFrom(ContainerGuidSpan)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
MetaHeader = new ResponseMetaHeader
|
MetaHeader = new ResponseMetaHeader
|
||||||
|
@ -69,7 +72,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
||||||
Container = new Container.Container
|
Container = new Container.Container
|
||||||
{
|
{
|
||||||
Version = grpcVersion,
|
Version = grpcVersion,
|
||||||
Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()),
|
Nonce = ByteString.CopyFrom(ContainerGuidSpan),
|
||||||
PlacementPolicy = PlacementPolicy.GetPolicy()
|
PlacementPolicy = PlacementPolicy.GetPolicy()
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -4,7 +4,6 @@ using System.Security.Cryptography;
|
||||||
using FrostFS.Object;
|
using FrostFS.Object;
|
||||||
using FrostFS.SDK.Client;
|
using FrostFS.SDK.Client;
|
||||||
using FrostFS.SDK.Client.Mappers.GRPC;
|
using FrostFS.SDK.Client.Mappers.GRPC;
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
|
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
|
||||||
|
@ -42,6 +41,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
|
|
||||||
public Collection<ByteString> RangeHashResponses { get; } = [];
|
public Collection<ByteString> RangeHashResponses { get; } = [];
|
||||||
|
|
||||||
|
public Action? Callback;
|
||||||
|
|
||||||
public override Mock<ObjectService.ObjectServiceClient> GetMock()
|
public override Mock<ObjectService.ObjectServiceClient> GetMock()
|
||||||
{
|
{
|
||||||
var mock = new Mock<ObjectService.ObjectServiceClient>();
|
var mock = new Mock<ObjectService.ObjectServiceClient>();
|
||||||
|
@ -166,9 +167,14 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
It.IsAny<CancellationToken>()))
|
It.IsAny<CancellationToken>()))
|
||||||
.Returns((PutSingleRequest r, Metadata m, DateTime? dt, CancellationToken ct) =>
|
.Returns((PutSingleRequest r, Metadata m, DateTime? dt, CancellationToken ct) =>
|
||||||
{
|
{
|
||||||
|
Callback?.Invoke();
|
||||||
Verifier.CheckRequest(r);
|
Verifier.CheckRequest(r);
|
||||||
|
|
||||||
PutSingleRequests.Add(r);
|
var req = r.Clone();
|
||||||
|
|
||||||
|
// Clone method does not clone the payload but keeps a reference
|
||||||
|
req.Body.Object.Payload = ByteString.CopyFrom(r.Body.Object.Payload.ToByteArray());
|
||||||
|
PutSingleRequests.Add(req);
|
||||||
|
|
||||||
return new AsyncUnaryCall<PutSingleResponse>(
|
return new AsyncUnaryCall<PutSingleResponse>(
|
||||||
Task.FromResult(putSingleResponse),
|
Task.FromResult(putSingleResponse),
|
||||||
|
|
|
@ -50,7 +50,7 @@ public class ContainerTests : SmokeTestsBase
|
||||||
backupFactor: 1,
|
backupFactor: 1,
|
||||||
selectors: [],
|
selectors: [],
|
||||||
filter: [],
|
filter: [],
|
||||||
containerAttributes: [new ("testKey1", "testValue1")],
|
containerAttributes: [new("testKey1", "testValue1")],
|
||||||
new FrostFsReplica(3));
|
new FrostFsReplica(3));
|
||||||
|
|
||||||
Assert.NotNull(containerId);
|
Assert.NotNull(containerId);
|
||||||
|
@ -60,11 +60,14 @@ public class ContainerTests : SmokeTestsBase
|
||||||
Assert.NotNull(container);
|
Assert.NotNull(container);
|
||||||
|
|
||||||
Assert.NotNull(container.Attributes);
|
Assert.NotNull(container.Attributes);
|
||||||
Assert.Equal(2, container.Attributes.Count);
|
|
||||||
Assert.Equal("testKey1", container.Attributes[0].Key);
|
Assert.Equal("testKey1", container.Attributes[0].Key);
|
||||||
Assert.Equal("testValue1", container.Attributes[0].Value);
|
Assert.Equal("testValue1", container.Attributes[0].Value);
|
||||||
Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key);
|
|
||||||
Assert.Equal("true", container.Attributes[1].Value);
|
//Assert.Equal("true", container.Attributes[1].Value);
|
||||||
|
|
||||||
|
// for cluster
|
||||||
|
//Assert.Equal(2, container.Attributes.Count);
|
||||||
|
//Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[1].Key);
|
||||||
|
|
||||||
Assert.True(container.PlacementPolicy.HasValue);
|
Assert.True(container.PlacementPolicy.HasValue);
|
||||||
|
|
||||||
|
@ -74,7 +77,7 @@ public class ContainerTests : SmokeTestsBase
|
||||||
Assert.Empty(container.PlacementPolicy.Value.Filters);
|
Assert.Empty(container.PlacementPolicy.Value.Filters);
|
||||||
|
|
||||||
Assert.Single(container.PlacementPolicy.Value.Replicas);
|
Assert.Single(container.PlacementPolicy.Value.Replicas);
|
||||||
Assert.Equal(3, container.PlacementPolicy.Value.Replicas[0].Count);
|
Assert.Equal(3u, container.PlacementPolicy.Value.Replicas[0].Count);
|
||||||
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount);
|
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount);
|
||||||
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount);
|
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount);
|
||||||
Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector);
|
Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector);
|
||||||
|
@ -101,8 +104,8 @@ public class ContainerTests : SmokeTestsBase
|
||||||
new ("filter2", "filterKey2", 2, "testValue2", [new ("subFilter2", "subFilterKey2", 3, "testValue3",[])])
|
new ("filter2", "filterKey2", 2, "testValue2", [new ("subFilter2", "subFilterKey2", 3, "testValue3",[])])
|
||||||
];
|
];
|
||||||
|
|
||||||
Collection<FrostFsSelector> selectors = [
|
Collection<FrostFsSelector> selectors = [
|
||||||
new ("selector1") {
|
new ("selector1") {
|
||||||
Count = 1,
|
Count = 1,
|
||||||
Clause = 1,
|
Clause = 1,
|
||||||
Attribute = "attribute1",
|
Attribute = "attribute1",
|
||||||
|
@ -133,9 +136,9 @@ public class ContainerTests : SmokeTestsBase
|
||||||
Assert.NotNull(container);
|
Assert.NotNull(container);
|
||||||
|
|
||||||
Assert.NotNull(container.Attributes);
|
Assert.NotNull(container.Attributes);
|
||||||
Assert.Single(container.Attributes);
|
//Assert.Single(container.Attributes);
|
||||||
Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key);
|
//Assert.Equal("__SYSTEM__DISABLE_HOMOMORPHIC_HASHING", container.Attributes[0].Key);
|
||||||
Assert.Equal("true", container.Attributes[0].Value);
|
//Assert.Equal("true", container.Attributes[0].Value);
|
||||||
|
|
||||||
Assert.True(container.PlacementPolicy.HasValue);
|
Assert.True(container.PlacementPolicy.HasValue);
|
||||||
|
|
||||||
|
@ -188,7 +191,7 @@ public class ContainerTests : SmokeTestsBase
|
||||||
Assert.Empty(subFilter.Filters);
|
Assert.Empty(subFilter.Filters);
|
||||||
|
|
||||||
Assert.Single(container.PlacementPolicy.Value.Replicas);
|
Assert.Single(container.PlacementPolicy.Value.Replicas);
|
||||||
Assert.Equal(1, container.PlacementPolicy.Value.Replicas[0].Count);
|
Assert.Equal(1u, container.PlacementPolicy.Value.Replicas[0].Count);
|
||||||
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount);
|
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcParityCount);
|
||||||
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount);
|
Assert.Equal(0u, container.PlacementPolicy.Value.Replicas[0].EcDataCount);
|
||||||
Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector);
|
Assert.Equal("", container.PlacementPolicy.Value.Replicas[0].Selector);
|
||||||
|
@ -233,12 +236,12 @@ public class ContainerTests : SmokeTestsBase
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
#pragma warning disable xUnit1031 // Timeout is used
|
#pragma warning disable xUnit1031 // Timeout is used
|
||||||
if (!Task.WaitAll(tasks, 20000))
|
if (!Task.WaitAll(tasks, 20000))
|
||||||
{
|
{
|
||||||
Assert.Fail("cannot create containers");
|
Assert.Fail("cannot create containers");
|
||||||
}
|
}
|
||||||
#pragma warning restore xUnit1031
|
#pragma warning restore xUnit1031
|
||||||
|
|
||||||
var containers = client.ListContainersAsync(new PrmContainerGetAll(), default);
|
var containers = client.ListContainersAsync(new PrmContainerGetAll(), default);
|
||||||
|
|
||||||
|
|
|
@ -29,8 +29,8 @@ public class InterceptorTests() : SmokeTestsBase
|
||||||
Assert.True(callbackInvoked);
|
Assert.True(callbackInvoked);
|
||||||
Assert.True(interceptorInvoked);
|
Assert.True(interceptorInvoked);
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
Assert.Equal(2u, result.Version.Major);
|
||||||
Assert.Equal(13, result.Version.Minor);
|
Assert.Equal(13u, result.Version.Minor);
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
Assert.Equal(NodeState.Online, result.State);
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
Assert.Equal(33, result.PublicKey.Length);
|
||||||
Assert.NotNull(result.Addresses);
|
Assert.NotNull(result.Addresses);
|
||||||
|
|
|
@ -28,12 +28,14 @@ public class SmokeClientTests : SmokeTestsBase
|
||||||
|
|
||||||
var result = await client.GetNodeInfoAsync(default);
|
var result = await client.GetNodeInfoAsync(default);
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
Assert.Equal(2u, result.Version.Major);
|
||||||
Assert.Equal(13, result.Version.Minor);
|
Assert.Equal(13u, result.Version.Minor);
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
Assert.Equal(NodeState.Online, result.State);
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
Assert.Equal(33, result.PublicKey.Length);
|
||||||
Assert.Equal(2, result.Addresses.Count);
|
Assert.Single(result.Addresses);
|
||||||
Assert.Equal(11, result.Attributes.Count);
|
Assert.Equal(9, result.Attributes.Count);
|
||||||
|
//Assert.Equal(2, result.Addresses.Count);
|
||||||
|
//Assert.Equal(11, result.Attributes.Count);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
@ -101,8 +103,8 @@ public class SmokeClientTests : SmokeTestsBase
|
||||||
Assert.True(callbackInvoked);
|
Assert.True(callbackInvoked);
|
||||||
Assert.True(interceptorInvoked);
|
Assert.True(interceptorInvoked);
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
Assert.Equal(2u, result.Version.Major);
|
||||||
Assert.Equal(13, result.Version.Minor);
|
Assert.Equal(13u, result.Version.Minor);
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
Assert.Equal(NodeState.Online, result.State);
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
Assert.Equal(33, result.PublicKey.Length);
|
||||||
Assert.NotNull(result.Addresses);
|
Assert.NotNull(result.Addresses);
|
||||||
|
|
|
@ -26,7 +26,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
|
||||||
[InlineData(false, 2, 3)]
|
[InlineData(false, 2, 3)]
|
||||||
[InlineData(true, 2, 1)]
|
[InlineData(true, 2, 1)]
|
||||||
[InlineData(false, 2, 1)]
|
[InlineData(false, 2, 1)]
|
||||||
public async void FullScenario(bool unique, uint backupFactor, int replicas)
|
public async void FullScenario(bool unique, uint backupFactor, uint replicas)
|
||||||
{
|
{
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
||||||
_testOutputHelper.WriteLine("client created");
|
_testOutputHelper.WriteLine("client created");
|
||||||
|
@ -121,9 +121,16 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
|
||||||
await ValidateFilters(client, containerId, objectId, null, (ulong)bytes.Length);
|
await ValidateFilters(client, containerId, objectId, null, (ulong)bytes.Length);
|
||||||
_testOutputHelper.WriteLine($"\tfilters validated");
|
_testOutputHelper.WriteLine($"\tfilters validated");
|
||||||
|
|
||||||
if (type != clientCut)
|
if (type != clientCut && bytes.Length > 1024 + 64 && bytes.Length < 20 * 1024 * 1024)
|
||||||
{
|
{
|
||||||
await ValidatePatch(client, containerId, bytes, objectId);
|
// patch payload only
|
||||||
|
await ValidatePatch(client, containerId, bytes, true, objectId, [], false);
|
||||||
|
|
||||||
|
// patch attributes only
|
||||||
|
await ValidatePatch(client, containerId, bytes, false, objectId, [new("a1", "v1"), new("a2", "v2")], false);
|
||||||
|
|
||||||
|
// patch payload and attributes
|
||||||
|
await ValidatePatch(client, containerId, bytes, true, objectId, [new("a3", "v3"), new("a4", "v4")], true);
|
||||||
_testOutputHelper.WriteLine($"\tpatch validated");
|
_testOutputHelper.WriteLine($"\tpatch validated");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -157,7 +164,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
|
||||||
Assert.NotNull(x);
|
Assert.NotNull(x);
|
||||||
Assert.True(x.Length > 0);
|
Assert.True(x.Length > 0);
|
||||||
|
|
||||||
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
|
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -199,51 +206,73 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
|
||||||
_testOutputHelper.WriteLine($"\t\trange {range.Offset};{range.Length} validated");
|
_testOutputHelper.WriteLine($"\t\trange {range.Offset};{range.Length} validated");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
|
private static async Task ValidatePatch(
|
||||||
|
IFrostFSClient client,
|
||||||
|
FrostFsContainerId containerId,
|
||||||
|
byte[] bytes,
|
||||||
|
bool patchPayload,
|
||||||
|
FrostFsObjectId objectId,
|
||||||
|
FrostFsAttributePair[] attributes,
|
||||||
|
bool replaceAttributes)
|
||||||
{
|
{
|
||||||
if (bytes.Length < 1024 + 64 || bytes.Length > 5900)
|
byte[]? patch = null;
|
||||||
return;
|
FrostFsRange range = new();
|
||||||
|
if (patchPayload)
|
||||||
var patch = new byte[1024];
|
|
||||||
for (int i = 0; i < patch.Length; i++)
|
|
||||||
{
|
{
|
||||||
patch[i] = 32;
|
patch = new byte[1024];
|
||||||
}
|
for (int i = 0; i < patch.Length; i++)
|
||||||
|
{
|
||||||
|
patch[i] = 32;
|
||||||
|
}
|
||||||
|
|
||||||
var range = new FrostFsRange(64, (ulong)patch.Length);
|
range = new FrostFsRange(64, (ulong)patch.Length);
|
||||||
|
}
|
||||||
|
|
||||||
var patchParams = new PrmObjectPatch(
|
var patchParams = new PrmObjectPatch(
|
||||||
new FrostFsAddress(containerId, objectId),
|
new FrostFsAddress(containerId, objectId),
|
||||||
payload: new MemoryStream(patch),
|
payload: new MemoryStream(patch ?? []),
|
||||||
maxChunkLength: 1024,
|
maxChunkLength: 1024,
|
||||||
range: range);
|
range: range,
|
||||||
|
replaceAttributes: replaceAttributes,
|
||||||
|
newAttributes: attributes);
|
||||||
|
|
||||||
var newIbjId = await client.PatchObjectAsync(patchParams, default);
|
var newIbjId = await client.PatchObjectAsync(patchParams, default);
|
||||||
|
|
||||||
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, newIbjId), default);
|
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, newIbjId), default);
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
if (patchPayload)
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
{
|
||||||
ms.Write(chunk.Value.Span);
|
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
||||||
|
MemoryStream ms = new(downloadedBytes);
|
||||||
|
|
||||||
|
ReadOnlyMemory<byte>? chunk;
|
||||||
|
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
||||||
|
{
|
||||||
|
ms.Write(chunk.Value.Span);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < (int)range.Offset; i++)
|
||||||
|
Assert.Equal(downloadedBytes[i], bytes[i]);
|
||||||
|
|
||||||
|
var rangeEnd = range.Offset + range.Length;
|
||||||
|
|
||||||
|
for (int i = (int)range.Offset; i < (int)rangeEnd; i++)
|
||||||
|
Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]);
|
||||||
|
|
||||||
|
for (int i = (int)rangeEnd; i < bytes.Length; i++)
|
||||||
|
Assert.Equal(downloadedBytes[i], bytes[i]);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < (int)range.Offset; i++)
|
if (attributes != null && attributes.Length > 0)
|
||||||
Assert.Equal(downloadedBytes[i], bytes[i]);
|
{
|
||||||
|
foreach (var newAttr in attributes)
|
||||||
var rangeEnd = range.Offset + range.Length;
|
{
|
||||||
|
var i = @object!.Header!.Attributes!.Count(p => p.Key == newAttr.Key && p.Value == newAttr.Value);
|
||||||
for (int i = (int)range.Offset; i < (int)rangeEnd; i++)
|
Assert.Equal(1, i);
|
||||||
Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]);
|
}
|
||||||
|
}
|
||||||
for (int i = (int)rangeEnd; i < bytes.Length; i++)
|
|
||||||
Assert.Equal(downloadedBytes[i], bytes[i]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async Task ValidateFilters(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId, SplitId? splitId, ulong length)
|
private async Task ValidateFilters(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId, SplitId? splitId, ulong length)
|
||||||
{
|
{
|
||||||
var ecdsaKey = keyString.LoadWif();
|
var ecdsaKey = keyString.LoadWif();
|
||||||
|
@ -338,7 +367,6 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
|
||||||
Assert.Null(objHeader.Split);
|
Assert.Null(objHeader.Split);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static async Task<FrostFsObjectId> CreateObjectServerCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
|
private static async Task<FrostFsObjectId> CreateObjectServerCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
|
||||||
{
|
{
|
||||||
var header = new FrostFsObjectHeader(
|
var header = new FrostFsObjectHeader(
|
||||||
|
|
|
@ -1,42 +0,0 @@
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using FrostFS.SDK.Client;
|
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Tests.Smoke;
|
|
||||||
|
|
||||||
public abstract class MultiThreadTestsBase
|
|
||||||
{
|
|
||||||
private TestNodeInfo[] nodes;
|
|
||||||
|
|
||||||
protected CallContext? Ctx { get; }
|
|
||||||
|
|
||||||
protected MultiThreadTestsBase()
|
|
||||||
{
|
|
||||||
nodes = new TestNodeInfo[4];
|
|
||||||
|
|
||||||
nodes[0] = new(new Uri(""), "");
|
|
||||||
nodes[1] = new(new Uri(""), "");
|
|
||||||
nodes[2] = new(new Uri(""), "");
|
|
||||||
nodes[3] = new(new Uri(""), "");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public class TestNodeInfo
|
|
||||||
{
|
|
||||||
internal Uri Uri;
|
|
||||||
|
|
||||||
protected ECDsa? Key { get; }
|
|
||||||
|
|
||||||
protected FrostFsOwner? OwnerId { get; }
|
|
||||||
|
|
||||||
protected FrostFsVersion? Version { get; }
|
|
||||||
|
|
||||||
public TestNodeInfo(Uri uri, string keyString)
|
|
||||||
{
|
|
||||||
Uri = uri;
|
|
||||||
Key = keyString.LoadWif();
|
|
||||||
OwnerId = FrostFsOwner.FromKey(Key);
|
|
||||||
Version = new FrostFsVersion(2, 13);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,544 +0,0 @@
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using FrostFS.SDK.Client;
|
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Tests.Smoke;
|
|
||||||
|
|
||||||
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
|
||||||
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
|
|
||||||
public class MultithreadPoolSmokeTests : SmokeTestsBase
|
|
||||||
{
|
|
||||||
private InitParameters GetDefaultParams()
|
|
||||||
{
|
|
||||||
return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
|
|
||||||
{
|
|
||||||
Key = keyString.LoadWif(),
|
|
||||||
NodeParams = [new(1, url, 100.0f)],
|
|
||||||
ClientBuilder = null,
|
|
||||||
GracefulCloseOnSwitchTimeout = 30_000_000,
|
|
||||||
Logger = null
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NetworkMapTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNetmapSnapshotAsync(default);
|
|
||||||
|
|
||||||
Assert.True(result.Epoch > 0);
|
|
||||||
Assert.Single(result.NodeInfoCollection);
|
|
||||||
|
|
||||||
var item = result.NodeInfoCollection[0];
|
|
||||||
Assert.Equal(2, item.Version.Major);
|
|
||||||
Assert.Equal(13, item.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, item.State);
|
|
||||||
Assert.True(item.PublicKey.Length > 0);
|
|
||||||
Assert.Single(item.Addresses);
|
|
||||||
Assert.Equal(9, item.Attributes.Count);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
|
||||||
Assert.Equal(13, result.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
|
||||||
Assert.Single(result.Addresses);
|
|
||||||
Assert.Equal(9, result.Attributes.Count);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoStatisticsTwoNodesTest()
|
|
||||||
{
|
|
||||||
var callbackText = string.Empty;
|
|
||||||
|
|
||||||
var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
|
|
||||||
{
|
|
||||||
Key = keyString.LoadWif(),
|
|
||||||
NodeParams = [
|
|
||||||
new(1, url, 100.0f),
|
|
||||||
new(2, url.Replace('0', '1'), 100.0f)
|
|
||||||
],
|
|
||||||
ClientBuilder = null,
|
|
||||||
GracefulCloseOnSwitchTimeout = 30_000_000,
|
|
||||||
Logger = null,
|
|
||||||
Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"
|
|
||||||
};
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.Zero);
|
|
||||||
|
|
||||||
var error = await pool.Dial(ctx).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
var statistics = pool.Statistic();
|
|
||||||
|
|
||||||
Assert.False(string.IsNullOrEmpty(callbackText));
|
|
||||||
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoStatisticsTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var callbackText = string.Empty;
|
|
||||||
options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.Zero);
|
|
||||||
|
|
||||||
var error = await pool.Dial(ctx).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.False(string.IsNullOrEmpty(callbackText));
|
|
||||||
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void GetSessionTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var prm = new PrmSessionCreate(100);
|
|
||||||
|
|
||||||
var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
var ownerHash = Base58.Decode(OwnerId!.Value);
|
|
||||||
|
|
||||||
Assert.NotNull(token);
|
|
||||||
Assert.NotEqual(Guid.Empty, token.Id);
|
|
||||||
Assert.Equal(33, token.SessionKey.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void CreateObjectWithSessionToken()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["key1", "value1"]);
|
|
||||||
|
|
||||||
var containerId = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(1024);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
sessionToken: token);
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void FilterTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
lightWait);
|
|
||||||
|
|
||||||
var containerId = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var bytes = new byte[] { 1, 2, 3 };
|
|
||||||
|
|
||||||
var ParentHeader = new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular)
|
|
||||||
{
|
|
||||||
PayloadLength = 3
|
|
||||||
};
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")],
|
|
||||||
new FrostFsSplit()));
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId, false), default);
|
|
||||||
|
|
||||||
var ecdsaKey = keyString.LoadWif();
|
|
||||||
|
|
||||||
var networkInfo = await pool.GetNetmapSnapshotAsync(default);
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
|
|
||||||
|
|
||||||
var checkSum = CheckSum.CreateCheckSum(bytes);
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByPhysicallyStored());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter)
|
|
||||||
{
|
|
||||||
var resultObjectsCount = 0;
|
|
||||||
|
|
||||||
PrmObjectSearch searchParam = new(containerId, null, [], filter);
|
|
||||||
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(searchParam, default))
|
|
||||||
{
|
|
||||||
resultObjectsCount++;
|
|
||||||
var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
|
|
||||||
[InlineData(6 * 1024 * 1024 + 100)]
|
|
||||||
public async void SimpleScenarioTest(int objectSize)
|
|
||||||
{
|
|
||||||
bool callbackInvoked = false;
|
|
||||||
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
options.Callback = new((cs) =>
|
|
||||||
{
|
|
||||||
callbackInvoked = true;
|
|
||||||
Assert.True(cs.ElapsedMicroSeconds > 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["testKey", "testValue"]);
|
|
||||||
|
|
||||||
var createdContainer = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default);
|
|
||||||
Assert.NotNull(container);
|
|
||||||
Assert.True(callbackInvoked);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: createdContainer,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]));
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default);
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes!);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes!.First().Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes!.First().Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
await foreach (var _ in pool.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
|
|
||||||
[InlineData(6 * 1024 * 1024 + 100)]
|
|
||||||
public async void SimpleScenarioWithSessionTest(int objectSize)
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0));
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams);
|
|
||||||
|
|
||||||
var container = await pool.PutContainerAsync(createContainerParam, ctx);
|
|
||||||
|
|
||||||
var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx);
|
|
||||||
Assert.NotNull(containerInfo);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: container,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
sessionToken: token);
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default);
|
|
||||||
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes.First().Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes.First().Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
await foreach (var _ in pool.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
|
|
||||||
[InlineData(64 * 1024 * 1024 - 1)]
|
|
||||||
[InlineData(64 * 1024 * 1024 + 1)]
|
|
||||||
[InlineData(2 * 64 * 1024 * 1024 + 256)]
|
|
||||||
[InlineData(200)]
|
|
||||||
public async void ClientCutScenarioTest(int objectSize)
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
lightWait);
|
|
||||||
|
|
||||||
var containerId = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(10));
|
|
||||||
|
|
||||||
// ctx.Interceptors.Add(new CallbackInterceptor());
|
|
||||||
|
|
||||||
var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx);
|
|
||||||
|
|
||||||
Assert.NotNull(container);
|
|
||||||
|
|
||||||
byte[] bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectClientCutPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
payload: new MemoryStream(bytes));
|
|
||||||
|
|
||||||
var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes[0].Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes[0].Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByRootObject());
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
await foreach (var cid in pool.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail($"Container {cid.GetValue()} exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,684 +0,0 @@
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using FrostFS.SDK.Client;
|
|
||||||
using FrostFS.SDK.Client.Interfaces;
|
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
using FrostFS.SDK.SmokeTests;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Tests.Smoke;
|
|
||||||
|
|
||||||
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
|
||||||
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
|
|
||||||
public class MultithreadSmokeClientTests : SmokeTestsBase
|
|
||||||
{
|
|
||||||
[Fact]
|
|
||||||
public async void AccountTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
var result = await client.GetBalanceAsync(default);
|
|
||||||
|
|
||||||
Assert.NotNull(result);
|
|
||||||
Assert.True(result.Value == 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NetworkMapTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
var result = await client.GetNetmapSnapshotAsync(default);
|
|
||||||
|
|
||||||
Assert.True(result.Epoch > 0);
|
|
||||||
Assert.Single(result.NodeInfoCollection);
|
|
||||||
|
|
||||||
var item = result.NodeInfoCollection[0];
|
|
||||||
Assert.Equal(2, item.Version.Major);
|
|
||||||
Assert.Equal(13, item.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, item.State);
|
|
||||||
Assert.True(item.PublicKey.Length > 0);
|
|
||||||
Assert.Single(item.Addresses);
|
|
||||||
Assert.Equal(9, item.Attributes.Count);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
var result = await client.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
|
||||||
Assert.Equal(13, result.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
|
||||||
Assert.Single(result.Addresses);
|
|
||||||
Assert.Equal(9, result.Attributes.Count);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoStatisticsTest()
|
|
||||||
{
|
|
||||||
var options = ClientOptions;
|
|
||||||
|
|
||||||
var callbackContent = string.Empty;
|
|
||||||
options.Value.Callback = (cs) => callbackContent = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
|
|
||||||
|
|
||||||
var client = FrostFSClient.GetInstance(options, GrpcChannel);
|
|
||||||
|
|
||||||
var result = await client.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.NotEmpty(callbackContent);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void GetSessionTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
var token = await client.CreateSessionAsync(new(100), default);
|
|
||||||
|
|
||||||
Assert.NotNull(token);
|
|
||||||
Assert.NotEqual(Guid.Empty, token.Id);
|
|
||||||
Assert.Equal(33, token.SessionKey.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void CreateObjectWithSessionToken()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["key1", "value1"]);
|
|
||||||
|
|
||||||
var containerId = await client.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(1024);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
sessionToken: token);
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default)
|
|
||||||
.ConfigureAwait(true);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk().ConfigureAwait(true)) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(client).ConfigureAwait(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void FilterTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
lightWait);
|
|
||||||
|
|
||||||
var containerId = await client.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var bytes = new byte[] { 1, 2, 3 };
|
|
||||||
|
|
||||||
var ParentHeader = new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular)
|
|
||||||
{
|
|
||||||
PayloadLength = 3
|
|
||||||
};
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")],
|
|
||||||
new FrostFsSplit()));
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var head = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var ecdsaKey = keyString.LoadWif();
|
|
||||||
|
|
||||||
var networkInfo = await client.GetNetmapSnapshotAsync(default);
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
|
|
||||||
|
|
||||||
var checkSum = CheckSum.CreateCheckSum(bytes);
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByPhysicallyStored());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter)
|
|
||||||
{
|
|
||||||
var resultObjectsCount = 0;
|
|
||||||
|
|
||||||
PrmObjectSearch searchParam = new(containerId, null, [], filter);
|
|
||||||
|
|
||||||
await foreach (var objId in client.SearchObjectsAsync(searchParam, default))
|
|
||||||
{
|
|
||||||
resultObjectsCount++;
|
|
||||||
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
|
|
||||||
[InlineData(6 * 1024 * 1024 + 100)]
|
|
||||||
public async void SimpleScenarioTest(int objectSize)
|
|
||||||
{
|
|
||||||
bool callbackInvoked = false;
|
|
||||||
|
|
||||||
var options = ClientOptions;
|
|
||||||
|
|
||||||
options.Value.Callback = new((cs) =>
|
|
||||||
{
|
|
||||||
callbackInvoked = true;
|
|
||||||
Assert.True(cs.ElapsedMicroSeconds > 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
var client = FrostFSClient.GetInstance(options, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["testKey", "testValue"]);
|
|
||||||
|
|
||||||
var createdContainer = await client.PutContainerAsync(createContainerParam, ctx);
|
|
||||||
|
|
||||||
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
|
|
||||||
Assert.NotNull(container);
|
|
||||||
Assert.True(callbackInvoked);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: createdContainer,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]));
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default);
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes.First().Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes.First().Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
await foreach (var _ in client.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void PatchTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["testKey", "testValue"]);
|
|
||||||
|
|
||||||
var createdContainer = await client.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
|
|
||||||
Assert.NotNull(container);
|
|
||||||
|
|
||||||
var bytes = new byte[1024];
|
|
||||||
for (int i = 0; i < 1024; i++)
|
|
||||||
{
|
|
||||||
bytes[i] = 31;
|
|
||||||
}
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: createdContainer,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]));
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var patch = new byte[16];
|
|
||||||
for (int i = 0; i < 16; i++)
|
|
||||||
{
|
|
||||||
patch[i] = 32;
|
|
||||||
}
|
|
||||||
|
|
||||||
var range = new FrostFsRange(8, (ulong)patch.Length);
|
|
||||||
|
|
||||||
var patchParams = new PrmObjectPatch(
|
|
||||||
new FrostFsAddress(createdContainer, objectId),
|
|
||||||
payload: new MemoryStream(patch),
|
|
||||||
maxChunkLength: 32,
|
|
||||||
range: range);
|
|
||||||
|
|
||||||
var newIbjId = await client.PatchObjectAsync(patchParams, default);
|
|
||||||
|
|
||||||
var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < (int)range.Offset; i++)
|
|
||||||
Assert.Equal(downloadedBytes[i], bytes[i]);
|
|
||||||
|
|
||||||
var rangeEnd = range.Offset + range.Length;
|
|
||||||
|
|
||||||
for (int i = (int)range.Offset; i < (int)rangeEnd; i++)
|
|
||||||
Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]);
|
|
||||||
|
|
||||||
for (int i = (int)rangeEnd; i < bytes.Length; i++)
|
|
||||||
Assert.Equal(downloadedBytes[i], bytes[i]);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
await foreach (var _ in client.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void RangeTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["testKey", "testValue"]);
|
|
||||||
|
|
||||||
var createdContainer = await client.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
|
|
||||||
Assert.NotNull(container);
|
|
||||||
|
|
||||||
var bytes = new byte[256];
|
|
||||||
for (int i = 0; i < 256; i++)
|
|
||||||
{
|
|
||||||
bytes[i] = (byte)i;
|
|
||||||
}
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(containerId: createdContainer, type: FrostFsObjectType.Regular));
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64));
|
|
||||||
|
|
||||||
var rangeReader = await client.GetRangeAsync(rangeParam, default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[rangeParam.Range.Length];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await rangeReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
await foreach (var _ in client.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void RangeHashTest()
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["testKey", "testValue"]);
|
|
||||||
|
|
||||||
var createdContainer = await client.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer), default);
|
|
||||||
Assert.NotNull(container);
|
|
||||||
|
|
||||||
var bytes = new byte[256];
|
|
||||||
for (int i = 0; i < 256; i++)
|
|
||||||
{
|
|
||||||
bytes[i] = (byte)i;
|
|
||||||
}
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: createdContainer,
|
|
||||||
type: FrostFsObjectType.Regular));
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [new FrostFsRange(100, 64)], bytes);
|
|
||||||
|
|
||||||
var hashes = await client.GetRangeHashAsync(rangeParam, default);
|
|
||||||
|
|
||||||
foreach (var hash in hashes)
|
|
||||||
{
|
|
||||||
var x = hash[..32].ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
await foreach (var _ in client.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
|
|
||||||
[InlineData(6 * 1024 * 1024 + 100)]
|
|
||||||
public async void SimpleScenarioWithSessionTest(int objectSize)
|
|
||||||
{
|
|
||||||
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
||||||
//Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
|
|
||||||
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams);
|
|
||||||
|
|
||||||
var container = await client.PutContainerAsync(createContainerParam, ctx);
|
|
||||||
|
|
||||||
var containerInfo = await client.GetContainerAsync(new PrmContainerGet(container), ctx);
|
|
||||||
Assert.NotNull(containerInfo);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: container,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
sessionToken: token);
|
|
||||||
|
|
||||||
var stream = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in client.SearchObjectsAsync(
|
|
||||||
new PrmObjectSearch(container, token, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default);
|
|
||||||
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes.First().Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes.First().Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await client.GetObjectAsync(new PrmObjectGet(container, objectId, token), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
await foreach (var _ in client.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
|
|
||||||
[InlineData(64 * 1024 * 1024 - 1)]
|
|
||||||
[InlineData(64 * 1024 * 1024 + 1)]
|
|
||||||
[InlineData(2 * 64 * 1024 * 1024 + 256)]
|
|
||||||
[InlineData(200)]
|
|
||||||
public async void ClientCutScenarioTest(int objectSize)
|
|
||||||
{
|
|
||||||
var options = ClientOptions;
|
|
||||||
options.Value.Interceptors.Add(new CallbackInterceptor());
|
|
||||||
|
|
||||||
var client = FrostFSClient.GetInstance(options, GrpcChannel);
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
lightWait);
|
|
||||||
|
|
||||||
var containerId = await client.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(10));
|
|
||||||
|
|
||||||
var container = await client.GetContainerAsync(new PrmContainerGet(containerId), ctx);
|
|
||||||
|
|
||||||
Assert.NotNull(container);
|
|
||||||
|
|
||||||
byte[] bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectClientCutPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
payload: new MemoryStream(bytes));
|
|
||||||
|
|
||||||
var objectId = await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes[0].Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes[0].Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await CheckFilter(client, containerId, new FilterByRootObject());
|
|
||||||
|
|
||||||
await Cleanup(client);
|
|
||||||
|
|
||||||
var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5));
|
|
||||||
|
|
||||||
IAsyncEnumerator<FrostFsContainerId>? enumerator = null;
|
|
||||||
do
|
|
||||||
{
|
|
||||||
if (deadline <= DateTime.UtcNow)
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
enumerator = client.ListContainersAsync(default, default).GetAsyncEnumerator();
|
|
||||||
await Task.Delay(500);
|
|
||||||
}
|
|
||||||
while (await enumerator!.MoveNextAsync());
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoCallbackAndInterceptorTest()
|
|
||||||
{
|
|
||||||
bool callbackInvoked = false;
|
|
||||||
bool intercepterInvoked = false;
|
|
||||||
|
|
||||||
var options = ClientOptions;
|
|
||||||
options.Value.Callback = (cs) =>
|
|
||||||
{
|
|
||||||
callbackInvoked = true;
|
|
||||||
Assert.True(cs.ElapsedMicroSeconds > 0);
|
|
||||||
};
|
|
||||||
|
|
||||||
options.Value.Interceptors.Add(new CallbackInterceptor(s => intercepterInvoked = true));
|
|
||||||
|
|
||||||
var client = FrostFSClient.GetInstance(options, GrpcChannel);
|
|
||||||
|
|
||||||
var result = await client.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.True(callbackInvoked);
|
|
||||||
Assert.True(intercepterInvoked);
|
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
|
||||||
Assert.Equal(13, result.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
|
||||||
Assert.Single(result.Addresses);
|
|
||||||
Assert.Equal(9, result.Attributes.Count);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,546 +0,0 @@
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
using FrostFS.SDK.Client;
|
|
||||||
using FrostFS.SDK.Cryptography;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.Tests.Smoke;
|
|
||||||
|
|
||||||
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
|
||||||
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
|
|
||||||
public class PoolSmokeTests : SmokeTestsBase
|
|
||||||
{
|
|
||||||
private InitParameters GetDefaultParams()
|
|
||||||
{
|
|
||||||
return new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
|
|
||||||
{
|
|
||||||
Key = keyString.LoadWif(),
|
|
||||||
NodeParams = [new(1, url, 100.0f)],
|
|
||||||
ClientBuilder = null,
|
|
||||||
GracefulCloseOnSwitchTimeout = 30_000_000,
|
|
||||||
Logger = null
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NetworkMapTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNetmapSnapshotAsync(default);
|
|
||||||
|
|
||||||
Assert.True(result.Epoch > 0);
|
|
||||||
Assert.Single(result.NodeInfoCollection);
|
|
||||||
|
|
||||||
var item = result.NodeInfoCollection[0];
|
|
||||||
Assert.Equal(2, item.Version.Major);
|
|
||||||
Assert.Equal(13, item.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, item.State);
|
|
||||||
Assert.True(item.PublicKey.Length > 0);
|
|
||||||
Assert.Single(item.Addresses);
|
|
||||||
Assert.Equal(9, item.Attributes.Count);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero));
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.Equal(2, result.Version.Major);
|
|
||||||
Assert.Equal(13, result.Version.Minor);
|
|
||||||
Assert.Equal(NodeState.Online, result.State);
|
|
||||||
Assert.Equal(33, result.PublicKey.Length);
|
|
||||||
Assert.Single(result.Addresses);
|
|
||||||
Assert.Equal(9, result.Attributes.Count);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoStatisticsTwoNodesTest()
|
|
||||||
{
|
|
||||||
var callbackText = string.Empty;
|
|
||||||
|
|
||||||
var options = new InitParameters((url) => Grpc.Net.Client.GrpcChannel.ForAddress(new Uri(url)))
|
|
||||||
{
|
|
||||||
Key = keyString.LoadWif(),
|
|
||||||
NodeParams = [
|
|
||||||
new(1, url, 100.0f),
|
|
||||||
new(2, url.Replace('0', '1'), 100.0f)
|
|
||||||
],
|
|
||||||
ClientBuilder = null,
|
|
||||||
GracefulCloseOnSwitchTimeout = 30_000_000,
|
|
||||||
Logger = null,
|
|
||||||
Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds"
|
|
||||||
};
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.Zero);
|
|
||||||
|
|
||||||
var error = await pool.Dial(ctx).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
var statistics = pool.Statistic();
|
|
||||||
|
|
||||||
Assert.False(string.IsNullOrEmpty(callbackText));
|
|
||||||
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void NodeInfoStatisticsTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var callbackText = string.Empty;
|
|
||||||
options.Callback = (cs) => callbackText = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.Zero);
|
|
||||||
|
|
||||||
var error = await pool.Dial(ctx).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var result = await pool.GetNodeInfoAsync(default);
|
|
||||||
|
|
||||||
Assert.False(string.IsNullOrEmpty(callbackText));
|
|
||||||
Assert.Contains(" took ", callbackText, StringComparison.Ordinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void GetSessionTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var prm = new PrmSessionCreate(100);
|
|
||||||
|
|
||||||
var token = await pool.CreateSessionAsync(prm, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
var ownerHash = Base58.Decode(OwnerId!.Value);
|
|
||||||
|
|
||||||
Assert.NotNull(token);
|
|
||||||
Assert.NotEqual(Guid.Empty, token.Id);
|
|
||||||
Assert.Equal(33, token.SessionKey.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void CreateObjectWithSessionToken()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["key1", "value1"]);
|
|
||||||
|
|
||||||
var containerId = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(1024);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
sessionToken: token);
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async void FilterTest()
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
lightWait);
|
|
||||||
|
|
||||||
var containerId = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var bytes = new byte[] { 1, 2, 3 };
|
|
||||||
|
|
||||||
var ParentHeader = new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular)
|
|
||||||
{
|
|
||||||
PayloadLength = 3
|
|
||||||
};
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")],
|
|
||||||
new FrostFsSplit()));
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var head = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var ecdsaKey = keyString.LoadWif();
|
|
||||||
|
|
||||||
var networkInfo = await pool.GetNetmapSnapshotAsync(default);
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header!.Split!.SplitId));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
|
|
||||||
|
|
||||||
var checkSum = CheckSum.CreateCheckSum(bytes);
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByPhysicallyStored());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task CheckFilter(Pool pool, FrostFsContainerId containerId, IObjectFilter filter)
|
|
||||||
{
|
|
||||||
var resultObjectsCount = 0;
|
|
||||||
|
|
||||||
PrmObjectSearch searchParam = new(containerId, null, [], filter);
|
|
||||||
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(searchParam, default))
|
|
||||||
{
|
|
||||||
resultObjectsCount++;
|
|
||||||
var objHeader = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
|
|
||||||
[InlineData(6 * 1024 * 1024 + 100)]
|
|
||||||
public async void SimpleScenarioTest(int objectSize)
|
|
||||||
{
|
|
||||||
bool callbackInvoked = false;
|
|
||||||
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
options.Callback = new((cs) =>
|
|
||||||
{
|
|
||||||
callbackInvoked = true;
|
|
||||||
Assert.True(cs.ElapsedMicroSeconds > 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.Zero);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams,
|
|
||||||
xheaders: ["testKey", "testValue"]);
|
|
||||||
|
|
||||||
var createdContainer = await pool.PutContainerAsync(createContainerParam, ctx);
|
|
||||||
|
|
||||||
var container = await pool.GetContainerAsync(new PrmContainerGet(createdContainer), default);
|
|
||||||
Assert.NotNull(container);
|
|
||||||
Assert.True(callbackInvoked);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: createdContainer,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]));
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(createdContainer, null, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId), default);
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes!);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes!.First().Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes!.First().Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(createdContainer, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
await foreach (var _ in pool.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
|
|
||||||
[InlineData(6 * 1024 * 1024 + 100)]
|
|
||||||
public async void SimpleScenarioWithSessionTest(int objectSize)
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
options.Callback = new((cs) => Assert.True(cs.ElapsedMicroSeconds > 0));
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
PrmWait.DefaultParams);
|
|
||||||
|
|
||||||
var container = await pool.PutContainerAsync(createContainerParam, ctx);
|
|
||||||
|
|
||||||
var containerInfo = await pool.GetContainerAsync(new PrmContainerGet(container), ctx);
|
|
||||||
Assert.NotNull(containerInfo);
|
|
||||||
|
|
||||||
var bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: container,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
sessionToken: token);
|
|
||||||
|
|
||||||
var stream = await pool.PutObjectAsync(param, new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
await stream.WriteAsync(bytes.AsMemory());
|
|
||||||
var objectId = await stream.CompleteAsync();
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
var objs = pool.SearchObjectsAsync(new PrmObjectSearch(container, token, [], filter), default);
|
|
||||||
await foreach (var objId in objs)
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId, false, token), default);
|
|
||||||
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader);
|
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes.First().Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes.First().Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(container, objectId, token), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
await foreach (var _ in pool.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail("Containers exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Theory]
|
|
||||||
[InlineData(1)]
|
|
||||||
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
|
|
||||||
[InlineData(64 * 1024 * 1024 - 1)]
|
|
||||||
[InlineData(64 * 1024 * 1024 + 1)]
|
|
||||||
[InlineData(2 * 64 * 1024 * 1024 + 256)]
|
|
||||||
[InlineData(200)]
|
|
||||||
public async void ClientCutScenarioTest(int objectSize)
|
|
||||||
{
|
|
||||||
var options = GetDefaultParams();
|
|
||||||
|
|
||||||
var pool = new Pool(options);
|
|
||||||
|
|
||||||
var error = await pool.Dial(new CallContext(TimeSpan.Zero)).ConfigureAwait(true);
|
|
||||||
|
|
||||||
Assert.Null(error);
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(
|
|
||||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
|
||||||
lightWait);
|
|
||||||
|
|
||||||
var containerId = await pool.PutContainerAsync(createContainerParam, default);
|
|
||||||
|
|
||||||
var ctx = new CallContext(TimeSpan.FromSeconds(10));
|
|
||||||
|
|
||||||
// ctx.Interceptors.Add(new CallbackInterceptor());
|
|
||||||
|
|
||||||
var container = await pool.GetContainerAsync(new PrmContainerGet(containerId), ctx);
|
|
||||||
|
|
||||||
Assert.NotNull(container);
|
|
||||||
|
|
||||||
byte[] bytes = GetRandomBytes(objectSize);
|
|
||||||
|
|
||||||
var param = new PrmObjectClientCutPut(
|
|
||||||
new FrostFsObjectHeader(
|
|
||||||
containerId: containerId,
|
|
||||||
type: FrostFsObjectType.Regular,
|
|
||||||
[new FrostFsAttributePair("fileName", "test")]),
|
|
||||||
payload: new MemoryStream(bytes));
|
|
||||||
|
|
||||||
var objectId = await pool.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
|
|
||||||
|
|
||||||
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
|
|
||||||
|
|
||||||
bool hasObject = false;
|
|
||||||
await foreach (var objId in pool.SearchObjectsAsync(new PrmObjectSearch(containerId, null, [], filter), default))
|
|
||||||
{
|
|
||||||
hasObject = true;
|
|
||||||
|
|
||||||
var res = await pool.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var objHeader = res.HeaderInfo;
|
|
||||||
Assert.NotNull(objHeader); Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
|
||||||
Assert.NotNull(objHeader.Attributes);
|
|
||||||
Assert.Single(objHeader.Attributes);
|
|
||||||
Assert.Equal("fileName", objHeader.Attributes[0].Key);
|
|
||||||
Assert.Equal("test", objHeader.Attributes[0].Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.True(hasObject);
|
|
||||||
|
|
||||||
var @object = await pool.GetObjectAsync(new PrmObjectGet(containerId, objectId), default);
|
|
||||||
|
|
||||||
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
||||||
MemoryStream ms = new(downloadedBytes);
|
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? chunk = null;
|
|
||||||
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
||||||
{
|
|
||||||
ms.Write(chunk.Value.Span);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
|
|
||||||
|
|
||||||
await CheckFilter(pool, containerId, new FilterByRootObject());
|
|
||||||
|
|
||||||
await Cleanup(pool);
|
|
||||||
|
|
||||||
await foreach (var cid in pool.ListContainersAsync(default, default))
|
|
||||||
{
|
|
||||||
Assert.Fail($"Container {cid.GetValue()} exist");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -16,19 +16,13 @@ namespace FrostFS.SDK.Tests.Smoke;
|
||||||
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
|
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
|
||||||
public abstract class SmokeTestsBase
|
public abstract class SmokeTestsBase
|
||||||
{
|
{
|
||||||
// cluster Ori
|
|
||||||
// internal readonly string url = "http://10.78.128.207:8080";
|
|
||||||
// internal readonly string keyString = "L4JWLdedUd4b21sriRHtCPGkjG2Mryz2AWLiVqTBSNyxxyAUcc7s";
|
|
||||||
|
|
||||||
// cluster
|
// cluster
|
||||||
internal readonly string url = "http://10.78.128.190:8080";
|
// internal readonly string url = "http://10.78.128.190:8080";
|
||||||
internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba";
|
// internal readonly string keyString = "L47c3bunc6bJd7uEAfPUae2VkyupFR9nizoH6jfPonzQxijqH2Ba";
|
||||||
|
|
||||||
// WSL2
|
// WSL2
|
||||||
// internal readonly string url = "http://172.29.238.97:8080";
|
internal readonly string url = "http://172.29.238.97:8080"; // "http://172.20.8.23:8080";
|
||||||
// internal readonly string keyString = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq"; // "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK";
|
internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK";
|
||||||
|
|
||||||
//"KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq";
|
|
||||||
|
|
||||||
protected ECDsa? Key { get; }
|
protected ECDsa? Key { get; }
|
||||||
|
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue