Compare commits

..

17 commits

Author SHA1 Message Date
c51e4bd44f [#224] Add multipart test cases with bucket without versioning 2024-04-24 11:15:47 +03:00
048f713f3f [#219] Add bucket/container listing check in multipart test case 2024-04-22 17:47:45 +03:00
370c1059f1 update-allure 2023-12-20 17:12:03 +00:00
6980cd15bd [#172] Executive command changed 1.3
Added exception of error 'Too many requests' in log analyzer

Signed-off-by: Mikhail Kadilov m.kadilov@yadro.com
2023-12-11 15:19:58 +03:00
8115d28bcd [#168] Added exception in log analyzer and fixed arguments order in get_filtered_logs
Added exception in log analyzer and fixed arguments order in get_filtered_logs

Signed-off-by: Mikhail Kadilov <m.kadilov@yadro.com>
2023-12-07 08:45:33 +00:00
8d2b3aee0e update-allure-title 2023-12-04 13:52:47 +03:00
fdd5fd55d4 Reduction of sanity tests
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
7f9517075b Change sanity makrs first step
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
849e052ad2 Remove sanity from awsclient
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
354c054a8a Skip teardown if sanity in markexpr 2023-11-28 08:20:48 +00:00
6d32372d93 Remove pytest hooks 2023-11-28 08:20:48 +00:00
b8dbf086be Add sanity marks
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
e2e4a0c667 [#160] Add try\except
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2023-11-28 08:14:35 +00:00
8f339ecbcd Add await
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2023-11-16 09:06:17 +03:00
9f8485f5eb Update check Policy: REP 1 IN SPB REP 1 IN MSK REP 3
Signed-off-by: Ekaterina Chernitsyna <e.chernitsyna@yadro.com>
2023-11-15 07:39:17 +00:00
1629caddec discard_dynamic_title
Signed-off-by: Ekaterina Chernitsyna <e.chernitsyna@yadro.com>
2023-11-15 07:38:30 +00:00
cd06a073a2 [#129] Updates for failover
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
(cherry picked from commit ae57672c7d)
2023-11-14 10:31:35 +03:00
59 changed files with 3331 additions and 4885 deletions

View file

@ -3,104 +3,108 @@ hosts:
attributes:
sudo_shell: false
plugin_name: docker
healthcheck_plugin_name: basic
attributes:
skip_readiness_check: True
force_transactions: True
services:
- name: frostfs-storage_01
- name: s01
attributes:
container_name: s01
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet01.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet01.json
wallet_password: ""
volume_name: storage_storage_s01
endpoint_data0: s01.frostfs.devenv:8080
control_endpoint: s01.frostfs.devenv:8081
un_locode: "RU MOW"
- name: frostfs-storage_02
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s02
attributes:
container_name: s02
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet02.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet02.json
wallet_password: ""
volume_name: storage_storage_s02
endpoint_data0: s02.frostfs.devenv:8080
control_endpoint: s02.frostfs.devenv:8081
un_locode: "RU LED"
- name: frostfs-storage_03
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s03
attributes:
container_name: s03
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet03.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet03.json
wallet_password: ""
volume_name: storage_storage_s03
endpoint_data0: s03.frostfs.devenv:8080
control_endpoint: s03.frostfs.devenv:8081
un_locode: "SE STO"
- name: frostfs-storage_04
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s04
attributes:
container_name: s04
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet04.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet04.json
wallet_password: ""
volume_name: storage_storage_s04
endpoint_data0: s04.frostfs.devenv:8080
control_endpoint: s04.frostfs.devenv:8081
un_locode: "FI HEL"
- name: frostfs-s3_01
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s3-gate01
attributes:
container_name: s3_gate
config_path: ../frostfs-dev-env/services/s3_gate/.s3.env
wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json
local_wallet_config_path: ./TemporaryDir/password-s3.yml
local_config_path: ./TemporaryDir/password-s3.yml
local_wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json
wallet_password: "s3"
endpoint_data0: https://s3.frostfs.devenv:8080
- name: frostfs-http_01
- name: http-gate01
attributes:
container_name: http_gate
config_path: ../frostfs-dev-env/services/http_gate/.http.env
wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json
local_wallet_config_path: ./TemporaryDir/password-other.yml
local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json
wallet_password: "one"
endpoint_data0: http://http.frostfs.devenv
- name: frostfs-ir_01
- name: ir01
attributes:
container_name: ir01
config_path: ../frostfs-dev-env/services/ir/.ir.env
wallet_path: ../frostfs-dev-env/services/ir/az.json
local_wallet_config_path: ./TemporaryDir/password-other.yml
local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/ir/az.json
wallet_password: "one"
- name: neo-go_01
- name: morph-chain01
attributes:
container_name: morph_chain
config_path: ../frostfs-dev-env/services/morph_chain/protocol.privnet.yml
wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json
local_wallet_config_path: ./TemporaryDir/password-other.yml
local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json
wallet_password: "one"
endpoint_internal0: http://morph-chain.frostfs.devenv:30333
- name: main-chain_01
- name: main-chain01
attributes:
container_name: main_chain
config_path: ../frostfs-dev-env/services/chain/protocol.privnet.yml
wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json
local_wallet_config_path: ./TemporaryDir/password-other.yml
local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json
wallet_password: "one"
endpoint_internal0: http://main-chain.frostfs.devenv:30333
- name: coredns_01
- name: coredns01
attributes:
container_name: coredns
clis:

1
.gitignore vendored
View file

@ -18,7 +18,6 @@ xunit_results.xml
# ignore caches under any path
**/__pycache__
**/.pytest_cache
*.egg-info
# ignore work directories and setup files
.setup

View file

@ -3,8 +3,8 @@
First, thank you for contributing! We love and encourage pull requests from
everyone. Please follow the guidelines:
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-testcases/issues) and
[pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-testcases/pulls) for existing
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-testcases/issues) and
[pull requests](https://github.com/TrueCloudLab/frostfs-testcases/pulls) for existing
discussions.
- Open an issue first, to discuss a new feature or enhancement.
@ -26,9 +26,9 @@ Start by forking the `frostfs-testcases` repository, make changes in a branch an
send a pull request. We encourage pull requests to discuss code changes. Here
are the steps in details:
### Set up your Git Repository
### Set up your GitHub Repository
Fork [FrosfFS testcases upstream](https://git.frostfs.info/TrueCloudLab/frostfs-testcases/forks) source
Fork [FrosfFS testcases upstream](https://github.com/TrueCloudLab/frostfs-testcases/fork) source
repository to your own personal repository. Copy the URL of your fork and clone it:
```shell
@ -38,7 +38,7 @@ $ git clone <url of your fork>
### Set up git remote as ``upstream``
```sh
$ cd frostfs-testcases
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-testcases
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-testcases
$ git fetch upstream
```
@ -94,8 +94,8 @@ $ git push origin feature/123-something_awesome
### Create a Pull Request
Pull requests can be created via Git. Refer to [this
document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for
Pull requests can be created via GitHub. Refer to [this
document](https://help.github.com/articles/creating-a-pull-request/) for
detailed steps on how to create a pull request. After a Pull Request gets peer
reviewed and approved, it will be merged.

View file

@ -9,27 +9,27 @@ These tests rely on resources and utility modules that have been originally deve
### Initial preparation
1. Install frostfs-cli
- `git clone git@git.frostfs.info:TrueCloudLab/frostfs-node.git`
- `git clone git@github.com:TrueCloudLab/frostfs-node.git`
- `cd frostfs-node`
- `make`
- `sudo cp bin/frostfs-cli /usr/local/bin/frostfs-cli`
2. Install frostfs-s3-authmate
- `git clone git@git.frostfs.info:TrueCloudLab/frostfs-s3-gw.git`
- `git clone git@github.com:TrueCloudLab/frostfs-s3-gw.git`
- `cd frostfs-s3-gw`
- `make`
- `sudo cp bin/frostfs-s3-authmate /usr/local/bin/frostfs-s3-authmate`
3. Install neo-go
- `git clone git@git.frostfs.info:TrueCloudLab/neo-go.git`
- `git clone git@github.com:nspcc-dev/neo-go.git`
- `cd neo-go`
- `git checkout v0.101.0` (or the current version in the frostfs-dev-env)
- `make`
- `sudo cp bin/neo-go /usr/local/bin/neo-go`
or download binary from releases: https://git.frostfs.info/TrueCloudLab/neo-go/releases
or download binary from releases: https://github.com/nspcc-dev/neo-go/releases
4. Clone frostfs-dev-env
`git clone git@git.frostfs.info:TrueCloudLab/frostfs-dev-env.git`
`git clone git@github.com:TrueCloudLab/frostfs-dev-env.git`
Note that we expect frostfs-dev-env to be located under
the `<testcases_root_dir>/../frostfs-dev-env` directory. If you put this repo in any other place,
@ -44,7 +44,7 @@ python3.10-dev
libssl-dev
```
As we use frostfs-dev-env, you'll also need to install
[prerequisites](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env#prerequisites) of this repository.
[prerequisites](https://github.com/TrueCloudLab/frostfs-dev-env#prerequisites) of this repository.
6. Prepare virtualenv

View file

@ -1,8 +1,8 @@
[tool.isort]
profile = "black"
src_paths = ["pytest_tests"]
line_length = 140
line_length = 120
[tool.black]
line-length = 140
line-length = 120
target-version = ["py310"]

View file

@ -3,7 +3,6 @@ from typing import List, Optional
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.acl import EACLOperation
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from pytest_tests.helpers.object_access import (
can_delete_object,
@ -17,47 +16,57 @@ from pytest_tests.helpers.object_access import (
def check_full_access_to_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
endpoint = cluster.default_rpc_endpoint
assert can_put_object(wallet, cid, file_name, shell, cluster, bearer, xhdr)
assert can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_get_range_hash_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_search_object(wallet, cid, shell, endpoint, oid, bearer, xhdr)
assert can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, xhdr)
assert can_delete_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_put_object(wallet, cid, file_name, shell, cluster, bearer, wallet_config, xhdr)
assert can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
assert can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
assert can_get_range_hash_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
assert can_search_object(wallet, cid, shell, endpoint, oid, bearer, wallet_config, xhdr)
assert can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, wallet_config, xhdr)
assert can_delete_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
def check_no_access_to_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
endpoint = cluster.default_rpc_endpoint
assert not can_put_object(wallet, cid, file_name, shell, cluster, bearer, xhdr)
assert not can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_get_range_hash_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_search_object(wallet, cid, shell, endpoint, oid, bearer, xhdr)
assert not can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, xhdr)
assert not can_delete_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_put_object(wallet, cid, file_name, shell, cluster, bearer, wallet_config, xhdr)
assert not can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
assert not can_get_range_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
assert not can_get_range_hash_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
assert not can_search_object(wallet, cid, shell, endpoint, oid, bearer, wallet_config, xhdr)
assert not can_get_object(
wallet, cid, oid, file_name, shell, cluster, bearer, wallet_config, xhdr
)
assert not can_delete_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
def check_custom_access_to_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
@ -66,6 +75,7 @@ def check_custom_access_to_container(
deny_operations: Optional[List[EACLOperation]] = None,
ignore_operations: Optional[List[EACLOperation]] = None,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
endpoint = cluster.default_rpc_endpoint
@ -73,39 +83,56 @@ def check_custom_access_to_container(
ignore_operations = [op.value for op in ignore_operations or []]
checks: dict = {}
if EACLOperation.PUT.value not in ignore_operations:
checks[EACLOperation.PUT.value] = can_put_object(wallet, cid, file_name, shell, cluster, bearer, xhdr)
checks[EACLOperation.PUT.value] = can_put_object(
wallet, cid, file_name, shell, cluster, bearer, wallet_config, xhdr
)
if EACLOperation.HEAD.value not in ignore_operations:
checks[EACLOperation.HEAD.value] = can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
checks[EACLOperation.HEAD.value] = can_get_head_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
if EACLOperation.GET_RANGE.value not in ignore_operations:
checks[EACLOperation.GET_RANGE.value] = can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
checks[EACLOperation.GET_RANGE.value] = can_get_range_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
if EACLOperation.GET_RANGE_HASH.value not in ignore_operations:
checks[EACLOperation.GET_RANGE_HASH.value] = can_get_range_hash_of_object(
wallet, cid, oid, shell, endpoint, bearer, xhdr
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
if EACLOperation.SEARCH.value not in ignore_operations:
checks[EACLOperation.SEARCH.value] = can_search_object(wallet, cid, shell, endpoint, oid, bearer, xhdr)
checks[EACLOperation.SEARCH.value] = can_search_object(
wallet, cid, shell, endpoint, oid, bearer, wallet_config, xhdr
)
if EACLOperation.GET.value not in ignore_operations:
checks[EACLOperation.GET.value] = can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, xhdr)
checks[EACLOperation.GET.value] = can_get_object(
wallet, cid, oid, file_name, shell, cluster, bearer, wallet_config, xhdr
)
if EACLOperation.DELETE.value not in ignore_operations:
checks[EACLOperation.DELETE.value] = can_delete_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
checks[EACLOperation.DELETE.value] = can_delete_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
failed_checks = [
f"allowed {action} failed"
for action, success in checks.items()
if not success and action not in deny_operations
] + [f"denied {action} succeeded" for action, success in checks.items() if success and action in deny_operations]
] + [
f"denied {action} succeeded"
for action, success in checks.items()
if success and action in deny_operations
]
assert not failed_checks, ", ".join(failed_checks)
def check_read_only_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
return check_custom_access_to_container(
@ -115,6 +142,7 @@ def check_read_only_container(
file_name,
deny_operations=[EACLOperation.PUT, EACLOperation.DELETE],
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
cluster=cluster,

View file

@ -1,6 +1,6 @@
from typing import Optional
from frostfs_testlib import reporter
import allure
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED
from frostfs_testlib.shell import Shell
@ -14,7 +14,6 @@ from frostfs_testlib.steps.cli.object import (
search_object,
)
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import string_utils
from frostfs_testlib.utils.file_utils import get_file_hash
@ -22,22 +21,24 @@ OPERATION_ERROR_TYPE = RuntimeError
def can_get_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
) -> bool:
with reporter.step("Try get object from container"):
with allure.step("Try get object from container"):
try:
got_file_path = get_object_from_random_node(
wallet,
cid,
oid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
cluster=cluster,
@ -52,22 +53,24 @@ def can_get_object(
def can_put_object(
wallet: WalletInfo,
wallet: str,
cid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
attributes: Optional[dict] = None,
) -> bool:
with reporter.step("Try put object to container"):
with allure.step("Try put object to container"):
try:
put_object_to_random_node(
wallet,
file_name,
cid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
attributes=attributes,
shell=shell,
@ -82,21 +85,23 @@ def can_put_object(
def can_delete_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
) -> bool:
with reporter.step("Try delete object from container"):
with allure.step("Try delete object from container"):
try:
delete_object(
wallet,
cid,
oid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -110,22 +115,24 @@ def can_delete_object(
def can_get_head_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
with reporter.step("Try get head of object"):
with allure.step("Try get head of object"):
try:
head_object(
wallet,
cid,
oid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -140,16 +147,17 @@ def can_get_head_object(
def can_get_range_of_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
with reporter.step("Try get range of object"):
with allure.step("Try get range of object"):
try:
get_range(
wallet,
@ -157,6 +165,7 @@ def can_get_range_of_object(
oid,
bearer=bearer,
range_cut="0:10",
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -171,16 +180,17 @@ def can_get_range_of_object(
def can_get_range_hash_of_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
with reporter.step("Try get range hash of object"):
with allure.step("Try get range hash of object"):
try:
get_range_hash(
wallet,
@ -188,6 +198,7 @@ def can_get_range_hash_of_object(
oid,
bearer=bearer,
range_cut="0:10",
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -202,21 +213,23 @@ def can_get_range_hash_of_object(
def can_search_object(
wallet: WalletInfo,
wallet: str,
cid: str,
shell: Shell,
endpoint: str,
oid: Optional[str] = None,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
with reporter.step("Try search object in container"):
with allure.step("Try search object in container"):
try:
oids = search_object(
wallet,
cid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,

View file

@ -1,6 +1,6 @@
import time
from frostfs_testlib import reporter
import allure
from frostfs_testlib.resources.common import STORAGE_GC_TIME
from frostfs_testlib.utils import datetime_utils
@ -33,5 +33,5 @@ def placement_policy_from_container(container_info: str) -> str:
def wait_for_gc_pass_on_storage_nodes() -> None:
wait_time = datetime_utils.parse_time(STORAGE_GC_TIME)
with reporter.step(f"Wait {wait_time}s until GC completes on storage nodes"):
with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"):
time.sleep(wait_time)

View file

@ -11,10 +11,8 @@ markers =
sanity: test runs in sanity testrun
smoke: test runs in smoke testrun
# controlling markers
order: manual control of test order
logs_after_session: Make the last test in session
no_healthcheck: skip healthcheck for this test
# functional markers
maintenance: tests for change mode node
container: tests for container creation
grpc_api: standard gRPC API tests
grpc_control: tests related to using frostfs-cli control commands
@ -45,25 +43,7 @@ markers =
failover_network: tests for network failure
failover_reboot: tests for system recovery after reboot of a node
interfaces: tests down interface to system
add_nodes: add nodes to cluster
check_binaries: check frostfs installed binaries versions
payments: tests for payment associated operations
load: performance tests
simple: tests with simple characteristics
complex: tests with complex characteristics
aws: AWS related tests
boto3: tests using the boto3
policy: policy tests
failover_baremetal: failover tests on hardware (baremetal)
failover_server: server failover tests
failover_storage: storage failover tests
failover_empty_map: failover tests for an empty map
failover_empty_map_offlne: offline failover tests for an empty map
failover_empty_map_stop_service: failover tests for stopped empty map service
failover_data_loss: failover tests in case of data loss
metabase_loss: tests for metadata loss
write_cache_loss: tests for write cache loss
time: time tests
replication: replication tests
ec_replication: replication EC
static_session_container: tests for a static session in a container
shard: shard management tests

View file

@ -1,10 +1,11 @@
import os
import uuid
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
@ -12,39 +13,57 @@ from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.acl import EACLRole
from frostfs_testlib.storage.dataclasses.frostfs_services import InnerRing, StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import generate_file
OBJECT_COUNT = 5
@dataclass
class Wallets:
wallets: dict[EACLRole, list[WalletInfo]]
class Wallet:
wallet_path: Optional[str] = None
config_path: Optional[str] = None
def get_wallet(self, role: EACLRole = EACLRole.USER) -> WalletInfo:
@dataclass
class Wallets:
wallets: dict[EACLRole, list[Wallet]]
def get_wallet(self, role: EACLRole = EACLRole.USER) -> Wallet:
return self.wallets[role][0]
def get_wallets_list(self, role: EACLRole = EACLRole.USER) -> list[WalletInfo]:
def get_wallets_list(self, role: EACLRole = EACLRole.USER) -> list[Wallet]:
return self.wallets[role]
@pytest.fixture(scope="module")
def wallets(default_wallet: WalletInfo, credentials_provider: CredentialsProvider, cluster: Cluster) -> Wallets:
other_wallets: list = []
for _ in range(2):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
other_wallets.append(credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0]))
def wallets(default_wallet: str, temp_directory: str, cluster: Cluster) -> Wallets:
other_wallets_paths = [
os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2)
]
for other_wallet_path in other_wallets_paths:
wallet_utils.init_wallet(other_wallet_path, DEFAULT_WALLET_PASS)
ir_node: InnerRing = cluster.ir_nodes[0]
storage_node: StorageNode = cluster.storage_nodes[0]
ir_wallet_path = ir_node.get_wallet_path()
ir_wallet_config = ir_node.get_wallet_config_path()
storage_wallet_path = storage_node.get_wallet_path()
storage_wallet_config = storage_node.get_wallet_config_path()
wallets_collection = Wallets(
wallets={
EACLRole.USER: [default_wallet],
EACLRole.OTHERS: other_wallets,
EACLRole.USER: [Wallet(wallet_path=default_wallet, config_path=DEFAULT_WALLET_CONFIG)],
EACLRole.OTHERS: [
Wallet(wallet_path=other_wallet_path, config_path=DEFAULT_WALLET_CONFIG)
for other_wallet_path in other_wallets_paths
],
EACLRole.SYSTEM: [
WalletInfo.from_node(ir_node),
WalletInfo.from_node(storage_node),
Wallet(wallet_path=ir_wallet_path, config_path=ir_wallet_config),
Wallet(wallet_path=storage_wallet_path, config_path=storage_wallet_config),
],
}
)
@ -53,28 +72,37 @@ def wallets(default_wallet: WalletInfo, credentials_provider: CredentialsProvide
if role == EACLRole.SYSTEM:
continue
for wallet in wallets:
reporter.attach(wallet.path, os.path.basename(wallet.path))
allure.attach.file(
wallet.wallet_path,
os.path.basename(wallet.wallet_path),
allure.attachment_type.JSON,
)
return wallets_collection
@pytest.fixture()
def file_path(object_size: ObjectSize) -> str:
yield generate_file(object_size.value)
@pytest.fixture(scope="function")
def eacl_container_with_objects(
wallets: Wallets, client_shell: Shell, cluster: Cluster, file_path: str
) -> tuple[str, list[str], str]:
user_wallet = wallets.get_wallet()
with reporter.step("Create eACL public container"):
with allure.step("Create eACL public container"):
cid = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PUBLIC_ACL,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
)
with reporter.step("Add test objects to container"):
with allure.step("Add test objects to container"):
objects_oids = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
attributes={"key1": "val1", "key": val, "key2": "abc"},
@ -86,5 +114,5 @@ def eacl_container_with_objects(
yield cid, objects_oids, file_path
# with reporter.step('Delete eACL public container'):
# with allure.step('Delete eACL public container'):
# delete_container(user_wallet, cid)

View file

@ -1,6 +1,5 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
@ -22,11 +21,11 @@ from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.acl_basic
class TestACLBasic(ClusterTestBase):
@pytest.fixture(scope="function")
def public_container(self, wallets: Wallets):
def public_container(self, wallets):
user_wallet = wallets.get_wallet()
with reporter.step("Create public container"):
with allure.step("Create public container"):
cid_public = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PUBLIC_ACL_F,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -34,15 +33,15 @@ class TestACLBasic(ClusterTestBase):
yield cid_public
# with reporter.step('Delete public container'):
# delete_container(user_wallet, cid_public)
# with allure.step('Delete public container'):
# delete_container(user_wallet.wallet_path, cid_public)
@pytest.fixture(scope="function")
def private_container(self, wallets: Wallets):
user_wallet = wallets.get_wallet()
with reporter.step("Create private container"):
with allure.step("Create private container"):
cid_private = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PRIVATE_ACL_F,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -50,15 +49,15 @@ class TestACLBasic(ClusterTestBase):
yield cid_private
# with reporter.step('Delete private container'):
# delete_container(user_wallet, cid_private)
# with allure.step('Delete private container'):
# delete_container(user_wallet.wallet_path, cid_private)
@pytest.fixture(scope="function")
def read_only_container(self, wallets: Wallets):
user_wallet = wallets.get_wallet()
with reporter.step("Create public readonly container"):
with allure.step("Create public readonly container"):
cid_read_only = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=READONLY_ACL_F,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -66,8 +65,8 @@ class TestACLBasic(ClusterTestBase):
yield cid_read_only
# with reporter.step('Delete public readonly container'):
# delete_container(user_wallet, cid_read_only)
# with allure.step('Delete public readonly container'):
# delete_container(user_wallet.wallet_path, cid_read_only)
@allure.title("Operations with basic ACL on public container (obj_size={object_size})")
def test_basic_acl_public(self, wallets: Wallets, public_container: str, file_path: str):
@ -78,11 +77,11 @@ class TestACLBasic(ClusterTestBase):
other_wallet = wallets.get_wallet(role=EACLRole.OTHERS)
cid = public_container
for wallet, desc in ((user_wallet, "owner"), (other_wallet, "other users")):
with reporter.step("Add test objects to container"):
with allure.step("Add test objects to container"):
# We create new objects for each wallet because check_full_access_to_container
# deletes the object
owner_object_oid = put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -90,16 +89,16 @@ class TestACLBasic(ClusterTestBase):
attributes={"created": "owner"},
)
other_object_oid = put_object_to_random_node(
other_wallet,
other_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
cluster=self.cluster,
attributes={"created": "other"},
)
with reporter.step(f"Check {desc} has full access to public container"):
with allure.step(f"Check {desc} has full access to public container"):
check_full_access_to_container(
wallet,
wallet.wallet_path,
cid,
owner_object_oid,
file_path,
@ -107,7 +106,7 @@ class TestACLBasic(ClusterTestBase):
cluster=self.cluster,
)
check_full_access_to_container(
wallet,
wallet.wallet_path,
cid,
other_object_oid,
file_path,
@ -123,17 +122,36 @@ class TestACLBasic(ClusterTestBase):
user_wallet = wallets.get_wallet()
other_wallet = wallets.get_wallet(role=EACLRole.OTHERS)
cid = private_container
with reporter.step("Add test objects to container"):
owner_object_oid = put_object_to_random_node(user_wallet, file_path, cid, self.shell, self.cluster)
with allure.step("Add test objects to container"):
owner_object_oid = put_object_to_random_node(
user_wallet.wallet_path, file_path, cid, shell=self.shell, cluster=self.cluster
)
with reporter.step("Check no one except owner has access to operations with container"):
check_no_access_to_container(other_wallet, cid, owner_object_oid, file_path, self.shell, self.cluster)
with allure.step("Check only owner has full access to private container"):
with allure.step("Check no one except owner has access to operations with container"):
check_no_access_to_container(
other_wallet.wallet_path,
cid,
owner_object_oid,
file_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step("Check owner has full access to private container"):
check_full_access_to_container(user_wallet, cid, owner_object_oid, file_path, self.shell, self.cluster)
with allure.step("Check owner has full access to private container"):
check_full_access_to_container(
user_wallet.wallet_path,
cid,
owner_object_oid,
file_path,
shell=self.shell,
cluster=self.cluster,
)
@allure.title("Operations with basic ACL on READONLY container (obj_size={object_size})")
def test_basic_acl_readonly(self, wallets: Wallets, client_shell: Shell, read_only_container: str, file_path: str):
def test_basic_acl_readonly(
self, wallets: Wallets, client_shell: Shell, read_only_container: str, file_path: str
):
"""
Test basic ACL Operations for Read-Only Container.
"""
@ -141,11 +159,27 @@ class TestACLBasic(ClusterTestBase):
other_wallet = wallets.get_wallet(role=EACLRole.OTHERS)
cid = read_only_container
with reporter.step("Add test objects to container"):
object_oid = put_object_to_random_node(user_wallet, file_path, cid, client_shell, self.cluster)
with allure.step("Add test objects to container"):
object_oid = put_object_to_random_node(
user_wallet.wallet_path, file_path, cid, shell=client_shell, cluster=self.cluster
)
with reporter.step("Check other has read-only access to operations with container"):
check_read_only_container(other_wallet, cid, object_oid, file_path, client_shell, self.cluster)
with allure.step("Check other has read-only access to operations with container"):
check_read_only_container(
other_wallet.wallet_path,
cid,
object_oid,
file_path,
shell=client_shell,
cluster=self.cluster,
)
with reporter.step("Check owner has full access to public container"):
check_full_access_to_container(user_wallet, cid, object_oid, file_path, client_shell, self.cluster)
with allure.step("Check owner has full access to public container"):
check_full_access_to_container(
user_wallet.wallet_path,
cid,
object_oid,
file_path,
shell=client_shell,
cluster=self.cluster,
)

View file

@ -1,7 +1,11 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.acl import create_eacl, form_bearertoken_file, set_eacl, wait_for_cache_expired
from frostfs_testlib.steps.acl import (
create_eacl,
form_bearertoken_file,
set_eacl,
wait_for_cache_expired,
)
from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, EACLRole, EACLRule
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
@ -30,70 +34,87 @@ class TestACLBearer(ClusterTestBase):
deny_wallet = wallets.get_wallet(role)
endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Check {role.value} has full access to container without bearer token"):
with allure.step(f"Check {role.value} has full access to container without bearer token"):
check_full_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step(f"Set deny all operations for {role.value} via eACL"):
eacl = [EACLRule(access=EACLAccess.DENY, role=role, operation=op) for op in EACLOperation]
with allure.step(f"Set deny all operations for {role.value} via eACL"):
eacl = [
EACLRule(access=EACLAccess.DENY, role=role, operation=op) for op in EACLOperation
]
eacl_file = create_eacl(cid, eacl, shell=self.shell)
set_eacl(user_wallet, cid, eacl_file, shell=self.shell, endpoint=endpoint)
set_eacl(user_wallet.wallet_path, cid, eacl_file, shell=self.shell, endpoint=endpoint)
wait_for_cache_expired()
with reporter.step(f"Create bearer token for {role.value} with all operations allowed"):
with allure.step(f"Create bearer token for {role.value} with all operations allowed"):
bearer = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=role) for op in EACLOperation],
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=role)
for op in EACLOperation
],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step(f"Check {role.value} without token has no access to all operations with container"):
with allure.step(
f"Check {role.value} without token has no access to all operations with container"
):
check_no_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step(f"Check {role.value} with token has access to all operations with container"):
with allure.step(
f"Check {role.value} with token has access to all operations with container"
):
check_full_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
bearer=bearer,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step(f"Set allow all operations for {role.value} via eACL"):
eacl = [EACLRule(access=EACLAccess.ALLOW, role=role, operation=op) for op in EACLOperation]
with allure.step(f"Set allow all operations for {role.value} via eACL"):
eacl = [
EACLRule(access=EACLAccess.ALLOW, role=role, operation=op) for op in EACLOperation
]
eacl_file = create_eacl(cid, eacl, shell=self.shell)
set_eacl(user_wallet, cid, eacl_file, shell=self.shell, endpoint=endpoint)
set_eacl(user_wallet.wallet_path, cid, eacl_file, shell=self.shell, endpoint=endpoint)
wait_for_cache_expired()
with reporter.step(f"Check {role.value} without token has access to all operations with container"):
with allure.step(
f"Check {role.value} without token has access to all operations with container"
):
check_full_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
@allure.title("BearerToken for compound operations (obj_size={object_size})")
def test_bearer_token_compound_operations(self, wallets: Wallets, eacl_container_with_objects):
def test_bearer_token_compound_operations(self, wallets, eacl_container_with_objects):
endpoint = self.cluster.default_rpc_endpoint
cid, objects_oids, file_path = eacl_container_with_objects
user_wallet = wallets.get_wallet()
@ -124,15 +145,21 @@ class TestACLBearer(ClusterTestBase):
}
deny_map_with_bearer = {
EACLRole.USER: [op for op in deny_map[EACLRole.USER] if op not in bearer_map[EACLRole.USER]],
EACLRole.OTHERS: [op for op in deny_map[EACLRole.OTHERS] if op not in bearer_map[EACLRole.OTHERS]],
EACLRole.USER: [
op for op in deny_map[EACLRole.USER] if op not in bearer_map[EACLRole.USER]
],
EACLRole.OTHERS: [
op for op in deny_map[EACLRole.OTHERS] if op not in bearer_map[EACLRole.OTHERS]
],
}
eacl_deny = []
for role, operations in deny_map.items():
eacl_deny += [EACLRule(access=EACLAccess.DENY, role=role, operation=op) for op in operations]
eacl_deny += [
EACLRule(access=EACLAccess.DENY, role=role, operation=op) for op in operations
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
eacl_table_path=create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -140,29 +167,31 @@ class TestACLBearer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step("Check rule consistency without bearer"):
with allure.step("Check rule consistency without bearer"):
check_custom_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map[EACLRole.USER],
wallet_config=user_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
check_custom_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map[EACLRole.OTHERS],
wallet_config=other_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step("Check rule consistency using bearer token"):
with allure.step("Check rule consistency using bearer token"):
bearer_user = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.USER)
@ -173,7 +202,7 @@ class TestACLBearer(ClusterTestBase):
)
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
@ -184,22 +213,24 @@ class TestACLBearer(ClusterTestBase):
)
check_custom_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map_with_bearer[EACLRole.USER],
bearer=bearer_user,
wallet_config=user_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
check_custom_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map_with_bearer[EACLRole.OTHERS],
bearer=bearer_other,
wallet_config=other_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)

View file

@ -1,6 +1,5 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.acl import create_eacl, set_eacl, wait_for_cache_expired
from frostfs_testlib.steps.cli.container import create_container
@ -27,22 +26,24 @@ from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.acl_extended
class TestEACLContainer(ClusterTestBase):
@pytest.fixture(scope="function")
def eacl_full_placement_container_with_object(self, wallets: Wallets, file_path: str) -> tuple[str, str, str]:
def eacl_full_placement_container_with_object(self, wallets: Wallets, file_path: str) -> str:
user_wallet = wallets.get_wallet()
storage_nodes = self.cluster.storage_nodes
node_count = len(storage_nodes)
with reporter.step("Create eACL public container with full placement rule"):
with allure.step("Create eACL public container with full placement rule"):
full_placement_rule = f"REP {node_count} IN X CBF 1 SELECT {node_count} FROM * AS X"
cid = create_container(
wallet=user_wallet,
wallet=user_wallet.wallet_path,
rule=full_placement_rule,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Add test object to container"):
oid = put_object_to_random_node(user_wallet, file_path, cid, shell=self.shell, cluster=self.cluster)
with allure.step("Add test object to container"):
oid = put_object_to_random_node(
user_wallet.wallet_path, file_path, cid, shell=self.shell, cluster=self.cluster
)
wait_object_replication(
cid,
oid,
@ -70,10 +71,10 @@ class TestEACLContainer(ClusterTestBase):
not_deny_role_str = "user" if deny_role == EACLRole.OTHERS else "all others"
cid, object_oids, file_path = eacl_container_with_objects
with reporter.step(f"Deny all operations for {deny_role_str} via eACL"):
with allure.step(f"Deny all operations for {deny_role_str} via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -81,10 +82,10 @@ class TestEACLContainer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step(f"Check only {not_deny_role_str} has full access to container"):
with reporter.step(f"Check {deny_role_str} has not access to any operations with container"):
with allure.step(f"Check only {not_deny_role_str} has full access to container"):
with allure.step(f"Check {deny_role_str} has not access to any operations with container"):
check_no_access_to_container(
deny_role_wallet,
deny_role_wallet.wallet_path,
cid,
object_oids[0],
file_path,
@ -92,9 +93,9 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step(f"Check {not_deny_role_str} has full access to eACL public container"):
with allure.step(f"Check {not_deny_role_wallet} has full access to eACL public container"):
check_full_access_to_container(
not_deny_role_wallet,
not_deny_role_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -102,10 +103,10 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step(f"Allow all operations for {deny_role_str} via eACL"):
with allure.step(f"Allow all operations for {deny_role_str} via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -113,9 +114,9 @@ class TestEACLContainer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step("Check all have full access to eACL public container"):
with allure.step("Check all have full access to eACL public container"):
check_full_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -123,7 +124,7 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -139,18 +140,18 @@ class TestEACLContainer(ClusterTestBase):
other_wallet, other_wallet_allow = wallets.get_wallets_list(EACLRole.OTHERS)[0:2]
cid, object_oids, file_path = eacl_container_with_objects
with reporter.step("Deny all operations for others except single wallet via eACL"):
with allure.step("Deny all operations for others except single wallet via eACL"):
eacl = [
EACLRule(
access=EACLAccess.ALLOW,
role=other_wallet_allow,
role=other_wallet_allow.wallet_path,
operation=op,
)
for op in EACLOperation
]
eacl += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl, shell=self.shell),
shell=self.shell,
@ -158,10 +159,10 @@ class TestEACLContainer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step("Check only owner and allowed other have full access to public container"):
with reporter.step("Check other has not access to operations with container"):
with allure.step("Check only owner and allowed other have full access to public container"):
with allure.step("Check other has not access to operations with container"):
check_no_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
object_oids[0],
file_path,
@ -169,9 +170,9 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step("Check owner has full access to public container"):
with allure.step("Check owner has full access to public container"):
check_full_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -179,9 +180,9 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step("Check allowed other has full access to public container"):
with allure.step("Check allowed other has full access to public container"):
check_full_access_to_container(
other_wallet_allow,
other_wallet_allow.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -200,11 +201,11 @@ class TestEACLContainer(ClusterTestBase):
storage_nodes = self.cluster.storage_nodes
storage_node = self.cluster.storage_nodes[0]
with reporter.step("Deny all operations for user via eACL"):
with allure.step("Deny all operations for user via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op) for op in EACLOperation]
eacl_deny += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -212,11 +213,11 @@ class TestEACLContainer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step("Drop object to check replication"):
with allure.step("Drop object to check replication"):
drop_object(storage_node, cid=cid, oid=oid)
storage_wallet_path = storage_node.get_wallet_path()
with reporter.step("Wait for dropped object replicated"):
with allure.step("Wait for dropped object replicated"):
wait_object_replication(
cid,
oid,
@ -233,122 +234,136 @@ class TestEACLContainer(ClusterTestBase):
cid, object_oids, file_path = eacl_container_with_objects
endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Check IR and STORAGE rules compliance"):
with allure.step("Check IR and STORAGE rules compliance"):
assert not can_put_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_put_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
object_oids[0],
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_get_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
object_oids[0],
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_head_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_head_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_search_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=ir_wallet.config_path,
)
assert can_search_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with reporter.step("Deny all operations for SYSTEM via eACL"):
with allure.step("Deny all operations for SYSTEM via eACL"):
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(
cid=cid,
@ -362,129 +377,143 @@ class TestEACLContainer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step("Check IR and STORAGE rules compliance with deny eACL"):
with allure.step("Check IR and STORAGE rules compliance with deny eACL"):
assert not can_put_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert not can_put_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_head_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_head_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_search_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_search_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_hash_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_hash_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with reporter.step("Allow all operations for SYSTEM via eACL"):
with allure.step("Allow all operations for SYSTEM via eACL"):
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(
cid=cid,
@ -498,115 +527,129 @@ class TestEACLContainer(ClusterTestBase):
)
wait_for_cache_expired()
with reporter.step("Check IR and STORAGE rules compliance with allow eACL"):
with allure.step("Check IR and STORAGE rules compliance with allow eACL"):
assert not can_put_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_put_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_get_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_head_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_head_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_search_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
shell=self.shell,
oid=object_oids[0],
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_search_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
shell=self.shell,
oid=object_oids[0],
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)

View file

@ -1,6 +1,5 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.acl import create_eacl, form_bearertoken_file, set_eacl, wait_for_cache_expired
from frostfs_testlib.steps.cli.container import create_container, delete_container
@ -62,18 +61,18 @@ class TestEACLFilters(ClusterTestBase):
@pytest.fixture(scope="function")
def eacl_container_with_objects(self, wallets: Wallets, file_path: str):
user_wallet = wallets.get_wallet()
with reporter.step("Create eACL public container"):
with allure.step("Create eACL public container"):
cid = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Add test objects to container"):
with allure.step("Add test objects to container"):
objects_with_header = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -85,7 +84,7 @@ class TestEACLFilters(ClusterTestBase):
objects_with_other_header = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -97,7 +96,7 @@ class TestEACLFilters(ClusterTestBase):
objects_without_header = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -108,9 +107,9 @@ class TestEACLFilters(ClusterTestBase):
yield cid, objects_with_header, objects_with_other_header, objects_without_header, file_path
with reporter.step("Delete eACL public container"):
with allure.step("Delete eACL public container"):
delete_container(
user_wallet,
user_wallet.wallet_path,
cid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -135,7 +134,7 @@ class TestEACLFilters(ClusterTestBase):
file_path,
) = eacl_container_with_objects
with reporter.step("Deny all operations for other with eACL request filter"):
with allure.step("Deny all operations for other with eACL request filter"):
equal_filter = EACLFilter(**self.REQ_EQUAL_FILTER.__dict__)
equal_filter.match_type = match_type
eacl_deny = [
@ -148,7 +147,7 @@ class TestEACLFilters(ClusterTestBase):
for op in EACLOperation
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -169,9 +168,9 @@ class TestEACLFilters(ClusterTestBase):
objects_with_other_header,
objects_without_header,
):
with reporter.step("Check other has full access when sending request without headers"):
with allure.step("Check other has full access when sending request without headers"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -179,9 +178,9 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step("Check other has full access when sending request with allowed headers"):
with allure.step("Check other has full access when sending request with allowed headers"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -190,9 +189,9 @@ class TestEACLFilters(ClusterTestBase):
xhdr=allow_headers,
)
with reporter.step("Check other has no access when sending request with denied headers"):
with allure.step("Check other has no access when sending request with denied headers"):
check_no_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -201,18 +200,18 @@ class TestEACLFilters(ClusterTestBase):
xhdr=deny_headers,
)
with reporter.step(
with allure.step(
"Check other has full access when sending request " "with denied headers and using bearer token"
):
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -240,7 +239,7 @@ class TestEACLFilters(ClusterTestBase):
file_path,
) = eacl_container_with_objects
with reporter.step("Deny all operations for other with object filter"):
with allure.step("Deny all operations for other with object filter"):
equal_filter = EACLFilter(**self.OBJ_EQUAL_FILTER.__dict__)
equal_filter.match_type = match_type
eacl_deny = [
@ -253,7 +252,7 @@ class TestEACLFilters(ClusterTestBase):
for op in self.OBJECT_ATTRIBUTES_FILTER_SUPPORTED_OPERATIONS
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -268,9 +267,9 @@ class TestEACLFilters(ClusterTestBase):
# but eACL rule should ignore request headers and validate
# only object headers
for xhdr in (self.ATTRIBUTE, self.OTHER_ATTRIBUTE, None):
with reporter.step("Check other have full access to objects without attributes"):
with allure.step("Check other have full access to objects without attributes"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
objs_without_header.pop(),
file_path,
@ -279,9 +278,9 @@ class TestEACLFilters(ClusterTestBase):
xhdr=xhdr,
)
with reporter.step("Check other have full access to objects without deny attribute"):
with allure.step("Check other have full access to objects without deny attribute"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
allow_objects.pop(),
file_path,
@ -290,10 +289,10 @@ class TestEACLFilters(ClusterTestBase):
xhdr=xhdr,
)
with reporter.step("Check other have no access to objects with deny attribute"):
with allure.step("Check other have no access to objects with deny attribute"):
with pytest.raises(AssertionError):
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
shell=self.shell,
@ -302,7 +301,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
file_path,
@ -311,9 +310,9 @@ class TestEACLFilters(ClusterTestBase):
xhdr=xhdr,
)
with reporter.step("Check other have access to objects with deny attribute and using bearer token"):
with allure.step("Check other have access to objects with deny attribute and using bearer token"):
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(
@ -327,7 +326,7 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects.pop(),
file_path,
@ -338,22 +337,22 @@ class TestEACLFilters(ClusterTestBase):
)
allow_attribute = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
with reporter.step("Check other can PUT objects without denied attribute"):
with allure.step("Check other can PUT objects without denied attribute"):
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
cluster=self.cluster,
attributes=allow_attribute,
)
assert can_put_object(other_wallet, cid, file_path, shell=self.shell, cluster=self.cluster)
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
deny_attribute = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
with reporter.step("Check other can not PUT objects with denied attribute"):
with allure.step("Check other can not PUT objects with denied attribute"):
with pytest.raises(AssertionError):
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -361,9 +360,9 @@ class TestEACLFilters(ClusterTestBase):
attributes=deny_attribute,
)
with reporter.step("Check other can PUT objects with denied attribute and using bearer token"):
with allure.step("Check other can PUT objects with denied attribute and using bearer token"):
bearer_other_for_put = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(
@ -376,7 +375,7 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -387,7 +386,6 @@ class TestEACLFilters(ClusterTestBase):
@allure.title("Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})")
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
@pytest.mark.parametrize("object_size", ["simple"], indirect=True)
def test_extended_acl_allow_filters_object(
self,
wallets: Wallets,
@ -404,7 +402,7 @@ class TestEACLFilters(ClusterTestBase):
file_path,
) = eacl_container_with_objects
with reporter.step("Deny all operations for others except few operations allowed by object filter"):
with allure.step("Deny all operations for others except few operations allowed by object filter"):
equal_filter = EACLFilter(**self.OBJ_EQUAL_FILTER.__dict__)
equal_filter.match_type = match_type
eacl = [
@ -420,7 +418,7 @@ class TestEACLFilters(ClusterTestBase):
for op in self.OBJECT_ATTRIBUTES_FILTER_SUPPORTED_OPERATIONS
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl, shell=self.shell),
shell=self.shell,
@ -439,11 +437,11 @@ class TestEACLFilters(ClusterTestBase):
allow_attribute = self.OTHER_ATTRIBUTE
deny_attribute = self.ATTRIBUTE
with reporter.step("Check other cannot get and put objects without attributes"):
with allure.step("Check other cannot get and put objects without attributes"):
oid = objects_without_header.pop()
with pytest.raises(AssertionError):
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
shell=self.shell,
@ -451,7 +449,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
file_path,
@ -459,11 +457,11 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
with pytest.raises(AssertionError):
assert can_put_object(other_wallet, cid, file_path, shell=self.shell, cluster=self.cluster)
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
with reporter.step("Check other can get and put objects without attributes and using bearer token"):
with allure.step("Check other can get and put objects without attributes and using bearer token"):
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(
@ -477,7 +475,7 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
objects_without_header[0],
shell=self.shell,
@ -485,7 +483,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
objects_without_header[0],
file_path,
@ -494,7 +492,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -502,17 +500,17 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
with reporter.step("Check other can get objects with attributes matching the filter"):
with allure.step("Check other can get objects with attributes matching the filter"):
oid = allow_objects.pop()
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
file_path,
@ -520,7 +518,7 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -528,10 +526,10 @@ class TestEACLFilters(ClusterTestBase):
attributes=allow_attribute,
)
with reporter.step("Check other cannot get objects without attributes matching the filter"):
with allure.step("Check other cannot get objects without attributes matching the filter"):
with pytest.raises(AssertionError):
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
shell=self.shell,
@ -539,7 +537,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
file_path,
@ -548,7 +546,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
attributes=deny_attribute,
@ -556,12 +554,12 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step(
with allure.step(
"Check other can get objects without attributes matching the filter " "and using bearer token"
):
oid = deny_objects.pop()
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
shell=self.shell,
@ -569,7 +567,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
file_path,
@ -578,7 +576,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,

View file

@ -1,108 +1,62 @@
import logging
import os
import random
import shutil
from datetime import datetime, timedelta, timezone
from importlib.metadata import entry_points
from typing import Optional
import allure
import pytest
import yaml
from dateutil import parser
from frostfs_testlib import plugins, reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting import Hosting
from frostfs_testlib.reporter import AllureHandler, StepsLogger
from frostfs_testlib.plugins import load_plugin
from frostfs_testlib.reporter import AllureHandler, get_reporter
from frostfs_testlib.resources.common import (
ASSETS_DIR,
COMPLEX_OBJECT_CHUNKS_COUNT,
COMPLEX_OBJECT_TAIL_SIZE,
DEFAULT_WALLET_PASS,
SIMPLE_OBJECT_SIZE,
)
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE
from frostfs_testlib.steps.cli.container import list_containers
from frostfs_testlib.steps.cli.object import get_netmap_netinfo
from frostfs_testlib.steps.node_management import storage_node_healthcheck
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage import get_service_registry
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils import env_utils, version_utils
from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.resources.common import HOSTING_CONFIG_FILE, TEST_CYCLES_COUNT
logger = logging.getLogger("NeoLogger")
SERVICE_ACTIVE_TIME = 20
# Add logs check test even if it's not fit to mark selectors
def pytest_configure(config: pytest.Config):
markers = config.option.markexpr
if markers != "" and "sanity" not in markers:
if markers != "":
config.option.markexpr = f"logs_after_session or ({markers})"
number_key = pytest.StashKey[str]()
start_time = pytest.StashKey[int]()
test_outcome = pytest.StashKey[str]()
# pytest hook. Do not rename
def pytest_collection_modifyitems(items: list[pytest.Item]):
# Change order of tests based on @pytest.mark.order(<int>) marker
def order(item: pytest.Item) -> int:
order_marker = item.get_closest_marker("order")
if order_marker and (len(order_marker.args) != 1 or not isinstance(order_marker.args[0], int)):
raise RuntimeError("Incorrect usage of pytest.mark.order")
# Make network tests last based on @pytest.mark.node_mgmt and logs_test to be latest
def priority(item: pytest.Item) -> int:
is_node_mgmt_test = 1 if item.get_closest_marker("node_mgmt") else 0
is_logs_check_test = 100 if item.get_closest_marker("logs_after_session") else 0
return is_node_mgmt_test + is_logs_check_test
order_value = order_marker.args[0] if order_marker else 0
return order_value
items.sort(key=lambda item: order(item))
items.sort(key=lambda item: priority(item))
# pytest hook. Do not rename
def pytest_collection_finish(session: pytest.Session):
items_total = len(session.items)
for number, item in enumerate(session.items, 1):
item.stash[number_key] = f"[{number}/{items_total}]"
item.stash[test_outcome] = ""
item.stash[start_time] = 0
# pytest hook. Do not rename
def pytest_runtest_setup(item: pytest.Item):
item.stash[start_time] = int(datetime.now().timestamp())
logger.info(f"STARTED {item.stash[number_key]}: {item.name}")
# pytest hook. Do not rename
def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo):
if call.excinfo is not None:
if call.excinfo.typename == "Skipped":
item.stash[start_time] = int(datetime.now().timestamp())
item.stash[test_outcome] += f"SKIPPED on {call.when}; "
else:
item.stash[test_outcome] += f"FAILED on {call.when}; "
if call.when == "teardown":
duration = int(datetime.now().timestamp()) - item.stash[start_time]
if not item.stash[test_outcome]:
outcome = "PASSED "
else:
outcome = item.stash[test_outcome]
logger.info(f"ENDED {item.stash[number_key]}: {item.name}: {outcome}(duration={duration}s)")
# pytest hook. Do not rename
def pytest_generate_tests(metafunc: pytest.Metafunc):
if (
TEST_CYCLES_COUNT <= 1
@ -121,17 +75,8 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
@pytest.fixture(scope="session")
def configure_testlib():
reporter.get_reporter().register_handler(AllureHandler())
reporter.get_reporter().register_handler(StepsLogger())
get_reporter().register_handler(AllureHandler())
logging.getLogger("paramiko").setLevel(logging.INFO)
# Register Services for cluster
registry = get_service_registry()
services = entry_points(group="frostfs.testlib.services")
for svc in services:
registry.register_service(svc.name, svc.load())
yield
@ -178,9 +123,9 @@ def require_multiple_interfaces(cluster: Cluster):
@pytest.fixture(scope="session")
def max_object_size(cluster: Cluster, client_shell: Shell) -> int:
storage_node = cluster.storage_nodes[0]
wallet = WalletInfo.from_node(storage_node)
net_info = get_netmap_netinfo(
wallet=wallet,
wallet=storage_node.get_wallet_path(),
wallet_config=storage_node.get_wallet_config_path(),
endpoint=storage_node.get_rpc_endpoint(),
shell=client_shell,
)
@ -193,11 +138,6 @@ def simple_object_size(max_object_size: int) -> ObjectSize:
return ObjectSize("simple", size)
@pytest.fixture()
def file_path(object_size: ObjectSize) -> str:
return generate_file(object_size.value)
@pytest.fixture(scope="session")
def complex_object_size(max_object_size: int) -> ObjectSize:
size = max_object_size * int(COMPLEX_OBJECT_CHUNKS_COUNT) + int(COMPLEX_OBJECT_TAIL_SIZE)
@ -220,28 +160,8 @@ def object_size(
@pytest.fixture(scope="session")
def rep_placement_policy() -> PlacementPolicy:
return PlacementPolicy("rep", DEFAULT_PLACEMENT_RULE)
@pytest.fixture(scope="session")
def ec_placement_policy() -> PlacementPolicy:
return PlacementPolicy("ec", DEFAULT_EC_PLACEMENT_RULE)
# By default we want all tests to be executed with both storage policies.
# This can be overriden in choosen tests if needed.
@pytest.fixture(
scope="session",
params=[pytest.param("rep", marks=pytest.mark.rep), pytest.param("ec", marks=pytest.mark.ec)],
)
def placement_policy(
rep_placement_policy: PlacementPolicy, ec_placement_policy: PlacementPolicy, request: pytest.FixtureRequest
) -> PlacementPolicy:
if request.param == "rep":
return rep_placement_policy
return ec_placement_policy
def wallet_factory(temp_directory: str, client_shell: Shell, cluster: Cluster) -> WalletFactory:
return WalletFactory(temp_directory, client_shell, cluster)
@pytest.fixture(scope="session")
@ -256,8 +176,8 @@ def cluster(temp_directory: str, hosting: Hosting, client_shell: Shell) -> Clust
yield cluster
@allure.title("[Session]: Provide S3 policy")
@pytest.fixture(scope="session")
@allure.step("[Class]: Provide S3 policy")
@pytest.fixture(scope="class")
def s3_policy(request: pytest.FixtureRequest):
policy = None
if "param" in request.__dict__:
@ -269,7 +189,7 @@ def s3_policy(request: pytest.FixtureRequest):
@pytest.fixture(scope="session")
@allure.title("[Session] Create healthcheck object")
def healthcheck(cluster: Cluster) -> Healthcheck:
healthcheck_cls = plugins.load_plugin(
healthcheck_cls = load_plugin(
"frostfs.testlib.healthcheck", cluster.cluster_nodes[0].host.config.healthcheck_plugin_name
)
@ -277,48 +197,43 @@ def healthcheck(cluster: Cluster) -> Healthcheck:
@pytest.fixture(scope="session")
def cluster_state_controller_session(
client_shell: Shell, cluster: Cluster, healthcheck: Healthcheck
) -> ClusterStateController:
def cluster_state_controller(client_shell: Shell, cluster: Cluster, healthcheck: Healthcheck) -> ClusterStateController:
controller = ClusterStateController(client_shell, cluster, healthcheck)
return controller
yield controller
@pytest.fixture
def cluster_state_controller(cluster_state_controller_session: ClusterStateController) -> ClusterStateController:
yield cluster_state_controller_session
cluster_state_controller_session.start_stopped_hosts()
cluster_state_controller_session.start_all_stopped_services()
@pytest.fixture(scope="session")
def credentials_provider(cluster: Cluster) -> CredentialsProvider:
return CredentialsProvider(cluster)
@allure.title("[Session]: Create S3 client")
@allure.step("[Class]: Create S3 client")
@pytest.fixture(
scope="session",
scope="class",
params=[
pytest.param(AwsCliClient, marks=[pytest.mark.aws, pytest.mark.weekly]),
pytest.param(Boto3ClientWrapper, marks=[pytest.mark.boto3, pytest.mark.nightly]),
pytest.param(AwsCliClient, marks=pytest.mark.aws),
pytest.param(Boto3ClientWrapper, marks=pytest.mark.boto3),
],
)
def s3_client(
default_user: User,
default_wallet: str,
client_shell: Shell,
s3_policy: Optional[str],
cluster: Cluster,
auth_container_placement_policy: str,
request: pytest.FixtureRequest,
credentials_provider: CredentialsProvider,
) -> S3ClientWrapper:
node = cluster.cluster_nodes[0]
credentials_provider.S3.provide(default_user, node, s3_policy)
wallet = WalletInfo(path=default_wallet, password=DEFAULT_WALLET_PASS)
(cid, access_key_id, secret_access_key) = s3_helper.init_s3_credentials(
wallet,
client_shell,
cluster,
s3gates=[cluster_node.s3_gate for cluster_node in cluster.cluster_nodes],
policy=s3_policy,
container_placement_policy=auth_container_placement_policy,
)
containers_list = list_containers(wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
s3_client_cls = request.param
client = s3_client_cls(
default_user.s3_credentials.access_key, default_user.s3_credentials.secret_key, cluster.default_s3_gate_endpoint
)
return client
client = s3_client_cls(access_key_id, secret_access_key, cluster.default_s3_gate_endpoint)
yield client
@pytest.fixture
@ -329,77 +244,40 @@ def versioning_status(request: pytest.FixtureRequest) -> VersioningStatus:
return VersioningStatus.UNDEFINED
def unique_name(prefix: str) -> str:
return f"{prefix}{hex(int(datetime.now().timestamp() * 1000000))}"
@allure.title("[Session] Bulk create buckets for tests")
@pytest.fixture(scope="session")
def buckets_pool(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
test_buckets: list = []
s3_client_type = type(s3_client).__name__
for test in request.session.items:
if s3_client_type not in test.name:
continue
if "bucket" in test.fixturenames:
test_buckets.append(unique_name("bucket-"))
if "two_buckets" in test.fixturenames:
test_buckets.append(unique_name("bucket-"))
test_buckets.append(unique_name("bucket-"))
if test_buckets:
parallel(s3_client.create_bucket, test_buckets)
return test_buckets
@allure.title("[Test] Create bucket")
@allure.step("Create/delete bucket")
@pytest.fixture
def bucket(buckets_pool: list[str], s3_client: S3ClientWrapper, versioning_status: VersioningStatus):
if buckets_pool:
bucket_name = buckets_pool.pop()
else:
bucket_name = s3_client.create_bucket()
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus, request: pytest.FixtureRequest):
bucket_name = s3_client.create_bucket()
if versioning_status:
s3_helper.set_bucket_versioning(s3_client, bucket_name, versioning_status)
return bucket_name
yield bucket_name
if "sanity" not in request.config.option.markexpr:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
@allure.title("[Test] Create two buckets")
@allure.step("Create two buckets")
@pytest.fixture
def two_buckets(buckets_pool: list[str], s3_client: S3ClientWrapper) -> list[str]:
buckets: list[str] = []
def two_buckets(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
bucket_1 = s3_client.create_bucket()
bucket_2 = s3_client.create_bucket()
yield bucket_1, bucket_2
for _ in range(2):
if buckets_pool:
buckets.append(buckets_pool.pop())
else:
buckets.append(s3_client.create_bucket())
return buckets
if "sanity" not in request.config.option.markexpr:
for bucket_name in [bucket_1, bucket_2]:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
@allure.title("[Autouse/Session] Collect binary versions")
@allure.step("[Autouse/Session] Check binary versions")
@pytest.fixture(scope="session", autouse=True)
def collect_binary_versions(hosting: Hosting, client_shell: Shell, request: pytest.FixtureRequest):
def check_binary_versions(hosting: Hosting, client_shell: Shell, request: pytest.FixtureRequest):
local_versions = version_utils.get_local_binaries_versions(client_shell)
remote_versions = version_utils.get_remote_binaries_versions(hosting)
remote_versions_keys = list(remote_versions.keys())
all_versions = {
**local_versions,
**{
f"{name}_{remote_versions_keys.index(host) + 1:02d}": version
for host, versions in remote_versions.items()
for name, version in versions.items()
},
}
all_versions = {**local_versions, **remote_versions}
environment_dir = request.config.getoption("--alluredir")
if not environment_dir:
@ -409,23 +287,23 @@ def collect_binary_versions(hosting: Hosting, client_shell: Shell, request: pyte
env_utils.save_env_properties(file_path, all_versions)
@reporter.step("Prepare tmp directory")
@allure.step("Prepare tmp directory")
@pytest.fixture(scope="session")
def temp_directory(configure_testlib):
with reporter.step("Prepare tmp directory"):
def temp_directory():
with allure.step("Prepare tmp directory"):
full_path = os.path.join(os.getcwd(), ASSETS_DIR)
shutil.rmtree(full_path, ignore_errors=True)
os.mkdir(full_path)
yield full_path
with reporter.step("Remove tmp directory"):
with allure.step("Remove tmp directory"):
shutil.rmtree(full_path)
@reporter.step("[Autouse/Session] Test session start time")
@allure.step("[Autouse/Session] Test session start time")
@pytest.fixture(scope="session", autouse=True)
def session_start_time(configure_testlib):
def session_start_time():
start_time = datetime.utcnow()
return start_time
@ -433,25 +311,23 @@ def session_start_time(configure_testlib):
@allure.title("[Autouse/Session] After deploy healthcheck")
@pytest.fixture(scope="session", autouse=True)
def after_deploy_healthcheck(cluster: Cluster):
with reporter.step("Wait for cluster readiness after deploy"):
with allure.step("Wait for cluster readiness after deploy"):
parallel(readiness_on_node, cluster.cluster_nodes)
@wait_for_success(60 * SERVICE_ACTIVE_TIME * 3, 60, title="Wait for {cluster_node} readiness")
def readiness_on_node(cluster_node: ClusterNode):
if (
"skip_readiness_check" in cluster_node.host.config.attributes
and cluster_node.host.config.attributes["skip_readiness_check"]
):
return
SERVICE_ACTIVE_TIME = 20
@wait_for_success(60 * SERVICE_ACTIVE_TIME * 3, 60)
@allure.step("Check readiness on node {cluster_node}")
def readiness_on_node(cluster_node: ClusterNode):
# TODO: Move to healtcheck classes
svc_name = cluster_node.service(StorageNode).get_service_systemctl_name()
with reporter.step(f"Check service {svc_name} is active"):
with allure.step(f"Check service {svc_name} is active"):
result = cluster_node.host.get_shell().exec(f"systemctl is-active {svc_name}")
assert "active" == result.stdout.strip(), f"Service {svc_name} should be in active state"
with reporter.step(f"Check service {svc_name} is active more than {SERVICE_ACTIVE_TIME} minutes"):
with allure.step(f"Check service {svc_name} is active more than {SERVICE_ACTIVE_TIME} minutes"):
result = cluster_node.host.get_shell().exec(
f"systemctl show {svc_name} --property ActiveEnterTimestamp | cut -d '=' -f 2"
)
@ -467,27 +343,41 @@ def readiness_on_node(cluster_node: ClusterNode):
), f"Service should be in active state more than {SERVICE_ACTIVE_TIME} minutes, current {active_minutes}m:{active_seconds}s"
@reporter.step("Prepare default user with wallet")
@allure.title("[Autouse/Test] Run health check for all nodes")
@pytest.fixture(autouse=True)
def run_health_check(cluster: Cluster, request: pytest.FixtureRequest):
if request.node.get_closest_marker("no_healthcheck"):
# Skip healthcheck for tests marked with no_healthcheck
return
parallel(healthcheck_on_node, cluster.cluster_nodes)
@allure.title("Run health check for {cluster_node}")
def healthcheck_on_node(cluster_node: ClusterNode):
health_check = storage_node_healthcheck(cluster_node.storage_node)
assert (
health_check.health_status == "READY" and health_check.network_status == "ONLINE"
), f"Node {cluster_node} is not healthy"
@allure.step("Prepare wallet and deposit")
@pytest.fixture(scope="session")
def default_user(credentials_provider: CredentialsProvider, cluster: Cluster) -> User:
# always unique username
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
node = cluster.cluster_nodes[0]
credentials_provider.GRPC.provide(user, node)
return user
def default_wallet(wallet_factory: WalletFactory) -> str:
wallet = wallet_factory.create_wallet(password=DEFAULT_WALLET_PASS)
allure.attach.file(wallet.path, os.path.basename(wallet.path), allure.attachment_type.JSON)
return wallet.path
@reporter.step("Get wallet for default user")
@pytest.fixture(scope="session")
def default_wallet(default_user: User) -> WalletInfo:
return default_user.wallet
@pytest.fixture()
@allure.title("Select random node for testing")
def node_under_test(cluster: Cluster) -> ClusterNode:
selected_node = random.choice(cluster.cluster_nodes)
reporter.attach(f"{selected_node}", "Selected node")
return selected_node
@allure.step("[Class]: Container placement policy for keys")
@pytest.fixture(scope="class")
def auth_container_placement_policy(cluster: Cluster, request: pytest.FixtureRequest):
placeholders = {
"$ALPHABET_NODE_COUNT$": 4 if len(cluster.cluster_nodes) < 8 else 8,
"$NODE_COUNT$": len(cluster.cluster_nodes),
}
placement_policy = None
if "param" in request.__dict__:
placement_policy = request.param
for key, value in placeholders.items():
placement_policy = placement_policy.replace(key, str(value))
return placement_policy

View file

@ -1,6 +1,7 @@
import json
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PRIVATE_ACL_F
from frostfs_testlib.steps.cli.container import (
create_container,
@ -10,7 +11,6 @@ from frostfs_testlib.steps.cli.container import (
wait_for_container_creation,
wait_for_container_deletion,
)
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.utility import placement_policy_from_container
@ -21,11 +21,13 @@ from pytest_tests.helpers.utility import placement_policy_from_container
class TestContainer(ClusterTestBase):
@pytest.mark.parametrize("name", ["", "test-container"], ids=["No name", "Set particular name"])
@pytest.mark.smoke
def test_container_creation(self, default_wallet: WalletInfo, name: str):
def test_container_creation(self, default_wallet: str, name: str):
scenario_title = "with name" if name else "without name"
allure.dynamic.title(f"Create container {scenario_title}")
wallet = default_wallet
with open(wallet) as file:
json_wallet = json.load(file)
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
cid = create_container(
@ -36,7 +38,9 @@ class TestContainer(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
containers = list_containers(wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
containers = list_containers(
wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid in containers, f"Expected container {cid} in containers: {containers}"
container_info: str = get_container(
@ -46,17 +50,19 @@ class TestContainer(ClusterTestBase):
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
container_info = container_info.casefold() # To ignore case when comparing with expected values
container_info = (
container_info.casefold()
) # To ignore case when comparing with expected values
info_to_check = {
f"basic ACL: {PRIVATE_ACL_F} (private)",
f"owner ID: {wallet.get_address_from_json(0)}",
f"CID: {cid}",
f"owner ID: {json_wallet.get('accounts')[0].get('address')}",
f"container ID: {cid}",
}
if name:
info_to_check.add(f"Name={name}")
with reporter.step("Check container has correct information"):
with allure.step("Check container has correct information"):
expected_policy = placement_rule.casefold()
actual_policy = placement_policy_from_container(container_info)
assert (
@ -65,9 +71,11 @@ class TestContainer(ClusterTestBase):
for info in info_to_check:
expected_info = info.casefold()
assert expected_info in container_info, f"Expected {expected_info} in container info:\n{container_info}"
assert (
expected_info in container_info
), f"Expected {expected_info} in container info:\n{container_info}"
with reporter.step("Delete container and check it was deleted"):
with allure.step("Delete container and check it was deleted"):
delete_container(
wallet,
cid,
@ -76,46 +84,46 @@ class TestContainer(ClusterTestBase):
await_mode=True,
)
self.tick_epoch()
wait_for_container_deletion(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
wait_for_container_deletion(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
@allure.title("Parallel container creation and deletion")
def test_container_creation_deletion_parallel(self, default_wallet: WalletInfo):
def test_container_creation_deletion_parallel(self, default_wallet: str):
containers_count = 3
wallet = default_wallet
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
iteration_count = 10
for iteration in range(iteration_count):
cids: list[str] = []
with reporter.step(f"Create {containers_count} containers"):
for _ in range(containers_count):
cids.append(
create_container(
wallet,
rule=placement_rule,
await_mode=False,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
wait_for_creation=False,
)
)
with reporter.step("Wait for containers occur in container list"):
for cid in cids:
wait_for_container_creation(
cids: list[str] = []
with allure.step(f"Create {containers_count} containers"):
for _ in range(containers_count):
cids.append(
create_container(
wallet,
cid,
sleep_interval=containers_count,
rule=placement_rule,
await_mode=False,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
wait_for_creation=False,
)
)
with reporter.step("Delete containers and check they were deleted"):
for cid in cids:
delete_container(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, await_mode=True
)
containers_list = list_containers(
wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid not in containers_list, "Container not deleted"
with allure.step("Wait for containers occur in container list"):
for cid in cids:
wait_for_container_creation(
wallet,
cid,
sleep_interval=containers_count,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with allure.step("Delete containers and check they were deleted"):
for cid in cids:
delete_container(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
self.tick_epoch()
wait_for_container_deletion(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)

File diff suppressed because it is too large Load diff

View file

@ -1,12 +1,18 @@
import os
import random
from datetime import datetime
import allure
import pytest
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers import ShardsWatcher
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import TestFile, generate_file
@pytest.fixture()
@allure.title("Select random node for testing")
def node_under_test(cluster: Cluster) -> ClusterNode:
selected_node = random.choice(cluster.cluster_nodes)
allure.attach(f"{selected_node}", "Selected node", allure.attachment_type.TEXT)
return selected_node
@pytest.fixture()
@ -21,10 +27,3 @@ def shards_watcher(node_under_test: ClusterNode) -> ShardsWatcher:
def test_start_time() -> datetime:
start_time = datetime.utcnow()
return start_time
@pytest.fixture()
@allure.title("Generate simple size file")
def simple_file(simple_object_size: ObjectSize) -> TestFile:
path_file = generate_file(size=simple_object_size.value)
return path_file

View file

@ -1,79 +0,0 @@
import datetime
from time import sleep
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.steps.cli.object import neo_go_query_height
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import datetime_utils
@pytest.mark.order(20)
@pytest.mark.failover
class TestTime(ClusterTestBase):
@reporter.step("Neo-go should continue to release blocks")
def check_nodes_block(self, cluster_state_controller: ClusterStateController):
count_blocks = {}
with reporter.step("Get current block id"):
for cluster_node in self.cluster.cluster_nodes:
cluster_state_controller.get_node_date(cluster_node)
count_blocks[cluster_node] = neo_go_query_height(
shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint()
)["Latest block"]
with reporter.step("Wait for 3 blocks"):
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 3)
with reporter.step("Current block id should be higher than before"):
for cluster_node in self.cluster.cluster_nodes:
shell = cluster_node.host.get_shell()
now_block = neo_go_query_height(shell=shell, endpoint=cluster_node.morph_chain.get_http_endpoint())[
"Latest block"
]
assert count_blocks[cluster_node] < now_block
@pytest.fixture()
def node_time_synchronizer(self, cluster_state_controller: ClusterStateController) -> None:
cluster_state_controller.set_sync_date_all_nodes(status="inactive")
yield
cluster_state_controller.set_sync_date_all_nodes(status="active")
@allure.title("Changing hardware and system time")
def test_system_time(self, cluster_state_controller: ClusterStateController, node_time_synchronizer: None):
cluster_nodes = self.cluster.cluster_nodes
timezone_utc = datetime.timezone.utc
node_1, node_2, node_3 = cluster_nodes[0:3]
with reporter.step("On node 1, move the system time forward by 5 days"):
cluster_state_controller.change_node_date(
node_1, (datetime.datetime.now(timezone_utc) + datetime.timedelta(days=5))
)
self.check_nodes_block(cluster_state_controller)
with reporter.step("On node 2, move the system time back 5 days."):
cluster_state_controller.change_node_date(
node_2, (datetime.datetime.now(timezone_utc) - datetime.timedelta(days=5))
)
self.check_nodes_block(cluster_state_controller)
with reporter.step("On node 3, move the system time forward by 10 days"):
cluster_state_controller.change_node_date(
node_3, (datetime.datetime.now(timezone_utc) + datetime.timedelta(days=10))
)
self.check_nodes_block(cluster_state_controller)
with reporter.step("Return the time on all nodes to the current one"):
for cluster_node in self.cluster.cluster_nodes:
cluster_state_controller.restore_node_date(cluster_node)
self.check_nodes_block(cluster_state_controller)
with reporter.step("Reboot all nodes"):
cluster_state_controller.shutdown_cluster(mode="soft")
cluster_state_controller.start_stopped_hosts()
self.check_nodes_block(cluster_state_controller)

View file

@ -5,9 +5,7 @@ from time import sleep
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.resources.wellknown_acl import EACL_PUBLIC_READ_WRITE, PUBLIC_ACL
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import (
get_object,
@ -21,10 +19,8 @@ from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.failover_utils import wait_all_storage_nodes_returned, wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
logger = logging.getLogger("NeoLogger")
@ -46,16 +42,16 @@ OBJECT_ATTRIBUTES = [
class TestFailoverNetwork(ClusterTestBase):
@pytest.fixture(autouse=True)
@allure.title("Restore network")
def restore_network(self, healthcheck: Healthcheck, cluster_state_controller: ClusterStateController):
def restore_network(self, cluster_state_controller: ClusterStateController):
yield
with reporter.step(f"Count blocked nodes {len(blocked_nodes)}"):
with allure.step(f"Count blocked nodes {len(blocked_nodes)}"):
not_empty = len(blocked_nodes) != 0
for node in list(blocked_nodes):
with reporter.step(f"Restore network for {node}"):
cluster_state_controller.restore_traffic(node=node)
with allure.step(f"Restore network for {node}"):
cluster_state_controller.restore_traffic(mode="ports", node=node)
blocked_nodes.remove(node)
if not_empty:
parallel(healthcheck.storage_healthcheck, self.cluster.cluster_nodes)
wait_all_storage_nodes_returned(self.shell, self.cluster)
@pytest.fixture()
@allure.title("Restore drop traffic to system")
@ -75,13 +71,13 @@ class TestFailoverNetwork(ClusterTestBase):
def storage_objects(
self,
simple_object_size: ObjectSize,
default_wallet: WalletInfo,
default_wallet: str,
) -> list[StorageObjectInfo]:
file_path = generate_file(simple_object_size.value)
file_hash = get_file_hash(file_path)
with reporter.step("Create container"):
with allure.step("Create container"):
placement_rule = "REP 1 CBF 1"
cid = create_container(
wallet=default_wallet,
@ -89,12 +85,11 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_rule,
await_mode=True,
basic_acl=EACL_PUBLIC_READ_WRITE,
)
storage_objects = []
with reporter.step("Put object"):
with allure.step("Put object"):
for attribute in OBJECT_ATTRIBUTES:
oid = put_object_to_random_node(
wallet=default_wallet,
@ -106,7 +101,7 @@ class TestFailoverNetwork(ClusterTestBase):
storage_object = StorageObjectInfo(cid=cid, oid=oid)
storage_object.size = simple_object_size.value
storage_object.wallet = default_wallet
storage_object.wallet_file_path = default_wallet
storage_object.file_path = file_path
storage_object.file_hash = file_hash
storage_object.attributes = attribute
@ -120,7 +115,7 @@ class TestFailoverNetwork(ClusterTestBase):
@allure.title("Block Storage node traffic")
def test_block_storage_node_traffic(
self,
default_wallet: WalletInfo,
default_wallet: str,
require_multiple_hosts,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
@ -151,47 +146,45 @@ class TestFailoverNetwork(ClusterTestBase):
# TODO: the intent of this logic is not clear, need to revisit
nodes_to_block = random.choices(nodes, k=2)
nodes_non_block = list(set(self.cluster.storage_nodes) - set(nodes_to_block))
nodes_non_block_cluster = [
cluster_node for cluster_node in self.cluster.cluster_nodes if cluster_node.storage_node in nodes_non_block
]
with reporter.step("Block traffic and check corrupted object"):
for node in nodes_non_block_cluster:
with reporter.step(f"Block incoming traffic at node {node}"):
blocked_nodes.append(node)
cluster_state_controller.drop_traffic(
node=node, wakeup_timeout=wakeup_node_timeout, name_interface="data", block_nodes=nodes_to_block
)
excluded_nodes = []
for node in nodes_to_block:
with allure.step(f"Block incoming traffic at node {node} on port {PORTS_TO_BLOCK}"):
block_node = [
cluster_node for cluster_node in self.cluster.cluster_nodes if cluster_node.storage_node == node
]
blocked_nodes.append(*block_node)
excluded_nodes.append(node)
cluster_state_controller.drop_traffic(
mode="ports",
node=node,
wakeup_timeout=wakeup_node_timeout,
ports=PORTS_TO_BLOCK,
)
with reporter.step(f"Check object is not stored on node {node}"):
new_nodes = wait_object_replication(
cid,
oid,
2,
shell=self.shell,
nodes=list(set(self.cluster.storage_nodes) - set(nodes_non_block)),
)
assert node.storage_node not in new_nodes
with allure.step(f"Check object is not stored on node {node}"):
new_nodes = wait_object_replication(
cid,
oid,
2,
shell=self.shell,
nodes=list(set(self.cluster.storage_nodes) - set(excluded_nodes)),
)
assert node not in new_nodes
with reporter.step("Check object data is not corrupted"):
got_file_path = get_object(
wallet, cid, oid, endpoint=new_nodes[0].get_rpc_endpoint(), shell=self.shell
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
with allure.step("Check object data is not corrupted"):
got_file_path = get_object(wallet, cid, oid, endpoint=new_nodes[0].get_rpc_endpoint(), shell=self.shell)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
with reporter.step(f"Unblock incoming traffic"):
for node in nodes_non_block_cluster:
with reporter.step(f"Unblock at host {node}"):
cluster_state_controller.restore_traffic(node=node)
block_node = [
cluster_node
for cluster_node in self.cluster.cluster_nodes
if cluster_node.storage_node == node.storage_node
]
blocked_nodes.remove(*block_node)
sleep(wakeup_node_timeout)
for node in nodes_to_block:
with allure.step(f"Unblock incoming traffic at host {node} on port {PORTS_TO_BLOCK}"):
cluster_state_controller.restore_traffic(mode="ports", node=node)
block_node = [
cluster_node for cluster_node in self.cluster.cluster_nodes if cluster_node.storage_node == node
]
blocked_nodes.remove(*block_node)
sleep(wakeup_node_timeout)
with reporter.step("Check object data is not corrupted"):
with allure.step("Check object data is not corrupted"):
new_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=new_nodes[0].get_rpc_endpoint())
@ -202,35 +195,37 @@ class TestFailoverNetwork(ClusterTestBase):
def test_block_data_interface(
self,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
restore_down_interfaces: None,
delete_file_after_test: None,
storage_objects: list[StorageObjectInfo],
):
storage_object = storage_objects[0]
with reporter.step("Search nodes with object"):
with allure.step("Search nodes with object"):
nodes_with_object = get_object_nodes(
cluster=self.cluster,
wallet=default_wallet,
cid=storage_object.cid,
oid=storage_object.oid,
alive_node=self.cluster.cluster_nodes[0],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Get data interface to node"):
with allure.step("Get data interface to node"):
config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys())
with reporter.step(f"Get data in {config_interfaces}"):
with allure.step(f"Get data in {config_interfaces}"):
data_interfaces = [interface for interface in config_interfaces if "data" in interface]
with reporter.step("Block data interfaces for node"):
with allure.step("Block data interfaces for node"):
for interface in data_interfaces:
cluster_state_controller.down_interface(nodes=nodes_with_object, interface=interface)
with reporter.step("Tick epoch and wait 2 block"):
with allure.step("Tick epoch and wait 2 block"):
nodes_without_an_object = list(set(self.cluster.cluster_nodes) - set(nodes_with_object))
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
with reporter.step("Get object for target nodes to data interfaces, expect false"):
with allure.step("Get object for target nodes to data interfaces, expect false"):
with pytest.raises(RuntimeError, match="return code: 1"):
get_object(
wallet=default_wallet,
@ -240,7 +235,7 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(),
)
with reporter.step(f"Get object others nodes, expect true"):
with allure.step(f"Get object others nodes, expect true"):
input_file = get_object(
wallet=default_wallet,
cid=storage_object.cid,
@ -250,7 +245,7 @@ class TestFailoverNetwork(ClusterTestBase):
)
file_wait_delete.append(input_file)
with reporter.step("Restore interface and tick 1 epoch, wait 2 block"):
with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
cluster_state_controller.restore_interfaces()
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
@ -259,7 +254,7 @@ class TestFailoverNetwork(ClusterTestBase):
def test_block_internal_interface(
self,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
restore_down_interfaces: None,
delete_file_after_test: None,
storage_objects: list[StorageObjectInfo],
@ -267,28 +262,30 @@ class TestFailoverNetwork(ClusterTestBase):
):
storage_object = storage_objects[0]
with reporter.step("Search nodes with object"):
with allure.step("Search nodes with object"):
nodes_with_object = get_object_nodes(
cluster=self.cluster,
wallet=default_wallet,
cid=storage_object.cid,
oid=storage_object.oid,
alive_node=self.cluster.cluster_nodes[0],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Get internal interface to node"):
with allure.step("Get internal interface to node"):
config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys())
with reporter.step(f"Get internal in {config_interfaces}"):
with allure.step(f"Get internal in {config_interfaces}"):
internal_interfaces = [interface for interface in config_interfaces if "internal" in interface]
with reporter.step("Block internal interfaces for node"):
with allure.step("Block internal interfaces for node"):
for interface in internal_interfaces:
cluster_state_controller.down_interface(nodes=nodes_with_object, interface=interface)
with reporter.step("Tick epoch and wait 2 block"):
with allure.step("Tick epoch and wait 2 block"):
nodes_without_an_object = list(set(self.cluster.cluster_nodes) - set(nodes_with_object))
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
with reporter.step("Get object others node, expect false"):
with allure.step("Get object others node, expect false"):
with pytest.raises(RuntimeError, match="return code: 1"):
get_object(
wallet=default_wallet,
@ -298,7 +295,7 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
)
with reporter.step("Put object, others node, expect false"):
with allure.step("Put object, others node, expect false"):
with pytest.raises(RuntimeError, match="return code: 1"):
put_object(
wallet=default_wallet,
@ -308,7 +305,7 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
)
with reporter.step(f"Get object nodes with object, expect true"):
with allure.step(f"Get object nodes with object, expect true"):
input_file = get_object(
wallet=default_wallet,
cid=storage_object.cid,
@ -318,7 +315,7 @@ class TestFailoverNetwork(ClusterTestBase):
)
file_wait_delete.append(input_file)
with reporter.step(f"Put object nodes with object, expect true"):
with allure.step(f"Put object nodes with object, expect true"):
temp_file_path = generate_file(simple_object_size.value)
_ = put_object(
wallet=default_wallet,
@ -328,7 +325,7 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(),
)
file_wait_delete.append(temp_file_path)
with reporter.step("Restore interface and tick 1 epoch, wait 2 block"):
with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
cluster_state_controller.restore_interfaces()
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
@ -343,7 +340,7 @@ class TestFailoverNetwork(ClusterTestBase):
self,
require_multiple_interfaces,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
simple_object_size: ObjectSize,
delete_file_after_test: None,
restore_down_interfaces: None,
@ -351,13 +348,13 @@ class TestFailoverNetwork(ClusterTestBase):
other_interface: Interfaces,
):
cluster_nodes = self.cluster.cluster_nodes
with reporter.step(f"Block {block_interface.value} interfaces"):
with allure.step(f"Block {block_interface.value} interfaces"):
cluster_state_controller.down_interface(cluster_nodes, block_interface.value)
with reporter.step("Tick 1 epoch and wait 2 block for sync all nodes"):
with allure.step("Tick 1 epoch and wait 2 block for sync all nodes"):
self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Create container"):
with allure.step("Create container"):
cid = create_container(
wallet=default_wallet,
shell=self.shell,
@ -365,7 +362,7 @@ class TestFailoverNetwork(ClusterTestBase):
rule="REP 4 CBF 1",
)
with reporter.step("Put object"):
with allure.step("Put object"):
file_path = generate_file(simple_object_size.value)
file_wait_delete.append(file_path)
@ -377,7 +374,7 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080",
)
with reporter.step("Get object"):
with allure.step("Get object"):
file_get_path = get_object(
wallet=default_wallet,
cid=cid,
@ -387,7 +384,7 @@ class TestFailoverNetwork(ClusterTestBase):
)
file_wait_delete.append(file_get_path)
with reporter.step("Restore interfaces all nodes"):
with allure.step("Restore interfaces all nodes"):
cluster_state_controller.restore_interfaces()
self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2)
@ -399,7 +396,7 @@ class TestFailoverNetwork(ClusterTestBase):
self,
require_multiple_interfaces,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
simple_object_size: ObjectSize,
delete_file_after_test: None,
restore_down_interfaces: None,
@ -408,19 +405,19 @@ class TestFailoverNetwork(ClusterTestBase):
cluster_nodes = self.cluster.cluster_nodes
latest_block = {}
with reporter.step("Get block all nodes"):
with allure.step("Get block all nodes"):
for cluster_node in cluster_nodes:
latest_block[cluster_node] = neo_go_query_height(
shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint()
)
with reporter.step(f"Block {interface} interfaces"):
with allure.step(f"Block {interface} interfaces"):
cluster_state_controller.down_interface(cluster_nodes, interface.value)
with reporter.step("Tick 1 epoch and wait 2 block for sync all nodes"):
with allure.step("Tick 1 epoch and wait 2 block for sync all nodes"):
self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Create container"):
with allure.step("Create container"):
cid = create_container(
wallet=default_wallet,
shell=self.shell,
@ -428,7 +425,7 @@ class TestFailoverNetwork(ClusterTestBase):
rule="REP 4 CBF 1",
)
with reporter.step(f"Put object, after down {interface}"):
with allure.step(f"Put object, after down {interface}"):
file_path = generate_file(simple_object_size.value)
file_wait_delete.append(file_path)
@ -440,7 +437,7 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Get object"):
with allure.step("Get object"):
file_get_path = get_object(
wallet=default_wallet,
cid=cid,
@ -452,19 +449,19 @@ class TestFailoverNetwork(ClusterTestBase):
now_block = {}
with reporter.step("Get actual block"):
with allure.step("Get actual block"):
for cluster_node in cluster_nodes:
now_block[cluster_node] = neo_go_query_height(
shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint()
)
with reporter.step(f"Compare block"):
with allure.step(f"Compare block"):
for cluster_node, items in now_block.items():
with reporter.step(
with allure.step(
f"Node - {cluster_node.host_ip}, old block - {latest_block[cluster_node]['Latest block']}, "
f"now block - {now_block[cluster_node]['Latest block']}"
):
assert latest_block[cluster_node]["Latest block"] < now_block[cluster_node]["Latest block"]
with reporter.step("Restore interfaces all nodes"):
with allure.step("Restore interfaces all nodes"):
cluster_state_controller.restore_interfaces()
self.tick_epochs(1, alive_node=self.cluster.cluster_nodes[0].storage_node, wait_block=2)

View file

@ -1,13 +1,14 @@
import itertools
import logging
import os.path
import random
import time
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object
from frostfs_testlib.steps.cli.object import get_object
from frostfs_testlib.steps.node_management import check_node_in_map, check_node_not_in_map
from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController
@ -15,8 +16,9 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils import datetime_utils
from frostfs_testlib.utils.failover_utils import wait_for_host_offline, wait_object_replication
from frostfs_testlib.utils.file_utils import get_file_hash
from pytest import FixtureRequest
@ -34,49 +36,35 @@ class TestFailoverServer(ClusterTestBase):
def wait_node_in_map(self, *args, **kwargs):
check_node_in_map(*args, **kwargs)
@allure.title("[Test] Create containers")
@allure.step("Create {count_containers} containers and {count_files} objects")
@pytest.fixture
def containers(
self,
request: FixtureRequest,
default_wallet: WalletInfo,
default_wallet: str,
) -> list[StorageContainer]:
placement_rule = "REP 2 CBF 2 SELECT 2 FROM *"
containers_count = request.param
results = parallel(
[create_container for _ in range(containers_count)],
wallet=default_wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
containers = []
containers = [
StorageContainer(StorageContainerInfo(result.result(), default_wallet), self.shell, self.cluster)
for result in results
]
for _ in range(request.param):
cont_id = create_container(
default_wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
wallet = WalletInfo(path=default_wallet)
storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
containers.append(
StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
)
return containers
@allure.title("[Test] Create container")
@pytest.fixture()
def container(self, default_wallet: WalletInfo) -> StorageContainer:
select = len(self.cluster.cluster_nodes)
placement_rule = f"REP {select - 1} CBF 1 SELECT {select} FROM *"
cont_id = create_container(
default_wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
storage_cont_info = StorageContainerInfo(cont_id, default_wallet)
return StorageContainer(storage_cont_info, self.shell, self.cluster)
@allure.title("[Class] Create objects")
@allure.step("Create object and delete after test")
@pytest.fixture(scope="class")
def storage_objects(
self,
@ -84,183 +72,153 @@ class TestFailoverServer(ClusterTestBase):
containers: list[StorageContainer],
simple_object_size: ObjectSize,
complex_object_size: ObjectSize,
) -> StorageObjectInfo:
count_object = request.param
object_sizes = [simple_object_size, complex_object_size]
object_list = []
for cont in containers:
for _ in range(count_object):
object_list.append(cont.generate_object(size=random.choice(object_sizes).value))
for storage_object in object_list:
os.remove(storage_object.file_path)
yield object_list
@allure.step("Select random node to stop and start it after test")
@pytest.fixture
def node_to_stop(
self, node_under_test: ClusterNode, cluster_state_controller: ClusterStateController
) -> ClusterNode:
yield node_under_test
with allure.step(f"start {node_under_test.storage_node}"):
cluster_state_controller.start_stopped_hosts()
@allure.step("Upload object with nodes and compare")
def get_corrupted_objects_list(
self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]
) -> list[StorageObjectInfo]:
object_count = request.param
sizes_samples = [simple_object_size, complex_object_size]
samples_count = len(sizes_samples)
assert object_count >= samples_count, f"Object count is too low, must be >= {samples_count}"
corrupted_objects = []
errors_get = []
for node in nodes:
for storage_object in storage_objects:
try:
got_file_path = get_object(
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
endpoint=node.get_rpc_endpoint(),
shell=self.shell,
timeout="60s",
)
if storage_object.file_hash != get_file_hash(got_file_path):
corrupted_objects.append(storage_object)
os.remove(got_file_path)
except RuntimeError:
errors_get.append(storage_object.oid)
sizes_weights = [2, 1]
sizes = sizes_samples + random.choices(sizes_samples, weights=sizes_weights, k=object_count - samples_count)
assert len(errors_get) == 0, f"Get failed - {errors_get}"
return corrupted_objects
results = parallel(
[container.generate_object for _ in sizes for container in containers],
size=itertools.cycle([size.value for size in sizes]),
)
return [result.result() for result in results]
@allure.title("[Test] Create objects and get nodes with object")
@pytest.fixture()
def object_and_nodes(
self, simple_object_size: ObjectSize, container: StorageContainer
) -> tuple[StorageObjectInfo, list[ClusterNode]]:
object_info = container.generate_object(simple_object_size.value)
object_nodes = get_object_nodes(self.cluster, object_info.cid, object_info.oid, self.cluster.cluster_nodes[0])
return object_info, object_nodes
def _verify_object(self, storage_object: StorageObjectInfo, node: StorageNode):
with reporter.step(f"Verify object {storage_object.oid} from node {node}"):
file_path = get_object(
storage_object.wallet,
def check_objects_replication(
self, storage_objects: list[StorageObjectInfo], storage_nodes: list[StorageNode]
) -> None:
for storage_object in storage_objects:
wait_object_replication(
storage_object.cid,
storage_object.oid,
endpoint=node.get_rpc_endpoint(),
2,
shell=self.shell,
timeout="60s",
nodes=storage_nodes,
sleep_interval=45,
attempts=60,
)
assert storage_object.file_hash == get_file_hash(file_path)
@reporter.step("Verify objects")
def verify_objects(self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]) -> None:
parallel(self._verify_object, storage_objects * len(nodes), node=itertools.cycle(nodes))
@allure.title("Full shutdown node")
@pytest.mark.parametrize("containers, storage_objects", [(5, 10)], indirect=True)
def test_complete_node_shutdown(
self,
containers: list[StorageContainer],
storage_objects: list[StorageObjectInfo],
node_under_test: ClusterNode,
default_wallet: str,
node_to_stop: ClusterNode,
cluster_state_controller: ClusterStateController,
):
with reporter.step(f"Remove one node from the list of nodes"):
alive_nodes = list(set(self.cluster.cluster_nodes) - {node_under_test})
with allure.step(f"Remove {node_to_stop} from the list of nodes"):
alive_nodes = list(set(self.cluster.cluster_nodes) - {node_to_stop})
storage_nodes = [cluster.storage_node for cluster in alive_nodes]
with reporter.step("Tick 2 epochs and wait for 2 blocks"):
self.tick_epochs(2, storage_nodes[0], wait_block=2)
with allure.step("Tick epoch"):
self.tick_epochs(1, storage_nodes[0])
with reporter.step(f"Stop node"):
cluster_state_controller.stop_node_host(node_under_test, "hard")
with allure.step("Wait 2 block time"):
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
with reporter.step("Verify that there are no corrupted objects"):
self.verify_objects(storage_nodes, storage_objects)
cluster_state_controller.stop_node_host(node=node_to_stop, mode="hard")
with reporter.step(f"Check node still in map"):
self.wait_node_in_map(node_under_test.storage_node, self.shell, alive_node=storage_nodes[0])
with allure.step(f"Check if the node {node_to_stop.storage_node} has stopped"):
wait_for_host_offline(self.shell, node_to_stop.storage_node)
with allure.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
with allure.step(f"check {node_to_stop.storage_node} in map"):
self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4
with reporter.step(f"Tick {count_tick_epoch} epochs and wait for 2 blocks"):
self.tick_epochs(count_tick_epoch, storage_nodes[0], wait_block=2)
with allure.step(f"Tick {count_tick_epoch} epoch, in {storage_nodes[0]} node"):
for tick in range(count_tick_epoch):
self.tick_epoch(storage_nodes[0])
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
with reporter.step(f"Check node in not map after {count_tick_epoch} epochs"):
self.wait_node_not_in_map(node_under_test.storage_node, self.shell, alive_node=storage_nodes[0])
with allure.step(f"Check if the node {node_to_stop.storage_node} has stopped"):
wait_for_host_offline(self.shell, node_to_stop.storage_node)
with reporter.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epochs"):
self.verify_objects(storage_nodes, storage_objects)
with allure.step(f"Check {node_to_stop} in not map"):
self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
with allure.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
@allure.title("Temporarily disable a node")
@pytest.mark.parametrize("containers, storage_objects", [(5, 10)], indirect=True)
def test_temporarily_disable_a_node(
self,
containers: list[StorageContainer],
storage_objects: list[StorageObjectInfo],
node_under_test: ClusterNode,
default_wallet: str,
node_to_stop: ClusterNode,
cluster_state_controller: ClusterStateController,
):
with reporter.step(f"Remove one node from the list"):
storage_nodes = list(set(self.cluster.storage_nodes) - {node_under_test.storage_node})
with allure.step(f"Remove {node_to_stop} from the list of nodes"):
storage_nodes = list(set(self.cluster.storage_nodes) - {node_to_stop.storage_node})
with reporter.step("Tick 2 epochs and wait for 2 blocks"):
self.tick_epochs(2, storage_nodes[0], wait_block=2)
with allure.step("Tick epoch"):
self.tick_epochs(1, storage_nodes[0])
with reporter.step(f"Stop node"):
cluster_state_controller.stop_node_host(node_under_test, "hard")
with allure.step("Wait 2 block time"):
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
with reporter.step("Verify that there are no corrupted objects"):
self.verify_objects(storage_nodes, storage_objects)
cluster_state_controller.stop_node_host(node=node_to_stop, mode="hard")
with reporter.step(f"Check node still in map"):
self.wait_node_in_map(node_under_test.storage_node, self.shell, alive_node=storage_nodes[0])
with allure.step(f"Check if the node {node_to_stop} has stopped"):
wait_for_host_offline(self.shell, node_to_stop.storage_node)
with reporter.step(f"Start node"):
cluster_state_controller.start_node_host(node_under_test)
with allure.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
with reporter.step("Verify that there are no corrupted objects"):
self.verify_objects(storage_nodes, storage_objects)
with allure.step(f"Check {node_to_stop} in map"):
self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
@allure.title("Not enough nodes in the container with policy - 'REP 3 CBF 1 SELECT 4 FROM *'")
def test_not_enough_nodes_in_container_rep_3(
self,
object_and_nodes: tuple[StorageObjectInfo, list[ClusterNode]],
default_wallet: WalletInfo,
cluster_state_controller: ClusterStateController,
simple_file: str,
):
object_info, object_nodes = object_and_nodes
endpoint_without_object = list(set(self.cluster.cluster_nodes) - set(object_nodes))[
0
].storage_node.get_rpc_endpoint()
endpoint_with_object = object_nodes[0].storage_node.get_rpc_endpoint()
cluster_state_controller.start_node_host(node_to_stop)
with reporter.step("Stop all nodes with object except first one"):
parallel(cluster_state_controller.stop_node_host, object_nodes[1:], mode="hard")
with reporter.step(f"Get object from node without object"):
get_object(default_wallet, object_info.cid, object_info.oid, self.shell, endpoint_without_object)
with reporter.step(f"Get object from node with object"):
get_object(default_wallet, object_info.cid, object_info.oid, self.shell, endpoint_with_object)
with reporter.step(f"[Negative] Put operation to node with object"):
with pytest.raises(RuntimeError):
put_object(default_wallet, simple_file, object_info.cid, self.shell, endpoint_with_object)
@allure.title("Not enough nodes in the container with policy - 'REP 2 CBF 2 SELECT 4 FROM *'")
def test_not_enough_nodes_in_container_rep_2(
self,
default_wallet: WalletInfo,
cluster_state_controller: ClusterStateController,
simple_file: str,
):
with reporter.step("Create container with full network map"):
node_count = len(self.cluster.cluster_nodes)
placement_rule = f"REP {node_count - 2} IN X CBF 2 SELECT {node_count} FROM * AS X"
cid = create_container(
default_wallet,
self.shell,
self.cluster.default_rpc_endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
with reporter.step("Put object"):
oid = put_object(default_wallet, simple_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Search nodes with object"):
object_nodes = get_object_nodes(self.cluster, cid, oid, self.cluster.cluster_nodes[0])
with reporter.step("Choose node to stop"):
node_under_test = random.choice(object_nodes)
alive_node_with_object = random.choice(list(set(object_nodes) - {node_under_test}))
alive_endpoint_with_object = alive_node_with_object.storage_node.get_rpc_endpoint()
with reporter.step("Stop random node with object"):
cluster_state_controller.stop_node_host(node_under_test, "hard")
with reporter.step("Put object to alive node with object"):
oid_2 = put_object(default_wallet, simple_file, cid, self.shell, alive_endpoint_with_object)
with reporter.step("Get object from alive node with object"):
get_object(default_wallet, cid, oid_2, self.shell, alive_endpoint_with_object)
with reporter.step("Create container on alive node"):
create_container(
default_wallet,
self.shell,
alive_endpoint_with_object,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
with allure.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list

View file

@ -5,10 +5,10 @@ from time import sleep
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import CommandOptions
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
from frostfs_testlib.steps.node_management import (
@ -20,7 +20,6 @@ from frostfs_testlib.steps.node_management import (
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.s3.s3_helper import search_nodes_with_bucket
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -45,6 +44,20 @@ def file_keeper():
keeper.restore_files()
@allure.step("Return all stopped hosts")
@pytest.fixture(scope="function", autouse=True)
def after_run_return_all_stopped_hosts(cluster_state_controller: ClusterStateController) -> str:
yield
cluster_state_controller.start_stopped_hosts()
@allure.step("Return all stopped services after test")
@pytest.fixture(scope="function")
def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController):
yield
cluster_state_controller.start_all_stopped_services()
@pytest.mark.failover
@pytest.mark.failover_storage
class TestFailoverStorage(ClusterTestBase):
@ -65,7 +78,7 @@ class TestFailoverStorage(ClusterTestBase):
source_file_path = generate_file(simple_object_size.value)
stopped_hosts_nodes = []
with reporter.step(f"Create container and put object"):
with allure.step(f"Create container and put object"):
cid = create_container(
wallet,
shell=self.shell,
@ -75,10 +88,10 @@ class TestFailoverStorage(ClusterTestBase):
)
oid = put_object_to_random_node(wallet, source_file_path, cid, shell=self.shell, cluster=self.cluster)
with reporter.step(f"Wait for replication and get nodes with object"):
with allure.step(f"Wait for replication and get nodes with object"):
nodes_with_object = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
with reporter.step(f"Stop 2 nodes with object and wait replication one by one"):
with allure.step(f"Stop 2 nodes with object and wait replication one by one"):
for storage_node in random.sample(nodes_with_object, 2):
stopped_hosts_nodes.append(storage_node)
@ -93,75 +106,124 @@ class TestFailoverStorage(ClusterTestBase):
nodes=list(set(self.cluster.storage_nodes) - {*stopped_hosts_nodes}),
)
with reporter.step("Check object data is not corrupted"):
with allure.step("Check object data is not corrupted"):
got_file_path = get_object(
wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
with reporter.step("Return all hosts"):
with allure.step("Return all hosts"):
cluster_state_controller.start_stopped_hosts()
with reporter.step("Check object data is not corrupted"):
with allure.step("Check object data is not corrupted"):
replicated_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
got_file_path = get_object(
wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint()
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
@allure.title("Panic reboot nodes (sequenced_reboots={sequence})")
@pytest.mark.parametrize("sequence", [True, False])
@pytest.mark.failover_panic
def test_panic_storage_node_host(
self,
default_wallet,
require_multiple_hosts,
sequence: bool,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
):
wallet = default_wallet
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
cid = create_container(
wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
oid = put_object_to_random_node(wallet, source_file_path, cid, shell=self.shell, cluster=self.cluster)
nodes_with_object = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
allure.attach(
"\n".join([str(node) for node in nodes_with_object]),
"Current nodes with object",
allure.attachment_type.TEXT,
)
new_nodes: list[StorageNode] = []
with allure.step(f"Hard reboot hosts via magic SysRq option"):
for storage_node in nodes_with_object:
cluster_state_controller.panic_reboot_host(self.cluster.node(storage_node))
if sequence:
new_nodes = wait_object_replication(
cid,
oid,
2,
shell=self.shell,
nodes=self.cluster.storage_nodes,
)
allure.attach(
"\n".join([str(new_node) for new_node in new_nodes]),
f"Nodes with object after {storage_node} fail",
allure.attachment_type.TEXT,
)
if not sequence:
new_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
allure.attach(
"\n".join([str(new_node) for new_node in new_nodes]),
"Nodes with object after nodes fail",
allure.attachment_type.TEXT,
)
got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=new_nodes[0].get_rpc_endpoint())
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
@allure.title("Do not ignore unhealthy tree endpoints (s3_client={s3_client})")
def test_unhealthy_tree(
self,
s3_client: S3ClientWrapper,
default_wallet: WalletInfo,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
after_run_return_all_stopped_services,
):
default_node = self.cluster.cluster_nodes[0]
with reporter.step("Turn S3 GW off on default node"):
with allure.step("Turn S3 GW off on default node"):
cluster_state_controller.stop_service_of_type(default_node, S3Gate)
with reporter.step("Turn off storage on default node"):
with allure.step("Turn off storage on default node"):
cluster_state_controller.stop_service_of_type(default_node, StorageNode)
with reporter.step("Turn on S3 GW on default node"):
with allure.step("Turn on S3 GW on default node"):
cluster_state_controller.start_service_of_type(default_node, S3Gate)
with reporter.step("Turn on storage on default node"):
with allure.step("Turn on storage on default node"):
cluster_state_controller.start_service_of_type(default_node, StorageNode)
with reporter.step("Create bucket with REP 1 SELECT 1 policy"):
with allure.step("Create bucket with REP 1 SELECT 1 policy"):
bucket = s3_client.create_bucket(
location_constraint="load-1-1",
)
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into bucket"):
with allure.step("Put object into bucket"):
put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
node_bucket = search_nodes_with_bucket(
cluster=self.cluster,
bucket_name=bucket,
wallet=default_wallet,
shell=self.shell,
endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(),
)[0]
with reporter.step("Turn off all storage nodes except bucket node"):
for node in [node_to_stop for node_to_stop in self.cluster.cluster_nodes if node_to_stop != node_bucket]:
with reporter.step(f"Stop storage service on node: {node}"):
with allure.step("Turn off all storage nodes except default"):
for node in self.cluster.cluster_nodes[1:]:
with allure.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_service_of_type(node, StorageNode)
with reporter.step(f"Change s3 endpoint to bucket node"):
s3_client.set_endpoint(node_bucket.s3_gate.get_endpoint())
with reporter.step("Check that object is available"):
with allure.step("Check that object is available"):
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
with reporter.step("Start storage nodes"):
with allure.step("Start storage nodes"):
cluster_state_controller.start_all_stopped_services()
@ -172,11 +234,11 @@ class TestEmptyMap(ClusterTestBase):
A set of tests for makes map empty and verify that we can read objects after that
"""
@reporter.step("Teardown after EmptyMap offline test")
@allure.step("Teardown after EmptyMap offline test")
@pytest.fixture()
def empty_map_offline_teardown(self):
yield
with reporter.step("Return all storage nodes to network map"):
with allure.step("Return all storage nodes to network map"):
for node in stopped_nodes:
include_node_to_network_map(node, node, shell=self.shell, cluster=self.cluster)
stopped_nodes.remove(node)
@ -208,31 +270,34 @@ class TestEmptyMap(ClusterTestBase):
file_name = s3_helper.object_key_from_file_path(file_path)
bucket_objects = [file_name]
with reporter.step("Put object into bucket"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
with reporter.step("Check that object exists in bucket"):
with allure.step("Check that object exists in bucket"):
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
storage_nodes = self.cluster.storage_nodes
with reporter.step("Exclude all storage nodes from network map"):
with allure.step("Exclude all storage nodes from network map"):
for node in storage_nodes:
stopped_nodes.append(node)
exclude_node_from_network_map(node, node, shell=self.shell, cluster=self.cluster)
with reporter.step("Return all storage nodes to network map"):
with allure.step("Return all storage nodes to network map"):
for node in storage_nodes:
include_node_to_network_map(node, node, shell=self.shell, cluster=self.cluster)
stopped_nodes.remove(node)
with reporter.step("Check that we can read object"):
with allure.step("Check that we can read object"):
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
@reporter.step("Teardown after EmptyMap stop service test")
@allure.step("Teardown after EmptyMap stop service test")
@pytest.fixture()
def empty_map_stop_service_teardown(self, cluster_state_controller: ClusterStateController):
yield
with reporter.step("Return all storage nodes to network map"):
with allure.step("Return all storage nodes to network map"):
cluster_state_controller.start_all_stopped_services()
for node in stopped_nodes:
check_node_in_map(node, shell=self.shell, alive_node=node)
@ -268,31 +333,34 @@ class TestEmptyMap(ClusterTestBase):
file_name = s3_helper.object_key_from_file_path(file_path)
bucket_objects = [file_name]
with reporter.step("Put object into bucket"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
with reporter.step("Check that object exists in bucket"):
with allure.step("Check that object exists in bucket"):
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
with reporter.step("Stop all storage nodes"):
with allure.step("Stop all storage nodes"):
cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Remove all nodes from network map"):
with allure.step("Remove all nodes from network map"):
remove_nodes_from_map_morph(
shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode)
)
with reporter.step("Return all storage nodes to network map"):
with allure.step("Return all storage nodes to network map"):
self.return_nodes_after_stop_with_check_empty_map(cluster_state_controller)
with reporter.step("Check that object exists in bucket"):
with allure.step("Check that object exists in bucket"):
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
@reporter.step("Return all nodes to cluster with check empty map first")
@allure.step("Return all nodes to cluster with check empty map first")
def return_nodes_after_stop_with_check_empty_map(self, cluster_state_controller: ClusterStateController) -> None:
first_node = self.cluster.cluster_nodes[0].service(StorageNode)
with reporter.step("Start first node and check network map"):
with allure.step("Start first node and check network map"):
cluster_state_controller.start_service_of_type(self.cluster.cluster_nodes[0], StorageNode)
wait_for_node_to_be_ready(first_node)
@ -316,41 +384,41 @@ class TestEmptyMap(ClusterTestBase):
s3_client: S3ClientWrapper,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
bucket: str,
):
bucket = s3_client.create_bucket()
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
object_versions = []
with reporter.step("Put object into one bucket"):
with allure.step("Put object into one bucket"):
put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
object_versions.append(put_object)
with reporter.step("Stop all storage nodes"):
with allure.step("Stop all storage nodes"):
cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Delete blobovnicza and fstree from all nodes"):
with allure.step("Delete blobovnicza and fstree from all nodes"):
for node in self.cluster.storage_nodes:
node.delete_blobovnicza()
node.delete_fstree()
with reporter.step("Start all storage nodes"):
with allure.step("Start all storage nodes"):
cluster_state_controller.start_all_stopped_services()
# need to get Delete Marker first
with reporter.step("Delete the object from the bucket"):
with allure.step("Delete the object from the bucket"):
delete_object = s3_client.delete_object(bucket, file_name)
object_versions.append(delete_object["VersionId"])
# and now delete all versions of object (including Delete Markers)
with reporter.step("Delete all versions of the object from the bucket"):
with allure.step("Delete all versions of the object from the bucket"):
for version in object_versions:
delete_object = s3_client.delete_object(bucket, file_name, version_id=version)
with reporter.step("Delete bucket"):
with allure.step("Delete bucket"):
s3_client.delete_bucket(bucket)
@allure.title("Object loss from fstree/blobovnicza (versioning=disabled, s3_client={s3_client})")
@ -359,30 +427,31 @@ class TestEmptyMap(ClusterTestBase):
s3_client: S3ClientWrapper,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
bucket: str,
):
bucket = s3_client.create_bucket()
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into one bucket"):
with allure.step("Put object into one bucket"):
s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
with reporter.step("Stop all storage nodes"):
with allure.step("Stop all storage nodes"):
cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Delete blobovnicza and fstree from all nodes"):
with allure.step("Delete blobovnicza and fstree from all nodes"):
for node in self.cluster.storage_nodes:
node.delete_blobovnicza()
node.delete_fstree()
with reporter.step("Start all storage nodes"):
with allure.step("Start all storage nodes"):
cluster_state_controller.start_all_stopped_services()
with reporter.step("Delete the object from the bucket"):
with allure.step("Delete the object from the bucket"):
s3_client.delete_object(bucket, file_name)
with reporter.step("Delete bucket"):
with allure.step("Delete bucket"):
s3_client.delete_bucket(bucket)
@pytest.mark.skip(reason="Need to increase cache lifetime")
@ -400,43 +469,53 @@ class TestEmptyMap(ClusterTestBase):
simple_object_size: ObjectSize,
versioning_status: VersioningStatus,
cluster_state_controller: ClusterStateController,
bucket: str,
):
bucket = s3_client.create_bucket()
s3_helper.set_bucket_versioning(s3_client, bucket, versioning_status)
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into one bucket"):
with allure.step("Put object into one bucket"):
s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
with reporter.step("Stop all storage nodes"):
with allure.step("Stop all storage nodes"):
cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Delete pilorama.db from all nodes"):
with allure.step("Delete pilorama.db from all nodes"):
for node in self.cluster.storage_nodes:
for shard in node.get_shards():
node.delete_file(shard.pilorama)
node.delete_pilorama()
with reporter.step("Start all storage nodes"):
with allure.step("Start all storage nodes"):
cluster_state_controller.start_all_stopped_services()
with reporter.step("Check list objects first time"):
with allure.step("Check list objects first time"):
objects_list = s3_client.list_objects(bucket)
assert objects_list, f"Expected not empty bucket"
with reporter.step("Check list objects second time"):
with allure.step("Check list objects second time"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with reporter.step("Delete bucket"):
with allure.step("Delete bucket"):
s3_client.delete_bucket(bucket)
@pytest.mark.failover
@pytest.mark.failover_data_loss
class TestStorageDataLoss(ClusterTestBase):
@allure.step("Get list of all piloramas on node")
def get_piloramas_list(self, node: StorageNode) -> list:
data_directory_path = node.get_data_directory()
cmd = f"sudo ls -1 {data_directory_path}/meta*/pilorama*"
shell = node.host.get_shell()
stdout = shell.exec(cmd).stdout
piloramas = stdout.split("\n")
return piloramas
@allure.title(
"After metabase loss on all nodes operations on objects and buckets should be still available via S3 (s3_client={s3_client})"
)
@ -447,10 +526,15 @@ class TestStorageDataLoss(ClusterTestBase):
simple_object_size: ObjectSize,
complex_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
after_run_return_all_stopped_services: str,
file_keeper: FileKeeper,
bucket: str,
):
with reporter.step("Put objects into bucket"):
allure.dynamic.description(after_run_return_all_stopped_services)
with allure.step("Create bucket"):
bucket = s3_client.create_bucket()
with allure.step("Put objects into bucket"):
simple_object_path = generate_file(simple_object_size.value)
simple_object_key = s3_helper.object_key_from_file_path(simple_object_path)
@ -460,44 +544,44 @@ class TestStorageDataLoss(ClusterTestBase):
s3_client.put_object(bucket, simple_object_path)
s3_client.put_object(bucket, complex_object_path)
with reporter.step("Check objects are in bucket"):
with allure.step("Check objects are in bucket"):
s3_helper.check_objects_in_bucket(
s3_client, bucket, expected_objects=[simple_object_key, complex_object_key]
)
with reporter.step("Stop storage services on all nodes"):
with allure.step("Stop storage services on all nodes"):
cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Delete metabase from all nodes"):
with allure.step("Delete metabase from all nodes"):
for node in cluster_state_controller.cluster.storage_nodes:
node.delete_metabase()
with reporter.step("Enable resync_metabase option for storage services"):
with allure.step("Enable resync_metabase option for storage services"):
for storage_node in cluster_state_controller.cluster.storage_nodes:
with reporter.step(f"Enable resync_metabase option for {storage_node}"):
with allure.step(f"Enable resync_metabase option for {storage_node}"):
config_file_path, config = storage_node.get_shards_config()
if not config["storage"]["shard"]["default"]["resync_metabase"]:
file_keeper.add(storage_node, config_file_path)
config["storage"]["shard"]["default"]["resync_metabase"] = True
storage_node.save_config(config, config_file_path)
with reporter.step("Start storage services on all nodes"):
with allure.step("Start storage services on all nodes"):
cluster_state_controller.start_all_stopped_services()
with reporter.step("Wait for tree rebalance"):
with allure.step("Wait for tree rebalance"):
# TODO: Use product metric when we have proper ones for this check
sleep(30)
with reporter.step("Delete objects from bucket"):
with reporter.step("Delete simple object from bucket"):
with allure.step("Delete objects from bucket"):
with allure.step("Delete simple object from bucket"):
with expect_not_raises():
s3_client.delete_object(bucket, simple_object_key)
with reporter.step("Delete complex object from bucket"):
with allure.step("Delete complex object from bucket"):
with expect_not_raises():
s3_client.delete_object(bucket, complex_object_key)
with reporter.step("Delete bucket"):
with allure.step("Delete bucket"):
with expect_not_raises():
s3_client.delete_bucket(bucket)
@ -509,12 +593,14 @@ class TestStorageDataLoss(ClusterTestBase):
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
shards_watcher: ShardsWatcher,
default_wallet: WalletInfo,
default_wallet: str,
test_start_time: datetime,
after_run_return_all_stopped_services: str,
):
exception_messages = []
allure.dynamic.description(after_run_return_all_stopped_services)
with reporter.step(f"Create container on node {node_under_test}"):
with allure.step(f"Create container on node {node_under_test}"):
locode = node_under_test.storage_node.get_un_locode()
placement_rule = f"""REP 1 IN X
CBF 1
@ -527,12 +613,12 @@ class TestStorageDataLoss(ClusterTestBase):
rule=placement_rule,
)
container = StorageContainer(
StorageContainerInfo(cid, default_wallet),
StorageContainerInfo(cid, WalletInfo(default_wallet)),
self.shell,
cluster_state_controller.cluster,
)
with reporter.step(f"Put couple objects to container on node {node_under_test}"):
with allure.step(f"Put couple objects to container on node {node_under_test}"):
storage_objects: list[StorageObjectInfo] = []
for _ in range(5):
storage_object = container.generate_object(
@ -541,48 +627,48 @@ class TestStorageDataLoss(ClusterTestBase):
)
storage_objects.append(storage_object)
with reporter.step("Take shards snapshot"):
with allure.step("Take shards snapshot"):
shards_watcher.take_shards_snapshot()
with reporter.step(f"Stop storage service on node {node_under_test}"):
with allure.step(f"Stop storage service on node {node_under_test}"):
cluster_state_controller.stop_service_of_type(node_under_test, StorageNode)
with reporter.step(f"Delete write cache from node {node_under_test}"):
with allure.step(f"Delete write cache from node {node_under_test}"):
node_under_test.storage_node.delete_write_cache()
with reporter.step(f"Start storage service on node {node_under_test}"):
with allure.step(f"Start storage service on node {node_under_test}"):
cluster_state_controller.start_all_stopped_services()
with reporter.step("Objects should be available"):
with allure.step("Objects should be available"):
for storage_object in storage_objects:
get_object(
storage_object.wallet,
storage_object.wallet_file_path,
container.get_id(),
storage_object.oid,
self.shell,
node_under_test.storage_node.get_rpc_endpoint(),
)
with reporter.step("No shards should have new errors"):
with allure.step("No shards should have new errors"):
shards_watcher.take_shards_snapshot()
shards_with_errors = shards_watcher.get_shards_with_new_errors()
if shards_with_errors:
exception_messages.append(f"Shards have new errors: {shards_with_errors}")
with reporter.step("No shards should have degraded status"):
with allure.step("No shards should have degraded status"):
snapshot = shards_watcher.get_shards_snapshot()
for shard in snapshot:
status = snapshot[shard]["mode"]
if status != "read-write":
exception_messages.append(f"Shard {shard} changed status to {status}")
with reporter.step("No related errors should be in log"):
with allure.step("No related errors should be in log"):
if node_under_test.host.is_message_in_logs(
message_regex=r"\Wno such file or directory\W", since=test_start_time
):
exception_messages.append(f"Node {node_under_test} have shard errors in logs")
with reporter.step("Pass test if no errors found"):
with allure.step("Pass test if no errors found"):
assert not exception_messages, "\n".join(exception_messages)
@allure.title(
@ -593,11 +679,12 @@ class TestStorageDataLoss(ClusterTestBase):
bucket,
s3_client: S3ClientWrapper,
simple_object_size: ObjectSize,
after_run_return_all_stopped_services,
cluster_state_controller: ClusterStateController,
):
# TODO: need to check that s3 gate is connected to localhost (such metric will be supported in 1.3)
with reporter.step("Stop one node and wait for rebalance connection of s3 gate to storage service"):
with allure.step("Stop one node and wait for rebalance connection of s3 gate to storage service"):
current_node = self.cluster.cluster_nodes[0]
cluster_state_controller.stop_service_of_type(current_node, StorageNode)
# waiting for rebalance connection of s3 gate to storage service
@ -605,7 +692,7 @@ class TestStorageDataLoss(ClusterTestBase):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into one bucket"):
with allure.step("Put object into one bucket"):
put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
@ -622,7 +709,7 @@ class TestStorageDataLoss(ClusterTestBase):
)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Check bucket versioning"):
with allure.step("Check bucket versioning"):
bucket_versioning = s3_client.get_bucket_versioning_status(bucket)
assert bucket_versioning == "Enabled", "Bucket should have enabled versioning"
@ -630,65 +717,59 @@ class TestStorageDataLoss(ClusterTestBase):
file_name = s3_helper.object_key_from_file_path(file_path)
object_versions = []
with reporter.step("Put object into one bucket"):
with allure.step("Put object into one bucket"):
put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
object_versions.append(put_object)
node_to_check = self.cluster.storage_nodes[0]
piloramas_list_before_removing = {}
with allure.step("Get list of all pilorama.db"):
piloramas_list_before_removing = self.get_piloramas_list(node_to_check)
piloramas_list_before_removing = []
with reporter.step("Get list of all pilorama.db on shards"):
for shard in node_to_check.get_shards():
piloramas_list_before_removing.append(shard.pilorama)
with reporter.step("Check that all pilorama.db files exist on node"):
for pilorama in piloramas_list_before_removing:
assert node_to_check.is_file_exist(pilorama), f"File {pilorama} does not exist"
with reporter.step("Stop all storage nodes"):
with allure.step("Stop all storage nodes"):
cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Delete pilorama.db from one node"):
for pilorama in piloramas_list_before_removing:
node_to_check.delete_file(pilorama)
with allure.step("Delete pilorama.db from one node"):
node_to_check.delete_pilorama()
with reporter.step("Start all storage nodes"):
with allure.step("Start all storage nodes"):
cluster_state_controller.start_all_stopped_services()
with reporter.step("Tick epoch to trigger sync and then wait for 1 minute"):
with allure.step("Tick epoch to trigger sync and then wait for 1 minute"):
self.tick_epochs(1)
sleep(120)
with reporter.step("Get list of all pilorama.db after sync"):
for pilorama in piloramas_list_before_removing:
assert node_to_check.is_file_exist(pilorama), f"File {pilorama} does not exist"
piloramas_list_afrer_removing = {}
with allure.step("Get list of all pilorama.db after sync"):
piloramas_list_afrer_removing = self.get_piloramas_list(node_to_check)
assert piloramas_list_afrer_removing == piloramas_list_before_removing, "List of pilorama.db is different"
with reporter.step("Check bucket versioning"):
with allure.step("Check bucket versioning"):
bucket_versioning = s3_client.get_bucket_versioning_status(bucket)
assert bucket_versioning == "Enabled", "Bucket should have enabled versioning"
with reporter.step("Check list objects"):
with allure.step("Check list objects"):
objects_list = s3_client.list_objects(bucket)
assert objects_list, f"Expected not empty bucket"
with reporter.step("Delete the object from the bucket"):
with allure.step("Delete the object from the bucket"):
delete_object = s3_client.delete_object(bucket, file_name)
assert "DeleteMarker" in delete_object.keys(), "Delete markers not found"
with reporter.step("Check list objects"):
with allure.step("Check list objects"):
objects_list = s3_client.list_objects_versions(bucket)
assert objects_list, f"Expected not empty bucket"
object_versions.append(delete_object["VersionId"])
# and now delete all versions of object (including Delete Markers)
with reporter.step("Delete all versions of the object from the bucket"):
with allure.step("Delete all versions of the object from the bucket"):
for version in object_versions:
delete_object = s3_client.delete_object(bucket, file_name, version_id=version)
with reporter.step("Check list objects"):
with allure.step("Check list objects"):
objects_list = s3_client.list_objects_versions(bucket)
assert not objects_list, f"Expected empty bucket"
with reporter.step("Delete bucket"):
with allure.step("Delete bucket"):
s3_client.delete_bucket(bucket)

View file

@ -1,553 +0,0 @@
import logging
import random
from time import sleep
from typing import Callable, Optional, Tuple
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.cli.netmap_parser import NetmapParser
from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container
from frostfs_testlib.steps.cli.object import (
delete_object,
get_object,
get_object_from_random_node,
head_object,
put_object,
put_object_to_random_node,
search_object,
)
from frostfs_testlib.steps.node_management import (
check_node_in_map,
delete_node_data,
drop_object,
exclude_node_from_network_map,
get_locode_from_random_node,
include_node_to_network_map,
node_shard_list,
node_shard_set_mode,
storage_node_set_status,
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeStatus
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import datetime_utils, string_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = []
@pytest.mark.node_mgmt
@pytest.mark.order(10)
class TestNodeManagement(ClusterTestBase):
@pytest.fixture
@allure.title("Create container and pick the node with data")
def create_container_and_pick_node(
self, default_wallet: WalletInfo, simple_object_size: ObjectSize
) -> Tuple[str, StorageNode]:
file_path = generate_file(simple_object_size.value)
placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
endpoint = self.cluster.default_rpc_endpoint
cid = create_container(
default_wallet,
shell=self.shell,
endpoint=endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, self.cluster)
nodes = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert len(nodes) == 1
node = nodes[0]
yield cid, node
shards = node_shard_list(node)
assert shards
for shard in shards:
node_shard_set_mode(node, shard, "read-write")
node_shard_list(node)
@reporter.step("Tick epoch with retries")
def tick_epoch_with_retries(self, attempts: int = 3, timeout: int = 3, wait_block: int = None):
for attempt in range(attempts):
try:
self.tick_epoch(wait_block=wait_block)
except RuntimeError:
sleep(timeout)
if attempt >= attempts - 1:
raise
continue
return
@pytest.fixture
def return_nodes_after_test_run(self):
yield
self.return_nodes()
@reporter.step("Return node to cluster")
def return_nodes(self, alive_node: Optional[StorageNode] = None) -> None:
for node in list(check_nodes):
with reporter.step(f"Start node {node}"):
node.start_service()
with reporter.step(f"Waiting status ready for node {node}"):
wait_for_node_to_be_ready(node)
# We need to wait for node to establish notifications from morph-chain
# Otherwise it will hang up when we will try to set status
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
with reporter.step(f"Move node {node} to online state"):
storage_node_set_status(node, status="online", retries=2)
check_nodes.remove(node)
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
self.tick_epoch_with_retries(3, wait_block=2)
check_node_in_map(node, shell=self.shell, alive_node=alive_node)
@allure.title("Add one node to cluster")
def test_add_nodes(
self,
default_wallet: WalletInfo,
simple_object_size: ObjectSize,
return_nodes_after_test_run,
):
"""
This test remove one node from frostfs_testlib.storage.cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
wallet = default_wallet
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
alive_node = random.choice(
[storage_node for storage_node in storage_nodes if storage_node.id != random_node.id]
)
check_node_in_map(random_node, shell=self.shell, alive_node=alive_node)
# Add node to recovery list before messing with it
check_nodes.append(random_node)
exclude_node_from_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
delete_node_data(random_node)
cid = create_container(
wallet,
rule=placement_rule_3,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
oid = put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
wait_object_replication(cid, oid, 3, shell=self.shell, nodes=storage_nodes)
self.return_nodes(alive_node)
with reporter.step("Check data could be replicated to new node"):
random_node = random.choice(list(set(storage_nodes) - {random_node, alive_node}))
# Add node to recovery list before messing with it
check_nodes.append(random_node)
exclude_node_from_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
wait_object_replication(
cid,
oid,
3,
shell=self.shell,
nodes=list(set(storage_nodes) - {random_node}),
)
include_node_to_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
wait_object_replication(cid, oid, 3, shell=self.shell, nodes=storage_nodes)
with reporter.step("Check container could be created with new node"):
cid = create_container(
wallet,
rule=placement_rule_4,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
oid = put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
wait_object_replication(cid, oid, 4, shell=self.shell, nodes=storage_nodes)
@allure.title("Drop object using control command")
def test_drop_object(self, default_wallet, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
Test checks object could be dropped using `frostfs-cli control drop-objects` command.
"""
wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint
file_path_simple = generate_file(simple_object_size.value)
file_path_complex = generate_file(complex_object_size.value)
locode = get_locode_from_random_node(self.cluster)
rule = f"REP 1 IN SE CBF 1 SELECT 1 FROM LOC AS SE FILTER 'UN-LOCODE' EQ '{locode}' AS LOC"
cid = create_container(wallet, rule=rule, shell=self.shell, endpoint=endpoint)
oid_simple = put_object_to_random_node(wallet, file_path_simple, cid, shell=self.shell, cluster=self.cluster)
oid_complex = put_object_to_random_node(wallet, file_path_complex, cid, shell=self.shell, cluster=self.cluster)
for oid in (oid_simple, oid_complex):
get_object_from_random_node(wallet, cid, oid, shell=self.shell, cluster=self.cluster)
head_object(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
nodes_with_object = get_nodes_with_object(cid, oid_simple, shell=self.shell, nodes=self.cluster.storage_nodes)
random_node = random.choice(nodes_with_object)
for oid in (oid_simple, oid_complex):
with reporter.step(f"Drop object {oid}"):
get_object_from_random_node(wallet, cid, oid, shell=self.shell, cluster=self.cluster)
head_object(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
drop_object(random_node, cid, oid)
self.wait_for_obj_dropped(wallet, cid, oid, endpoint, get_object)
self.wait_for_obj_dropped(wallet, cid, oid, endpoint, head_object)
@pytest.mark.skip(reason="Need to clarify scenario")
@allure.title("Control Operations with storage nodes")
def test_shards(
self,
default_wallet,
create_container_and_pick_node,
simple_object_size: ObjectSize,
):
wallet = default_wallet
file_path = generate_file(simple_object_size.value)
cid, node = create_container_and_pick_node
original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
# for mode in ('read-only', 'degraded'):
for mode in ("degraded",):
shards = node_shard_list(node)
assert shards
for shard in shards:
node_shard_set_mode(node, shard, mode)
shards = node_shard_list(node)
assert shards
with pytest.raises(RuntimeError):
put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
with pytest.raises(RuntimeError):
delete_object(wallet, cid, original_oid, self.shell, self.cluster.default_rpc_endpoint)
get_object_from_random_node(wallet, cid, original_oid, self.shell, self.cluster)
for shard in shards:
node_shard_set_mode(node, shard, "read-write")
shards = node_shard_list(node)
assert shards
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
delete_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
@allure.title("Put object with stopped node")
def test_stop_node(self, default_wallet, return_nodes_after_test_run, simple_object_size: ObjectSize):
wallet = default_wallet
placement_rule = "REP 3 IN X SELECT 4 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
alive_node = random.choice(
[storage_node for storage_node in storage_nodes if storage_node.id != random_node.id]
)
cid = create_container(
wallet,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=random_node.get_rpc_endpoint(),
)
with reporter.step("Stop the random node"):
check_nodes.append(random_node)
random_node.stop_service()
with reporter.step("Try to put an object and expect success"):
put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
self.return_nodes(alive_node)
@reporter.step("Wait for object to be dropped")
def wait_for_obj_dropped(self, wallet: str, cid: str, oid: str, endpoint: str, checker: Callable) -> None:
for _ in range(3):
try:
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
wait_for_gc_pass_on_storage_nodes()
except Exception as err:
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
return
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')
raise AssertionError(f"Object {oid} was not dropped from node")
@pytest.mark.maintenance
@pytest.mark.order(9)
class TestMaintenanceMode(ClusterTestBase):
@pytest.fixture()
@allure.title("Init Frostfs CLI remote")
def frostfs_cli_remote(self, node_under_test: ClusterNode) -> FrostfsCli:
host = node_under_test.host
service_config = host.get_service_config(node_under_test.storage_node.name)
wallet_path = service_config.attributes["wallet_path"]
wallet_password = service_config.attributes["wallet_password"]
shell = host.get_shell()
wallet_config_path = f"/tmp/{node_under_test.storage_node.name}-config.yaml"
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config_path)
return cli
@pytest.fixture()
@allure.title("Init Frostfs CLI remote")
def frostfs_cli(self, default_wallet: WalletInfo) -> FrostfsCli:
cli = FrostfsCli(
shell=self.shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=default_wallet.config_path
)
return cli
@pytest.fixture()
def restore_node_status(self, cluster_state_controller: ClusterStateController, default_wallet: WalletInfo):
nodes_to_restore = []
yield nodes_to_restore
for node_to_restore in nodes_to_restore:
cluster_state_controller.set_node_status(node_to_restore, default_wallet, NodeStatus.ONLINE)
def check_node_status(
self, expected_status: NodeStatus, node_under_test: ClusterNode, frostfs_cli: FrostfsCli, rpc_endpoint: str
):
netmap = frostfs_cli.netmap.snapshot(rpc_endpoint).stdout
all_snapshots = NetmapParser.snapshot_all_nodes(netmap)
node_snapshot = [snapshot for snapshot in all_snapshots if node_under_test.host_ip == snapshot.node]
if expected_status == NodeStatus.OFFLINE and not node_snapshot:
assert (
node_under_test.host_ip not in netmap
), f"{node_under_test} status should be {expected_status}. See netmap:\n{netmap}"
return
assert (
node_snapshot
), f"{node_under_test} status should be {expected_status}, but was not in netmap. See netmap:\n{netmap}"
node_snapshot = node_snapshot[0]
assert (
expected_status == node_snapshot.node_status
), f"{node_under_test} status should be {expected_status}, but was {node_snapshot.node_status}. See netmap:\n{netmap}"
@allure.title("Test of basic node operations in maintenance mode")
def test_maintenance_mode(
self,
default_wallet: WalletInfo,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
restore_node_status: list[ClusterNode],
):
with reporter.step("Create container and create\put object"):
cid = create_container(
wallet=default_wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule="REP 1 CBF 1",
)
nodes_with_container = search_nodes_with_container(
wallet=default_wallet,
cid=cid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
cluster=self.cluster,
)
node_under_test = nodes_with_container[0]
endpoint = node_under_test.storage_node.get_rpc_endpoint()
file_path = generate_file(simple_object_size.value)
oid = put_object(
wallet=default_wallet,
path=file_path,
cid=cid,
shell=self.shell,
endpoint=endpoint,
)
with reporter.step("Set node status to 'maintenance'"):
restore_node_status.append(node_under_test)
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)
node_under_maintenance_error = "node is under maintenance"
with reporter.step("Run basic operations with node in maintenance"):
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
get_object(default_wallet, cid, oid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
search_object(default_wallet, cid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
delete_object(default_wallet, cid, oid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
put_object(default_wallet, file_path, cid, self.shell, endpoint)
with reporter.step("Run basic operations with node not in maintenance"):
other_nodes = list(set(self.cluster.cluster_nodes) - set(nodes_with_container))
endpoint = other_nodes[0].storage_node.get_rpc_endpoint()
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
get_object(default_wallet, cid, oid, self.shell, endpoint)
search_object(default_wallet, cid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
delete_object(default_wallet, cid, oid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
put_object(default_wallet, file_path, cid, self.shell, endpoint)
@pytest.mark.sanity
@allure.title("MAINTENANCE and OFFLINE mode transitions")
def test_mode_transitions(
self,
cluster_state_controller: ClusterStateController,
node_under_test: ClusterNode,
default_wallet: WalletInfo,
frostfs_cli: FrostfsCli,
restore_node_status: list[ClusterNode],
):
restore_node_status.append(node_under_test)
alive_nodes = list(set(self.cluster.cluster_nodes) - {node_under_test})
alive_storage_node = alive_nodes[0].storage_node
alive_rpc_endpoint = alive_storage_node.get_rpc_endpoint()
with reporter.step("Set node status to 'offline'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.OFFLINE)
with reporter.step("Check node status is 'offline' after update the network map"):
self.check_node_status(NodeStatus.OFFLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Restart storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Check node status is 'online' after storage service restart"):
self.check_node_status(NodeStatus.ONLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Set node status to 'maintenance'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)
with reporter.step("Restart storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Check node staus is 'maintenance' after storage service restart"):
self.check_node_status(NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Set node status to 'offline'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.OFFLINE)
with reporter.step("Stop storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Start storage service"):
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Check node status is 'online' after storage service start"):
self.check_node_status(NodeStatus.ONLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Set node status to 'maintenance'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)
with reporter.step("Stop storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Start storage service"):
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Check node status is 'maintenance'"):
self.check_node_status(NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Check node status is 'maintenance'"):
self.check_node_status(NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, alive_rpc_endpoint)
@allure.title("A node cannot go into maintenance if maintenance is prohibited globally in the network")
def test_maintenance_globally_forbidden(
self,
cluster_state_controller: ClusterStateController,
node_under_test: ClusterNode,
frostfs_cli_remote: FrostfsCli,
default_wallet: WalletInfo,
restore_node_status: list[ClusterNode],
):
restore_node_status.append(node_under_test)
control_endpoint = node_under_test.service(StorageNode).get_control_endpoint()
with reporter.step("Set MaintenanceModeAllowed = false"):
cluster_state_controller.set_maintenance_mode_allowed("false", node_under_test)
with reporter.step("Set node status to 'maintenance'"):
with pytest.raises(RuntimeError, match="maintenance mode is not allowed by the network"):
frostfs_cli_remote.control.set_status(endpoint=control_endpoint, status="maintenance")
with reporter.step("Set MaintenanceModeAllowed = true"):
cluster_state_controller.set_maintenance_mode_allowed("true", node_under_test)
with reporter.step("Set node status to 'maintenance'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)

View file

@ -1,87 +0,0 @@
import math
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import delete_object, put_object_to_random_node
from frostfs_testlib.steps.metrics import check_metrics_counter
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.container
class TestContainerMetrics(ClusterTestBase):
@allure.title("Container metrics (obj_size={object_size})")
def test_container_metrics(
self, object_size: ObjectSize, max_object_size: int, default_wallet: WalletInfo, cluster: Cluster
):
file_path = generate_file(object_size.value)
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
copies = 2
object_chunks = 0
head_object = 1
link_object = 0
if object_size.value > max_object_size:
object_chunks = math.ceil(object_size.value / max_object_size)
link_object = 1
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
with reporter.step("Put object to random node"):
oid = put_object_to_random_node(
wallet=default_wallet,
path=file_path,
cid=cid,
shell=self.shell,
cluster=cluster,
)
with reporter.step("Get object nodes"):
object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
object_nodes = [
cluster_node
for cluster_node in cluster.cluster_nodes
if cluster_node.storage_node in object_storage_nodes
]
with reporter.step("Check metric appears in node where the object is located"):
count_metrics = (object_chunks + head_object + link_object) * copies
check_metrics_counter(
object_nodes, counter_exp=count_metrics, command="container_objects_total", cid=cid, type="phy"
)
check_metrics_counter(
object_nodes, counter_exp=count_metrics, command="container_objects_total", cid=cid, type="logic"
)
check_metrics_counter(
object_nodes, counter_exp=copies, command="container_objects_total", cid=cid, type="user"
)
with reporter.step("Delete file, wait until gc remove object"):
delete_object(default_wallet, cid, oid, self.shell, cluster.default_rpc_endpoint)
with reporter.step(f"Check container metrics 'the counter should equal {len(object_nodes)}' in object nodes"):
check_metrics_counter(
object_nodes, counter_exp=len(object_nodes), command="container_objects_total", cid=cid, type="phy"
)
check_metrics_counter(
object_nodes, counter_exp=len(object_nodes), command="container_objects_total", cid=cid, type="logic"
)
check_metrics_counter(object_nodes, counter_exp=0, command="container_objects_total", cid=cid, type="user")
with reporter.step("Check metrics(Phy, Logic, User) in each nodes"):
# Phy and Logic metrics are 4, because in rule 'CBF 2 SELECT 2 FROM', cbf2*sel2=4
check_metrics_counter(
cluster.cluster_nodes, counter_exp=4, command="container_objects_total", cid=cid, type="phy"
)
check_metrics_counter(
cluster.cluster_nodes, counter_exp=4, command="container_objects_total", cid=cid, type="logic"
)
check_metrics_counter(
cluster.cluster_nodes, counter_exp=0, command="container_objects_total", cid=cid, type="user"
)

View file

@ -1,115 +0,0 @@
import random
import re
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import delete_object, put_object, put_object_to_random_node
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.file_utils import generate_file
class TestGarbageCollectorMetrics(ClusterTestBase):
@wait_for_success(interval=10)
def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
counter_act = 0
try:
metric_result = cluster_node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
counter_act += self.calc_metrics_count_from_stdout(metric_result.stdout)
except RuntimeError as e:
...
assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"
@staticmethod
def calc_metrics_count_from_stdout(metric_result_stdout: str):
result = re.findall(r"}\s(\d+)", metric_result_stdout)
return sum(map(int, result))
@allure.title("Garbage collector expire_at object")
def test_garbage_collector_metrics_expire_at_object(
self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster
):
file_path = generate_file(simple_object_size.value)
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
metrics_step = 1
with reporter.step("Get current garbage collector metrics for each nodes"):
metrics_counter = {}
for node in cluster.cluster_nodes:
metrics_counter[node] = get_metrics_value(
node, command="frostfs_node_garbage_collector_marked_for_removal_objects_total"
)
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
with reporter.step("Put object to random node with expire_at"):
current_epoch = self.get_epoch()
oid = put_object_to_random_node(
default_wallet,
file_path,
cid,
self.shell,
cluster,
expire_at=current_epoch + 1,
)
with reporter.step("Get object nodes"):
object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
object_nodes = [
cluster_node
for cluster_node in cluster.cluster_nodes
if cluster_node.storage_node in object_storage_nodes
]
with reporter.step("Tick Epoch"):
self.tick_epochs(epochs_to_tick=2, wait_block=2)
with reporter.step(
f"Check garbage collector metrics 'the counter should increase by {metrics_step}' in object nodes"
):
for node in object_nodes:
metrics_counter[node] += metrics_step
for node, counter in metrics_counter.items():
check_metrics_counter(
[node],
counter_exp=counter,
command="frostfs_node_garbage_collector_marked_for_removal_objects_total",
)
@allure.title("Garbage collector delete object")
def test_garbage_collector_metrics_deleted_objects(
self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster
):
file_path = generate_file(simple_object_size.value)
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
metrics_step = 1
with reporter.step("Select random node"):
node = random.choice(cluster.cluster_nodes)
with reporter.step("Get current garbage collector metrics for selected node"):
metrics_counter = get_metrics_value(node, command="frostfs_node_garbage_collector_deleted_objects_total")
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
with reporter.step("Put object to selected node"):
oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step("Delete file, wait until gc remove object"):
delete_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"):
metrics_counter += metrics_step
check_metrics_counter(
[node], counter_exp=metrics_counter, command="frostfs_node_garbage_collector_deleted_objects_total"
)

View file

@ -1,224 +0,0 @@
import random
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.steps.cli.container import create_container, get_container, list_containers
from frostfs_testlib.steps.cli.object import get_object, head_object, put_object, search_object
from frostfs_testlib.steps.cli.tree import get_tree_list
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
class TestGRPCMetrics(ClusterTestBase):
@pytest.fixture
def disable_policer(self, cluster_state_controller: ClusterStateController):
config_manager = cluster_state_controller.manager(ConfigStateManager)
config_manager.set_on_all_nodes(StorageNode, {"policer:unsafe_disable": "true"})
yield
cluster_state_controller.manager(ConfigStateManager).revert_all()
@allure.title("GRPC metrics container operations")
def test_grpc_metrics_container_operations(self, default_wallet: WalletInfo, cluster: Cluster):
placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
with reporter.step("Select random node"):
node = random.choice(cluster.cluster_nodes)
with reporter.step("Get current gRPC metrics for method 'Put'"):
metrics_counter_put = get_metrics_value(
node, command="grpc_server_handled_total", service="ContainerService", method="Put"
)
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
metrics_counter_put += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_put,
command="grpc_server_handled_total",
service="ContainerService",
method="Put",
)
with reporter.step("Get current gRPC metrics for method 'Get'"):
metrics_counter_get = get_metrics_value(
node, command="grpc_server_handled_total", service="ContainerService", method="Get"
)
with reporter.step(f"Get container"):
get_container(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
metrics_counter_get += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_get,
command="grpc_server_handled_total",
service="ContainerService",
method="Get",
)
with reporter.step("Get current gRPC metrics for method 'List'"):
metrics_counter_list = get_metrics_value(
node, command="grpc_server_handled_total", service="ContainerService", method="List"
)
with reporter.step(f"Get container list"):
list_containers(default_wallet, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics method=List, 'the counter should increase by 1'"):
metrics_counter_list += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_list,
command="grpc_server_handled_total",
service="ContainerService",
method="List",
)
@allure.title("GRPC metrics object operations")
def test_grpc_metrics_object_operations(
self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, disable_policer
):
file_path = generate_file(simple_object_size.value)
placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
with reporter.step("Select random node"):
node = random.choice(cluster.cluster_nodes)
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
with reporter.step("Get current gRPC metrics for method 'Put'"):
metrics_counter_put = get_metrics_value(
node, command="grpc_server_handled_total", service="ObjectService", method="Put"
)
with reporter.step("Put object to selected node"):
oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
metrics_counter_put += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_put,
command="grpc_server_handled_total",
service="ObjectService",
method="Put",
)
with reporter.step("Get current gRPC metrics for method 'Get'"):
metrics_counter_get = get_metrics_value(
node, command="grpc_server_handled_total", service="ObjectService", method="Get"
)
with reporter.step(f"Get object"):
get_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
metrics_counter_get += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_get,
command="grpc_server_handled_total",
service="ObjectService",
method="Get",
)
with reporter.step("Get current gRPC metrics for method 'Search'"):
metrics_counter_search = get_metrics_value(
node, command="grpc_server_handled_total", service="ObjectService", method="Search"
)
with reporter.step(f"Search object"):
search_object(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics method=Search, 'the counter should increase by 1'"):
metrics_counter_search += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_search,
command="grpc_server_handled_total",
service="ObjectService",
method="Search",
)
with reporter.step("Get current gRPC metrics for method 'Head'"):
metrics_counter_head = get_metrics_value(
node, command="grpc_server_handled_total", service="ObjectService", method="Head"
)
with reporter.step(f"Head object"):
head_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics method=Head, 'the counter should increase by 1'"):
metrics_counter_head += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter_head,
command="grpc_server_handled_total",
service="ObjectService",
method="Head",
)
@allure.title("GRPC metrics Tree healthcheck")
def test_grpc_metrics_tree_service(self, cluster: Cluster, healthcheck: Healthcheck):
with reporter.step("Select random node"):
node = random.choice(cluster.cluster_nodes)
with reporter.step("Get current gRPC metrics for Healthcheck"):
metrics_counter = get_metrics_value(
node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck"
)
with reporter.step("Query Tree healthcheck status"):
healthcheck.tree_healthcheck(node)
with reporter.step(f"Check gRPC metrics for Healthcheck, 'the counter should increase'"):
check_metrics_counter(
[node],
">",
metrics_counter,
command="grpc_server_handled_total",
service="TreeService",
method="Healthcheck",
)
@allure.title("GRPC metrics Tree list")
def test_grpc_metrics_tree_list(self, default_wallet: WalletInfo, cluster: Cluster):
placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
with reporter.step("Select random node"):
node = random.choice(cluster.cluster_nodes)
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
with reporter.step("Get current gRPC metrics for Tree List"):
metrics_counter = get_metrics_value(
node, command="grpc_server_handled_total", service="TreeService", method="TreeList"
)
with reporter.step("Query Tree List"):
get_tree_list(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step(f"Check gRPC metrics for Tree List, 'the counter should increase by 1'"):
metrics_counter += 1
check_metrics_counter(
[node],
counter_exp=metrics_counter,
command="grpc_server_handled_total",
service="TreeService",
method="TreeList",
)

View file

@ -1,62 +0,0 @@
import random
import re
from datetime import datetime, timezone
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.metrics import get_metrics_value
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success
class TestLogsMetrics(ClusterTestBase):
@pytest.fixture
def restart_storage_service(self, cluster_state_controller: ClusterStateController) -> datetime:
config_manager = cluster_state_controller.manager(ConfigStateManager)
config_manager.csc.stop_services_of_type(StorageNode)
restart_time = datetime.now(timezone.utc)
config_manager.csc.start_services_of_type(StorageNode)
yield restart_time
cluster_state_controller.manager(ConfigStateManager).revert_all()
@wait_for_success(interval=10)
def check_metrics_in_node(self, cluster_node: ClusterNode, restart_time: datetime, log_priority: str = None, **metrics_greps):
counter_logs = self.get_count_logs_by_level(cluster_node, metrics_greps.get("level"), restart_time, log_priority)
counter_metrics = get_metrics_value(cluster_node, **metrics_greps)
assert counter_logs == counter_metrics, f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
@staticmethod
def get_count_logs_by_level(cluster_node: ClusterNode, log_level: str, after_time: datetime, log_priority: str):
count_logs = 0
try:
logs = cluster_node.host.get_filtered_logs(log_level, unit="frostfs-storage", since=after_time, priority=log_priority)
result = re.findall(rf"\s+{log_level}\s+", logs)
count_logs += len(result)
except RuntimeError as e:
...
return count_logs
@allure.title("Metrics for the log counter")
def test_log_counter_metrics(self, cluster: Cluster, restart_storage_service: datetime):
restart_time = restart_storage_service
with reporter.step("Select random node"):
node = random.choice(cluster.cluster_nodes)
with reporter.step(f"Check metrics count logs with level 'info'"):
self.check_metrics_in_node(
node,
restart_time,
log_priority="6..6",
command="frostfs_node_logger_entry_count",
level="info",
dropped="false",
)
with reporter.step(f"Check metrics count logs with level 'error'"):
self.check_metrics_in_node(node, restart_time, command="frostfs_node_logger_entry_count", level="error", dropped="false")

View file

@ -1,293 +0,0 @@
import random
import re
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.container import create_container, delete_container, search_nodes_with_container
from frostfs_testlib.steps.cli.object import delete_object, lock_object, put_object, put_object_to_random_node
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
class TestObjectMetrics(ClusterTestBase):
@allure.title("Object metrics of removed container (obj_size={object_size})")
def test_object_metrics_removed_container(self, object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster):
file_path = generate_file(object_size.value)
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
copies = 2
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
with reporter.step("Put object to random node"):
oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, cluster)
with reporter.step("Check metric appears in node where the object is located"):
object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
object_nodes = [cluster_node for cluster_node in cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes]
check_metrics_counter(
object_nodes,
counter_exp=copies,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
with reporter.step("Delete container"):
delete_container(default_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
with reporter.step("Tick Epoch"):
self.tick_epochs(epochs_to_tick=2, wait_block=2)
with reporter.step("Check metrics of removed containers doesn't appear in the storage node"):
check_metrics_counter(object_nodes, counter_exp=0, command="frostfs_node_engine_container_objects_total", cid=cid, type="user")
check_metrics_counter(object_nodes, counter_exp=0, command="frostfs_node_engine_container_size_byte", cid=cid)
for node in object_nodes:
all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte")
assert cid not in all_metrics.stdout, "metrics of removed containers shouldn't appear in the storage node"
@allure.title("Object metrics, locked object (obj_size={object_size}, policy={placement_policy})")
@pytest.mark.parametrize("placement_policy", ["REP 1 IN X CBF 1 SELECT 1 FROM * AS X", "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"])
def test_object_metrics_blocked_object(
self, object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, placement_policy: str
):
file_path = generate_file(object_size.value)
metric_step = int(re.search(r"REP\s(\d+)", placement_policy).group(1))
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
with reporter.step("Search container nodes"):
container_nodes = search_nodes_with_container(
wallet=default_wallet,
cid=cid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
cluster=cluster,
)
with reporter.step("Get current metrics for metric_type=user"):
objects_metric_counter = 0
for node in container_nodes:
objects_metric_counter += get_metrics_value(node, command="frostfs_node_engine_objects_total", type="user")
with reporter.step("Put object to container node"):
oid = put_object(default_wallet, file_path, cid, self.shell, container_nodes[0].storage_node.get_rpc_endpoint())
with reporter.step(f"Check metric user 'the counter should increase by {metric_step}'"):
objects_metric_counter += metric_step
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
check_metrics_counter(
container_nodes,
counter_exp=metric_step,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
with reporter.step("Delete object"):
delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step(f"Check metric user 'the counter should decrease by {metric_step}'"):
objects_metric_counter -= metric_step
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
check_metrics_counter(
container_nodes,
counter_exp=0,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
with reporter.step("Put object and lock it to next epoch"):
oid = put_object(default_wallet, file_path, cid, self.shell, container_nodes[0].storage_node.get_rpc_endpoint())
current_epoch = self.get_epoch()
lock_object(
default_wallet,
cid,
oid,
self.shell,
container_nodes[0].storage_node.get_rpc_endpoint(),
expire_at=current_epoch + 1,
)
with reporter.step(f"Check metric user 'the counter should increase by {metric_step}'"):
objects_metric_counter += metric_step
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
check_metrics_counter(
container_nodes,
counter_exp=metric_step,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
with reporter.step(f"Wait until remove locking 'the counter doesn't change'"):
self.tick_epochs(epochs_to_tick=2)
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
with reporter.step("Delete object"):
delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step(f"Check metric user 'the counter should decrease by {metric_step}'"):
objects_metric_counter -= metric_step
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
check_metrics_counter(
container_nodes,
counter_exp=0,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
with reporter.step("Put object with expire_at"):
current_epoch = self.get_epoch()
oid = put_object(
default_wallet,
file_path,
cid,
self.shell,
container_nodes[0].storage_node.get_rpc_endpoint(),
expire_at=current_epoch + 1,
)
with reporter.step(f"Check metric user 'the counter should increase by {metric_step}'"):
objects_metric_counter += metric_step
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
check_metrics_counter(
container_nodes,
counter_exp=metric_step,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
with reporter.step("Tick Epoch"):
self.tick_epochs(epochs_to_tick=2)
with reporter.step(f"Check metric user 'the counter should decrease by {metric_step}'"):
objects_metric_counter -= metric_step
check_metrics_counter(
container_nodes,
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
)
check_metrics_counter(
container_nodes,
counter_exp=0,
command="frostfs_node_engine_container_objects_total",
cid=cid,
type="user",
)
@allure.title("Object metrics, stop the node (obj_size={object_size})")
def test_object_metrics_stop_node(
self,
object_size: ObjectSize,
default_wallet: WalletInfo,
cluster_state_controller: ClusterStateController,
):
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
file_path = generate_file(object_size.value)
copies = 2
with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, placement_policy)
with reporter.step(f"Check object metrics in container 'should be zero'"):
check_metrics_counter(
self.cluster.cluster_nodes,
counter_exp=0,
command="frostfs_node_engine_container_objects_total",
type="user",
cid=cid,
)
with reporter.step("Get current metrics for each nodes"):
objects_metric_counter: dict[ClusterNode:int] = {}
for node in self.cluster.cluster_nodes:
objects_metric_counter[node] = get_metrics_value(node, command="frostfs_node_engine_objects_total", type="user")
with reporter.step("Put object"):
oid = put_object(default_wallet, file_path, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Get object nodes"):
object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, self.cluster.storage_nodes)
object_nodes = [
cluster_node for cluster_node in self.cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes
]
with reporter.step(f"Check metrics in object nodes 'the counter should increase by {copies}'"):
counter_exp = sum(objects_metric_counter[node] for node in object_nodes) + copies
check_metrics_counter(object_nodes, counter_exp=counter_exp, command="frostfs_node_engine_objects_total", type="user")
check_metrics_counter(
object_nodes,
counter_exp=copies,
command="frostfs_node_engine_container_objects_total",
type="user",
cid=cid,
)
with reporter.step(f"Select node to stop"):
node_to_stop = random.choice(object_nodes)
alive_nodes = set(object_nodes).difference({node_to_stop})
with reporter.step(f"Stop the node, wait until the object is replicated to another node"):
cluster_state_controller.stop_node_host(node_to_stop, "hard")
objects_metric_counter[node_to_stop] += 1
with reporter.step(f"Check metric in alive nodes 'the counter should increase'"):
counter_exp = sum(objects_metric_counter[node] for node in alive_nodes)
check_metrics_counter(alive_nodes, ">=", counter_exp, command="frostfs_node_engine_objects_total", type="user")
with reporter.step("Start node"):
cluster_state_controller.start_node_host(node_to_stop)
with reporter.step(f"Check metric in restarted node, 'the counter doesn't change'"):
check_metrics_counter(
object_nodes,
counter_exp=copies,
command="frostfs_node_engine_container_objects_total",
type="user",
cid=cid,
)

View file

@ -1,177 +0,0 @@
import random
import re
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import EACL_PUBLIC_READ_WRITE
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import get_object, put_object
from frostfs_testlib.steps.metrics import check_metrics_counter
from frostfs_testlib.steps.node_management import node_shard_list, node_shard_set_mode
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers import ShardsWatcher
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel, wait_for_success
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
class TestShardMetrics(ClusterTestBase):
@pytest.fixture()
@allure.title("Get two shards for set mode")
def two_shards_and_node(self, cluster: Cluster) -> tuple[str, str, ClusterNode]:
node = random.choice(cluster.cluster_nodes)
shards = node_shard_list(node.storage_node)
two_shards = random.sample(shards, k=2)
yield two_shards[0], two_shards[1], node
for shard in two_shards:
node_shard_set_mode(node.storage_node, shard, "read-write")
node_shard_list(node.storage_node)
@pytest.fixture()
@allure.title("Revert all shards mode")
def revert_all_shards_mode(self):
yield
parallel(self.set_shard_rw_mode, self.cluster.cluster_nodes)
def set_shard_rw_mode(self, node: ClusterNode):
watcher = ShardsWatcher(node)
shards = watcher.get_shards()
for shard in shards:
watcher.set_shard_mode(shard["shard_id"], mode="read-write")
watcher.await_for_all_shards_status(status="read-write")
@staticmethod
def get_error_count_from_logs(cluster_node: ClusterNode, object_path: str, object_name: str):
error_count = 0
try:
logs = cluster_node.host.get_filtered_logs("error count", unit="frostfs-storage")
# search error logs for current object
for error_line in logs.split("\n"):
if object_path in error_line and object_name in error_line:
result = re.findall(r'"error\scount":\s(\d+)', error_line)
error_count += sum(map(int, result))
except RuntimeError as e:
...
return error_count
@staticmethod
@wait_for_success(180, 30)
def get_object_path_and_name_file(oid: str, cid: str, node: ClusterNode) -> tuple[str, str]:
oid_path = f"{oid[0]}/{oid[1]}/{oid[2]}/{oid[3]}"
object_path = None
with reporter.step("Search object file"):
node_shell = node.storage_node.host.get_shell()
data_path = node.storage_node.get_data_directory()
all_datas = node_shell.exec(f"ls -la {data_path}/data | awk '{{ print $9 }}'").stdout.strip()
for data_dir in all_datas.replace(".", "").strip().split("\n"):
check_dir = node_shell.exec(
f" [ -d {data_path}/data/{data_dir}/data/{oid_path} ] && echo 1 || echo 0"
).stdout
if "1" in check_dir:
object_path = f"{data_path}/data/{data_dir}/data/{oid_path}"
object_name = f"{oid[4:]}.{cid}"
break
assert object_path is not None, f"{oid} object not found in directory - {data_path}/data"
return object_path, object_name
@allure.title("Metric for shard mode")
def test_shard_metrics_set_mode(self, two_shards_and_node: tuple[str, str, ClusterNode]):
metrics_counter = 1
shard1, shard2, node = two_shards_and_node
with reporter.step("Shard1 set to mode 'read-only'"):
node_shard_set_mode(node.storage_node, shard1, "read-only")
with reporter.step(f"Check shard metrics, 'the mode will change to 'READ_ONLY'"):
check_metrics_counter(
[node],
counter_exp=metrics_counter,
command="frostfs_node_engine_mode_info",
mode="READ_ONLY",
shard_id=shard1,
)
with reporter.step("Shard2 set to mode 'degraded-read-only'"):
node_shard_set_mode(node.storage_node, shard2, "degraded-read-only")
with reporter.step(f"Check shard metrics, 'the mode will change to 'DEGRADED_READ_ONLY'"):
check_metrics_counter(
[node],
counter_exp=metrics_counter,
command="frostfs_node_engine_mode_info",
mode="DEGRADED_READ_ONLY",
shard_id=shard2,
)
with reporter.step("Both shards set to mode 'read-write'"):
for shard in [shard1, shard2]:
node_shard_set_mode(node.storage_node, shard, "read-write")
with reporter.step(f"Check shard metrics, 'the mode will change to 'READ_WRITE'"):
for shard in [shard1, shard2]:
check_metrics_counter(
[node],
counter_exp=metrics_counter,
command="frostfs_node_engine_mode_info",
mode="READ_WRITE",
shard_id=shard,
)
@allure.title("Metric for error count on shard")
def test_shard_metrics_error_count(
self, max_object_size: int, default_wallet: WalletInfo, cluster: Cluster, revert_all_shards_mode
):
file_path = generate_file(round(max_object_size * 0.8))
with reporter.step(f"Create container"):
cid = create_container(
wallet=default_wallet,
shell=self.shell,
endpoint=cluster.default_rpc_endpoint,
rule="REP 1 CBF 1",
basic_acl=EACL_PUBLIC_READ_WRITE,
)
with reporter.step("Put object"):
oid = put_object(default_wallet, file_path, cid, self.shell, cluster.default_rpc_endpoint)
with reporter.step("Get object nodes"):
object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
object_nodes = [
cluster_node
for cluster_node in cluster.cluster_nodes
if cluster_node.storage_node in object_storage_nodes
]
node = random.choice(object_nodes)
with reporter.step("Search object in system."):
object_path, object_name = self.get_object_path_and_name_file(oid, cid, node)
with reporter.step("Block read file"):
node.host.get_shell().exec(f"chmod a-r {object_path}/{object_name}")
with reporter.step("Get object, expect error"):
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
get_object(
wallet=default_wallet,
cid=cid,
oid=oid,
shell=self.shell,
endpoint=node.storage_node.get_rpc_endpoint(),
)
with reporter.step(f"Get shard error count from logs"):
counter = self.get_error_count_from_logs(node, object_path, object_name)
with reporter.step(f"Check shard error metrics"):
check_metrics_counter([node], counter_exp=counter, command="frostfs_node_engine_errors_total")

View file

@ -0,0 +1,324 @@
import logging
import random
from time import sleep
from typing import Optional, Tuple
import allure
import pytest
from frostfs_testlib.resources.common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container, get_container
from frostfs_testlib.steps.cli.object import (
delete_object,
get_object,
get_object_from_random_node,
head_object,
put_object,
put_object_to_random_node,
)
from frostfs_testlib.steps.epoch import tick_epoch
from frostfs_testlib.steps.node_management import (
check_node_in_map,
delete_node_data,
drop_object,
exclude_node_from_network_map,
get_locode_from_random_node,
include_node_to_network_map,
node_shard_list,
node_shard_set_mode,
storage_node_healthcheck,
storage_node_set_status,
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.storage_policy import get_nodes_with_object, get_simple_object_copies
from frostfs_testlib.storage.cluster import StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import datetime_utils, string_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = []
@allure.title("Add one node to cluster")
@pytest.mark.add_nodes
@pytest.mark.node_mgmt
class TestNodeManagement(ClusterTestBase):
@pytest.fixture
@allure.title("Create container and pick the node with data")
def create_container_and_pick_node(
self, default_wallet: str, simple_object_size: ObjectSize
) -> Tuple[str, StorageNode]:
file_path = generate_file(simple_object_size.value)
placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
endpoint = self.cluster.default_rpc_endpoint
cid = create_container(
default_wallet,
shell=self.shell,
endpoint=endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, self.cluster)
nodes = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert len(nodes) == 1
node = nodes[0]
yield cid, node
shards = node_shard_list(node)
assert shards
for shard in shards:
node_shard_set_mode(node, shard, "read-write")
node_shard_list(node)
@allure.step("Tick epoch with retries")
def tick_epoch_with_retries(self, attempts: int = 3, timeout: int = 3, wait_block: int = None):
for attempt in range(attempts):
try:
self.tick_epoch(wait_block=wait_block)
except RuntimeError:
sleep(timeout)
if attempt >= attempts - 1:
raise
continue
return
@pytest.fixture
def after_run_start_all_nodes(self):
yield
self.return_nodes()
@pytest.fixture
def return_nodes_after_test_run(self):
yield
self.return_nodes()
@allure.step("Return node to cluster")
def return_nodes(self, alive_node: Optional[StorageNode] = None) -> None:
for node in list(check_nodes):
with allure.step(f"Start node {node}"):
node.start_service()
with allure.step(f"Waiting status ready for node {node}"):
wait_for_node_to_be_ready(node)
# We need to wait for node to establish notifications from morph-chain
# Otherwise it will hang up when we will try to set status
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
with allure.step(f"Move node {node} to online state"):
storage_node_set_status(node, status="online", retries=2)
check_nodes.remove(node)
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
self.tick_epoch_with_retries(3, wait_block=2)
check_node_in_map(node, shell=self.shell, alive_node=alive_node)
@allure.title("Add one node to cluster")
@pytest.mark.add_nodes
def test_add_nodes(
self,
default_wallet: str,
simple_object_size: ObjectSize,
return_nodes_after_test_run,
):
"""
This test remove one node from frostfs_testlib.storage.cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
wallet = default_wallet
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
alive_node = random.choice(
[storage_node for storage_node in storage_nodes if storage_node.id != random_node.id]
)
check_node_in_map(random_node, shell=self.shell, alive_node=alive_node)
# Add node to recovery list before messing with it
check_nodes.append(random_node)
exclude_node_from_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
delete_node_data(random_node)
cid = create_container(
wallet,
rule=placement_rule_3,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
oid = put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
wait_object_replication(cid, oid, 3, shell=self.shell, nodes=storage_nodes)
self.return_nodes(alive_node)
with allure.step("Check data could be replicated to new node"):
random_node = random.choice(list(set(storage_nodes) - {random_node, alive_node}))
# Add node to recovery list before messing with it
check_nodes.append(random_node)
exclude_node_from_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
wait_object_replication(
cid,
oid,
3,
shell=self.shell,
nodes=list(set(storage_nodes) - {random_node}),
)
include_node_to_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
wait_object_replication(cid, oid, 3, shell=self.shell, nodes=storage_nodes)
with allure.step("Check container could be created with new node"):
cid = create_container(
wallet,
rule=placement_rule_4,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
oid = put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
wait_object_replication(cid, oid, 4, shell=self.shell, nodes=storage_nodes)
@pytest.mark.node_mgmt
@allure.title("Drop object using control command")
def test_drop_object(self, default_wallet, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
Test checks object could be dropped using `frostfs-cli control drop-objects` command.
"""
wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint
file_path_simple = generate_file(simple_object_size.value)
file_path_complex = generate_file(complex_object_size.value)
locode = get_locode_from_random_node(self.cluster)
rule = f"REP 1 IN SE CBF 1 SELECT 1 FROM LOC AS SE FILTER 'UN-LOCODE' EQ '{locode}' AS LOC"
cid = create_container(wallet, rule=rule, shell=self.shell, endpoint=endpoint)
oid_simple = put_object_to_random_node(wallet, file_path_simple, cid, shell=self.shell, cluster=self.cluster)
oid_complex = put_object_to_random_node(wallet, file_path_complex, cid, shell=self.shell, cluster=self.cluster)
for oid in (oid_simple, oid_complex):
get_object_from_random_node(wallet, cid, oid, shell=self.shell, cluster=self.cluster)
head_object(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
nodes_with_object = get_nodes_with_object(cid, oid_simple, shell=self.shell, nodes=self.cluster.storage_nodes)
random_node = random.choice(nodes_with_object)
for oid in (oid_simple, oid_complex):
with allure.step(f"Drop object {oid}"):
get_object_from_random_node(wallet, cid, oid, shell=self.shell, cluster=self.cluster)
head_object(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
drop_object(random_node, cid, oid)
self.wait_for_obj_dropped(wallet, cid, oid, endpoint, get_object)
self.wait_for_obj_dropped(wallet, cid, oid, endpoint, head_object)
@pytest.mark.node_mgmt
@pytest.mark.skip(reason="Need to clarify scenario")
@allure.title("Control Operations with storage nodes")
def test_shards(
self,
default_wallet,
create_container_and_pick_node,
simple_object_size: ObjectSize,
):
wallet = default_wallet
file_path = generate_file(simple_object_size.value)
cid, node = create_container_and_pick_node
original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
# for mode in ('read-only', 'degraded'):
for mode in ("degraded",):
shards = node_shard_list(node)
assert shards
for shard in shards:
node_shard_set_mode(node, shard, mode)
shards = node_shard_list(node)
assert shards
with pytest.raises(RuntimeError):
put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
with pytest.raises(RuntimeError):
delete_object(wallet, cid, original_oid, self.shell, self.cluster.default_rpc_endpoint)
get_object_from_random_node(wallet, cid, original_oid, self.shell, self.cluster)
for shard in shards:
node_shard_set_mode(node, shard, "read-write")
shards = node_shard_list(node)
assert shards
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
delete_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
@pytest.mark.node_mgmt
@allure.title("Put object with stopped node")
def test_stop_node(self, default_wallet, return_nodes_after_test_run, simple_object_size: ObjectSize):
wallet = default_wallet
placement_rule = "REP 3 IN X SELECT 4 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
alive_node = random.choice(
[storage_node for storage_node in storage_nodes if storage_node.id != random_node.id]
)
cid = create_container(
wallet,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=random_node.get_rpc_endpoint(),
)
with allure.step("Stop the random node"):
check_nodes.append(random_node)
random_node.stop_service()
with allure.step("Try to put an object and expect success"):
put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
self.return_nodes(alive_node)
@allure.step("Wait for object to be dropped")
def wait_for_obj_dropped(self, wallet: str, cid: str, oid: str, endpoint: str, checker) -> None:
for _ in range(3):
try:
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
wait_for_gc_pass_on_storage_nodes()
except Exception as err:
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
return
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')
raise AssertionError(f"Object {oid} was not dropped from node")

View file

@ -4,34 +4,29 @@ import sys
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import (
INVALID_LENGTH_SPECIFIER,
INVALID_OFFSET_SPECIFIER,
INVALID_RANGE_OVERFLOW,
INVALID_RANGE_ZERO_LENGTH,
OBJECT_ALREADY_REMOVED,
OUT_OF_RANGE,
)
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import (
get_object_from_random_node,
get_range,
get_range_hash,
head_object,
put_object,
put_object_to_random_node,
search_object,
)
from frostfs_testlib.steps.complex_object_actions import get_complex_object_split_ranges
from frostfs_testlib.steps.storage_object import delete_object, delete_objects
from frostfs_testlib.steps.storage_object import delete_objects
from frostfs_testlib.steps.storage_policy import get_complex_object_copies, get_simple_object_copies
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_content, get_file_hash
@ -91,52 +86,23 @@ def generate_ranges(
return file_ranges_to_test
@pytest.fixture(scope="module")
def common_container(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> str:
rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
with reporter.step(f"Create container with {rule} and put object"):
cid = create_container(default_wallet, client_shell, cluster.default_rpc_endpoint, rule)
return cid
@pytest.fixture(scope="module")
def container_nodes(
default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, common_container: str
) -> list[ClusterNode]:
return search_nodes_with_container(
default_wallet, common_container, client_shell, cluster.default_rpc_endpoint, cluster
)
@pytest.fixture(scope="module")
def non_container_nodes(cluster: Cluster, container_nodes: list[ClusterNode]) -> list[ClusterNode]:
return list(set(cluster.cluster_nodes) - set(container_nodes))
@pytest.fixture(
# Scope session to upload/delete each files set only once
scope="module"
)
def storage_objects(
default_wallet: WalletInfo,
client_shell: Shell,
cluster: Cluster,
object_size: ObjectSize,
placement_policy: PlacementPolicy,
default_wallet: str, client_shell: Shell, cluster: Cluster, object_size: ObjectSize
) -> list[StorageObjectInfo]:
wallet = default_wallet
# Separate containers for complex/simple objects to avoid side-effects
cid = create_container(
wallet, shell=client_shell, rule=placement_policy.value, endpoint=cluster.default_rpc_endpoint
)
cid = create_container(wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
file_path = generate_file(object_size.value)
file_hash = get_file_hash(file_path)
storage_objects = []
with reporter.step("Put objects"):
with allure.step("Put objects"):
# We need to upload objects multiple times with different attributes
for attributes in OBJECT_ATTRIBUTES:
storage_object_id = put_object_to_random_node(
@ -150,7 +116,7 @@ def storage_objects(
storage_object = StorageObjectInfo(cid, storage_object_id)
storage_object.size = object_size.value
storage_object.wallet = wallet
storage_object.wallet_file_path = wallet
storage_object.file_path = file_path
storage_object.file_hash = file_hash
storage_object.attributes = attributes
@ -163,32 +129,24 @@ def storage_objects(
delete_objects(storage_objects, client_shell, cluster)
@pytest.fixture()
def expected_object_copies(placement_policy: PlacementPolicy) -> int:
if placement_policy.name == "rep":
return 2
return 4
@pytest.mark.sanity
@pytest.mark.grpc_api
class TestObjectApi(ClusterTestBase):
@allure.title("Storage policy by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Storage policy by native API (obj_size={object_size})")
def test_object_storage_policies(
self,
storage_objects: list[StorageObjectInfo],
simple_object_size: ObjectSize,
expected_object_copies: int,
):
"""
Validate object storage policy
"""
with reporter.step("Validate storage policy for objects"):
with allure.step("Validate storage policy for objects"):
for storage_object in storage_objects:
if storage_object.size == simple_object_size.value:
copies = get_simple_object_copies(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
shell=self.shell,
@ -196,24 +154,24 @@ class TestObjectApi(ClusterTestBase):
)
else:
copies = get_complex_object_copies(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
)
assert copies == expected_object_copies, f"Expected {expected_object_copies} copies"
assert copies == 2, "Expected 2 copies"
@allure.title("Get object by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Get object by native API (obj_size={object_size})")
def test_get_object_api(self, storage_objects: list[StorageObjectInfo]):
"""
Validate get object native API
"""
with reporter.step("Get objects and compare hashes"):
with allure.step("Get objects and compare hashes"):
for storage_object in storage_objects:
file_path = get_object_from_random_node(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -222,7 +180,7 @@ class TestObjectApi(ClusterTestBase):
file_hash = get_file_hash(file_path)
assert storage_object.file_hash == file_hash
@allure.title("Head object by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Head object by native API (obj_size={object_size})")
def test_head_object_api(self, storage_objects: list[StorageObjectInfo]):
"""
Validate head object native API
@ -231,16 +189,16 @@ class TestObjectApi(ClusterTestBase):
storage_object_1 = storage_objects[0]
storage_object_2 = storage_objects[1]
with reporter.step("Head object and validate"):
with allure.step("Head object and validate"):
head_object(
storage_object_1.wallet,
storage_object_1.wallet_file_path,
storage_object_1.cid,
storage_object_1.oid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
head_info = head_object(
storage_object_2.wallet,
storage_object_2.wallet_file_path,
storage_object_2.cid,
storage_object_2.oid,
shell=self.shell,
@ -248,32 +206,14 @@ class TestObjectApi(ClusterTestBase):
)
self.check_header_is_presented(head_info, storage_object_2.attributes)
@allure.title("Head deleted object with --raw arg (obj_size={object_size}, policy={placement_policy})")
def test_object_head_raw(self, default_wallet: str, object_size: ObjectSize, placement_policy: PlacementPolicy):
with reporter.step("Create container"):
cid = create_container(
default_wallet, self.shell, self.cluster.default_rpc_endpoint, placement_policy.value
)
with reporter.step("Upload object"):
file_path = generate_file(object_size.value)
oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, self.cluster)
with reporter.step("Delete object"):
delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Call object head --raw and expect error"):
with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED):
head_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint, is_raw=True)
@allure.title("Search objects by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Search objects by native API (obj_size={object_size})")
def test_search_object_api(self, storage_objects: list[StorageObjectInfo]):
"""
Validate object search by native API
"""
oids = [storage_object.oid for storage_object in storage_objects]
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
test_table = [
@ -282,7 +222,7 @@ class TestObjectApi(ClusterTestBase):
(COMMON_ATTRIBUTE, oids[1:3]),
]
with reporter.step("Search objects"):
with allure.step("Search objects"):
# Search with no attributes
result = search_object(
wallet,
@ -308,7 +248,7 @@ class TestObjectApi(ClusterTestBase):
assert sorted(expected_oids) == sorted(result)
@allure.title("Search objects with removed items (obj_size={object_size})")
def test_object_search_should_return_tombstone_items(self, default_wallet: WalletInfo, object_size: ObjectSize):
def test_object_search_should_return_tombstone_items(self, default_wallet: str, object_size: ObjectSize):
"""
Validate object search with removed items
"""
@ -316,7 +256,7 @@ class TestObjectApi(ClusterTestBase):
wallet = default_wallet
cid = create_container(wallet, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Upload file"):
with allure.step("Upload file"):
file_path = generate_file(object_size.value)
file_hash = get_file_hash(file_path)
@ -324,25 +264,25 @@ class TestObjectApi(ClusterTestBase):
cid=cid,
oid=put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster),
size=object_size.value,
wallet=wallet,
wallet_file_path=wallet,
file_path=file_path,
file_hash=file_hash,
)
with reporter.step("Search object"):
with allure.step("Search object"):
# Root Search object should return root object oid
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
assert result == [storage_object.oid]
with reporter.step("Delete file"):
with allure.step("Delete file"):
delete_objects([storage_object], self.shell, self.cluster)
with reporter.step("Search deleted object with --root"):
with allure.step("Search deleted object with --root"):
# Root Search object should return nothing
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
assert len(result) == 0
with reporter.step("Search deleted object with --phy should return only tombstones"):
with allure.step("Search deleted object with --phy should return only tombstones"):
# Physical Search object should return only tombstones
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True)
assert storage_object.tombstone in result, "Search result should contain tombstone of removed object"
@ -360,14 +300,14 @@ class TestObjectApi(ClusterTestBase):
object_type == "TOMBSTONE"
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
@allure.title("Get range hash by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Get range hash by native API (obj_size={object_size})")
@pytest.mark.grpc_api
def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size):
"""
Validate get_range_hash for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
@ -377,7 +317,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}"
with reporter.step(f"Get range hash ({range_cut})"):
with allure.step(f"Get range hash ({range_cut})"):
for oid in oids:
range_hash = get_range_hash(
wallet,
@ -391,14 +331,14 @@ class TestObjectApi(ClusterTestBase):
get_file_hash(file_path, range_len, range_start) == range_hash
), f"Expected range hash to match {range_cut} slice of file payload"
@allure.title("Get range by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Get range by native API (obj_size={object_size})")
@pytest.mark.grpc_api
def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size):
"""
Validate get_range for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
@ -408,7 +348,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}"
with reporter.step(f"Get range ({range_cut})"):
with allure.step(f"Get range ({range_cut})"):
for oid in oids:
_, range_content = get_range(
wallet,
@ -423,7 +363,7 @@ class TestObjectApi(ClusterTestBase):
== range_content
), f"Expected range content to match {range_cut} slice of file payload"
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size})")
@pytest.mark.grpc_api
def test_object_get_range_negatives(
self,
@ -433,7 +373,7 @@ class TestObjectApi(ClusterTestBase):
Validate get_range negative for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_size = storage_objects[0].size
@ -459,7 +399,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len, expected_error in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}"
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
with reporter.step(f"Get range ({range_cut})"):
with allure.step(f"Get range ({range_cut})"):
for oid in oids:
with pytest.raises(Exception, match=expected_error):
get_range(
@ -471,7 +411,7 @@ class TestObjectApi(ClusterTestBase):
range_cut=range_cut,
)
@allure.title("[NEGATIVE] Get invalid range hash by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("[NEGATIVE] Get invalid range hash by native API (obj_size={object_size})")
def test_object_get_range_hash_negatives(
self,
storage_objects: list[StorageObjectInfo],
@ -480,7 +420,7 @@ class TestObjectApi(ClusterTestBase):
Validate get_range_hash negative for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_size = storage_objects[0].size
@ -506,7 +446,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len, expected_error in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}"
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
with reporter.step(f"Get range hash ({range_cut})"):
with allure.step(f"Get range hash ({range_cut})"):
for oid in oids:
with pytest.raises(Exception, match=expected_error):
get_range_hash(
@ -518,80 +458,6 @@ class TestObjectApi(ClusterTestBase):
range_cut=range_cut,
)
@allure.title("Get range from container and non-container nodes (object_size={object_size})")
def test_get_range_from_different_node(
self,
default_wallet: str,
common_container: str,
container_nodes: list[ClusterNode],
non_container_nodes: list[ClusterNode],
file_path: str,
):
with reporter.step("Put object to container"):
container_node = random.choice(container_nodes)
oid = put_object(
default_wallet, file_path, common_container, self.shell, container_node.storage_node.get_rpc_endpoint()
)
with reporter.step("Get range from container node endpoint"):
get_range(
default_wallet,
common_container,
oid,
"0:10",
self.shell,
container_node.storage_node.get_rpc_endpoint(),
)
with reporter.step("Get range from non-container node endpoint"):
non_container_node = random.choice(non_container_nodes)
get_range(
default_wallet,
common_container,
oid,
"0:10",
self.shell,
non_container_node.storage_node.get_rpc_endpoint(),
)
@allure.title("Get range hash from container and non-container nodes (object_size={object_size})")
def test_get_range_hash_from_different_node(
self,
default_wallet: str,
common_container: str,
container_nodes: list[ClusterNode],
non_container_nodes: list[ClusterNode],
file_path: str,
):
with reporter.step("Put object to container"):
container_node = random.choice(container_nodes)
oid = put_object(
default_wallet, file_path, common_container, self.shell, container_node.storage_node.get_rpc_endpoint()
)
with reporter.step("Get range hash from container node endpoint"):
get_range_hash(
default_wallet,
common_container,
oid,
"0:10",
self.shell,
container_node.storage_node.get_rpc_endpoint(),
)
with reporter.step("Get range hash from non-container node endpoint"):
non_container_node = random.choice(non_container_nodes)
get_range_hash(
default_wallet,
common_container,
oid,
"0:10",
self.shell,
non_container_node.storage_node.get_rpc_endpoint(),
)
def check_header_is_presented(self, head_info: dict, object_header: dict) -> None:
for key_to_check, val_to_check in object_header.items():
assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}"

View file

@ -1,6 +1,5 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import EACL_PUBLIC_READ_WRITE
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.acl import form_bearertoken_file
@ -25,7 +24,7 @@ from pytest import FixtureRequest
@pytest.fixture(scope="module")
@allure.title("Create bearer token for OTHERS with all operations allowed for all containers")
def bearer_token_file_all_allow(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> str:
def bearer_token_file_all_allow(default_wallet: str, client_shell: Shell, cluster: Cluster) -> str:
bearer = form_bearertoken_file(
default_wallet,
"",
@ -40,7 +39,7 @@ def bearer_token_file_all_allow(default_wallet: WalletInfo, client_shell: Shell,
@pytest.fixture(scope="module")
@allure.title("Create user container for bearer token usage")
def user_container(
default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, request: FixtureRequest
default_wallet: str, client_shell: Shell, cluster: Cluster, request: FixtureRequest
) -> StorageContainer:
container_id = create_container(
default_wallet,
@ -94,17 +93,18 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
storage_objects: list[StorageObjectInfo],
bearer_token_file_all_allow: str,
):
s3_gate_wallet = WalletInfo.from_node(self.cluster.s3_gates[0])
with reporter.step("Try to delete each object from first storage node"):
s3_gate_wallet = self.cluster.s3_gates[0]
with allure.step("Try to delete each object from first storage node"):
for storage_object in storage_objects:
with expect_not_raises():
delete_object(
s3_gate_wallet,
s3_gate_wallet.get_wallet_path(),
storage_object.cid,
storage_object.oid,
self.shell,
endpoint=self.cluster.default_rpc_endpoint,
bearer=bearer_token_file_all_allow,
wallet_config=s3_gate_wallet.get_wallet_config_path(),
)
@allure.title("Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})")
@ -119,21 +119,22 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
object_size: ObjectSize,
bearer_token_file_all_allow: str,
):
s3_gate_wallet = WalletInfo.from_node(self.cluster.s3_gates[0])
with reporter.step("Put one object to container"):
s3_gate_wallet = self.cluster.s3_gates[0]
with allure.step("Put one object to container"):
epoch = self.get_epoch()
storage_object = user_container.generate_object(
object_size.value, epoch + 3, bearer_token=bearer_token_file_all_allow
)
with reporter.step("Try to fetch object from each storage node"):
with allure.step("Try to fetch object from each storage node"):
for node in self.cluster.storage_nodes:
with expect_not_raises():
get_object(
s3_gate_wallet,
s3_gate_wallet.get_wallet_path(),
storage_object.cid,
storage_object.oid,
self.shell,
endpoint=node.get_rpc_endpoint(),
bearer=bearer_token_file_all_allow,
wallet_config=s3_gate_wallet.get_wallet_config_path(),
)

View file

@ -2,13 +2,15 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node
from frostfs_testlib.steps.cli.object import (
get_object_from_random_node,
head_object,
put_object_to_random_node,
)
from frostfs_testlib.steps.epoch import get_epoch
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
@ -21,7 +23,7 @@ logger = logging.getLogger("NeoLogger")
@pytest.mark.grpc_api
class TestObjectApiLifetime(ClusterTestBase):
@allure.title("Object is removed when lifetime expired (obj_size={object_size})")
def test_object_api_lifetime(self, default_wallet: WalletInfo, object_size: ObjectSize):
def test_object_api_lifetime(self, default_wallet: str, object_size: ObjectSize):
"""
Test object deleted after expiration epoch.
"""
@ -34,29 +36,31 @@ class TestObjectApiLifetime(ClusterTestBase):
file_hash = get_file_hash(file_path)
epoch = get_epoch(self.shell, self.cluster)
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster, expire_at=epoch + 1)
oid = put_object_to_random_node(
wallet, file_path, cid, self.shell, self.cluster, expire_at=epoch + 1
)
got_file = get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster)
assert get_file_hash(got_file) == file_hash
with reporter.step("Tick two epochs"):
with allure.step("Tick two epochs"):
for _ in range(2):
self.tick_epoch()
# Wait for GC, because object with expiration is counted as alive until GC removes it
wait_for_gc_pass_on_storage_nodes()
with reporter.step("Check object deleted because it expires on epoch"):
with allure.step("Check object deleted because it expires on epoch"):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster)
with reporter.step("Tick additional epoch"):
with allure.step("Tick additional epoch"):
self.tick_epoch()
wait_for_gc_pass_on_storage_nodes()
with reporter.step("Check object deleted because it expires on previous epoch"):
with allure.step("Check object deleted because it expires on previous epoch"):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):

View file

@ -1,11 +1,8 @@
import logging
import re
from datetime import datetime
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.resources.common import STORAGE_GC_TIME
from frostfs_testlib.resources.error_patterns import (
LIFETIME_REQUIRED,
@ -27,7 +24,7 @@ from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import LockObjectInfo, StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
from frostfs_testlib.utils import datetime_utils
@ -40,20 +37,26 @@ FIXTURE_LOCK_LIFETIME = 5
FIXTURE_OBJECT_LIFETIME = 10
@pytest.fixture(scope="module")
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create user wallet with container"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
@pytest.fixture(
scope="module",
)
def user_wallet(wallet_factory: WalletFactory):
with allure.step("Create user wallet with container"):
wallet_file = wallet_factory.create_wallet()
return wallet_file
@pytest.fixture(scope="module")
@pytest.fixture(
scope="module",
)
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
container_id = create_container(user_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
container_id = create_container(user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
@pytest.fixture(scope="module")
@pytest.fixture(
scope="module",
)
def locked_storage_object(
user_container: StorageContainer,
client_shell: Shell,
@ -63,7 +66,7 @@ def locked_storage_object(
"""
Intention of this fixture is to provide storage object which is NOT expected to be deleted during test act phase
"""
with reporter.step("Creating locked object"):
with allure.step("Creating locked object"):
current_epoch = ensure_fresh_epoch(client_shell, cluster)
expiration_epoch = current_epoch + FIXTURE_LOCK_LIFETIME
@ -71,7 +74,7 @@ def locked_storage_object(
object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME
)
lock_object_id = lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
client_shell,
@ -84,17 +87,17 @@ def locked_storage_object(
yield storage_object
with reporter.step("Delete created locked object"):
with allure.step("Delete created locked object"):
current_epoch = get_epoch(client_shell, cluster)
epoch_diff = expiration_epoch - current_epoch + 1
if epoch_diff > 0:
with reporter.step(f"Tick {epoch_diff} epochs"):
with allure.step(f"Tick {epoch_diff} epochs"):
for _ in range(epoch_diff):
tick_epoch(client_shell, cluster)
try:
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
client_shell,
@ -109,10 +112,10 @@ def locked_storage_object(
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
def check_object_not_found(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(
wallet,
wallet_file_path,
cid,
oid,
shell,
@ -120,10 +123,10 @@ def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell,
)
def verify_object_available(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
def verify_object_available(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with expect_not_raises():
head_object(
wallet,
wallet_file_path,
cid,
oid,
shell,
@ -139,14 +142,14 @@ class TestObjectLockWithGrpc(ClusterTestBase):
Intention of this fixture is to provide new storage object for tests which may delete or corrupt the object or it's complementary objects
So we need a new one each time we ask for it
"""
with reporter.step("Creating locked object"):
with allure.step("Creating locked object"):
current_epoch = self.get_epoch()
storage_object = user_container.generate_object(
object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME
)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -166,7 +169,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
locked_storage_object.oid,
self.shell,
@ -185,7 +188,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
lock_object = locked_storage_object.locks[0]
wallet_path = locked_storage_object.wallet
wallet_path = locked_storage_object.wallet_file_path
with pytest.raises(Exception, match=LOCK_OBJECT_REMOVAL):
delete_object(
@ -208,7 +211,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
lock_object_info = locked_storage_object.locks[0]
wallet_path = locked_storage_object.wallet
wallet_path = locked_storage_object.wallet_file_path
with pytest.raises(Exception, match=LOCK_NON_REGULAR_OBJECT):
lock_object(
@ -248,7 +251,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
lock_object_info = locked_storage_object.locks[0]
wallet_path = locked_storage_object.wallet
wallet_path = locked_storage_object.wallet_file_path
with pytest.raises(Exception, match=expected_error):
lock_object(
@ -275,9 +278,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
with reporter.step("Lock object for couple epochs"):
with allure.step("Lock object for couple epochs"):
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -285,7 +288,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
lifetime=2,
)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -293,23 +296,23 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=current_epoch + 2,
)
with reporter.step("Check object is not deleted at expiration time"):
with allure.step("Check object is not deleted at expiration time"):
self.tick_epochs(2)
# Must wait to ensure object is not deleted
wait_for_gc_pass_on_storage_nodes()
with expect_not_raises():
head_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Wait for object to be deleted after third epoch"):
with allure.step("Wait for object to be deleted after third epoch"):
self.tick_epoch()
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -329,12 +332,12 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = ensure_fresh_epoch(self.shell, self.cluster)
storage_objects: list[StorageObjectInfo] = []
with reporter.step("Generate three objects"):
with allure.step("Generate three objects"):
for _ in range(3):
storage_objects.append(user_container.generate_object(object_size.value, expire_at=current_epoch + 5))
lock_object(
storage_objects[0].wallet,
storage_objects[0].wallet_file_path,
storage_objects[0].cid,
",".join([storage_object.oid for storage_object in storage_objects]),
self.shell,
@ -343,17 +346,17 @@ class TestObjectLockWithGrpc(ClusterTestBase):
)
for storage_object in storage_objects:
with reporter.step(f"Try to delete object {storage_object.oid}"):
with allure.step(f"Try to delete object {storage_object.oid}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Tick two epochs"):
with allure.step("Tick two epochs"):
self.tick_epoch()
self.tick_epoch()
@ -380,7 +383,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
match=LOCK_OBJECT_EXPIRATION.format(expiration_epoch=expiration_epoch, current_epoch=current_epoch),
):
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -404,7 +407,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -415,7 +418,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.tick_epochs(2)
with expect_not_raises():
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -438,7 +441,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -450,7 +453,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with expect_not_raises():
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -474,10 +477,10 @@ class TestObjectLockWithGrpc(ClusterTestBase):
chunk_object_ids = get_storage_object_chunks(locked_storage_object, self.shell, self.cluster)
for chunk_object_id in chunk_object_ids:
with reporter.step(f"Try to delete chunk object {chunk_object_id}"):
with allure.step(f"Try to delete chunk object {chunk_object_id}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
chunk_object_id,
self.shell,
@ -494,14 +497,14 @@ class TestObjectLockWithGrpc(ClusterTestBase):
)
def test_link_object_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
link_object_id = get_link_object(
new_locked_storage_object.wallet,
new_locked_storage_object.wallet_file_path,
new_locked_storage_object.cid,
new_locked_storage_object.oid,
self.shell,
self.cluster.storage_nodes,
)
with reporter.step(f"Drop link object with id {link_object_id} from nodes"):
with allure.step(f"Drop link object with id {link_object_id} from nodes"):
nodes_with_object = get_nodes_with_object(
new_locked_storage_object.cid,
link_object_id,
@ -524,7 +527,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
chunk_objects = get_storage_object_chunks(new_locked_storage_object, self.shell, self.cluster)
for chunk_object_id in chunk_objects:
with reporter.step(f"Drop chunk object with id {chunk_object_id} from nodes"):
with allure.step(f"Drop chunk object with id {chunk_object_id} from nodes"):
nodes_with_object = get_nodes_with_object(
new_locked_storage_object.cid,
chunk_object_id,
@ -565,17 +568,17 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
link_object_id = get_link_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
locked_storage_object.oid,
self.shell,
self.cluster.storage_nodes,
is_direct=False,
)
with reporter.step(f"Try to delete link object {link_object_id}"):
with allure.step(f"Try to delete link object {link_object_id}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
link_object_id,
self.shell,
@ -591,9 +594,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
with reporter.step("Apply first lock to object for 3 epochs"):
with allure.step("Apply first lock to object for 3 epochs"):
lock_object_id_0 = lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -603,18 +606,18 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.tick_epochs(2)
with reporter.step("Check first lock is still available"):
with allure.step("Check first lock is still available"):
verify_object_available(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
lock_object_id_0,
self.shell,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Apply second lock to object for 3 more epochs"):
with allure.step("Apply second lock to object for 3 more epochs"):
lock_object_id_1 = lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -624,27 +627,27 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.tick_epochs(2)
with reporter.step("Verify first lock is expired and removed"):
with allure.step("Verify first lock is expired and removed"):
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
lock_object_id_0,
self.shell,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Verify second lock is still available"):
with allure.step("Verify second lock is still available"):
verify_object_available(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
lock_object_id_1,
self.shell,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Apply third lock to object for 3 more epochs"):
with allure.step("Apply third lock to object for 3 more epochs"):
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -652,10 +655,10 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=current_epoch + 7,
)
with reporter.step("Verify object is deleted after all locks are expired"):
with allure.step("Verify object is deleted after all locks are expired"):
self.tick_epochs(4)
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -673,7 +676,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch()
storage_objects: list[StorageObjectInfo] = []
with reporter.step("Generate two objects"):
with allure.step("Generate two objects"):
for epoch_i in range(2):
storage_objects.append(
user_container.generate_object(object_size.value, expire_at=current_epoch + epoch_i + 3)
@ -681,9 +684,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.tick_epoch()
with reporter.step("Lock objects for 4 epochs"):
with allure.step("Lock objects for 4 epochs"):
lock_object(
storage_objects[0].wallet,
storage_objects[0].wallet_file_path,
storage_objects[0].cid,
",".join([storage_object.oid for storage_object in storage_objects]),
self.shell,
@ -691,24 +694,24 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=current_epoch + 4,
)
with reporter.step("Verify objects are available during next three epochs"):
with allure.step("Verify objects are available during next three epochs"):
for epoch_i in range(3):
self.tick_epoch()
with reporter.step(f"Check objects at epoch {current_epoch + epoch_i + 2}"):
with allure.step(f"Check objects at epoch {current_epoch + epoch_i + 2}"):
for storage_object in storage_objects:
verify_object_available(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Verify objects are deleted after lock was expired"):
with allure.step("Verify objects are deleted after lock was expired"):
self.tick_epoch()
for storage_object in storage_objects:
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,

View file

@ -1,578 +0,0 @@
import json
from dataclasses import dataclass
import allure
import pytest
import yaml
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli
from frostfs_testlib.cli.netmap_parser import NetmapParser
from frostfs_testlib.credentials.interfaces import User
from frostfs_testlib.resources.cli import FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import get_object, put_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeNetmapInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.cli_utils import parse_netmap_output
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from pytest_tests.resources.common import HOSTING_CONFIG_FILE
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if "ec_policy" not in metafunc.fixturenames:
return
with open(HOSTING_CONFIG_FILE, "r") as file:
hosting_config = yaml.full_load(file)
node_count = len(hosting_config["hosts"])
ec_map = {
4: ["EC 1.1", "EC 2.1", "EC 3.1"],
8: ["EC 5.3", "EC 3.2", "EC 7.1", "EC 4.4"],
16: ["EC 12.4", "EC 8.4", "EC 5.3", "EC 4.4"],
100: ["EC 12.4", "EC 8.4", "EC 5.3", "EC 4.4"],
}
metafunc.parametrize("ec_policy, node_count", ((ec_policy, node_count) for ec_policy in ec_map[node_count]))
@dataclass
class Chunk:
def __init__(self, object_id: str, required_nodes: list, confirmed_nodes: list, ec_parent_object_id: str, ec_index: int) -> None:
self.object_id = object_id
self.required_nodes = required_nodes
self.confirmed_nodes = confirmed_nodes
self.ec_parent_object_id = ec_parent_object_id
self.ec_index = ec_index
def __str__(self) -> str:
return self.object_id
@allure.title("Initialized local FrostfsCli")
@pytest.fixture()
def frostfs_local_cli(client_shell: Shell, default_user: User) -> FrostfsCli:
return FrostfsCli(client_shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=default_user.wallet.config_path)
@allure.title("Initialized remote FrostfsAdm")
@pytest.fixture
def frostfs_remote_adm(cluster: Cluster) -> FrostfsAdm:
node = cluster.cluster_nodes[0]
shell = node.host.get_shell()
return FrostfsAdm(shell, frostfs_adm_exec_path=FROSTFS_ADM_EXEC, config_file=FROSTFS_ADM_CONFIG_PATH)
@pytest.mark.replication
@pytest.mark.ec_replication
class TestECReplication(ClusterTestBase):
@allure.title("Restore chunk maximum params in network params ")
@pytest.fixture
def restore_network_config(self, frostfs_remote_adm: FrostfsAdm) -> None:
yield
frostfs_remote_adm.morph.set_config(set_key_value='"MaxECDataCount=12" "MaxECParityCount=5"')
@reporter.step("Get object nodes output ")
def get_object_nodes(self, cli: FrostfsCli, cid: str, oid: str, endpoint: str = None) -> dict:
if not endpoint:
endpoint = self.cluster.default_rpc_endpoint
return json.loads(cli.object.nodes(endpoint, cid, oid=oid, json=True).stdout)
@reporter.step("Get all chunks object ")
def get_all_chunks_object(self, cli: FrostfsCli, cid: str, oid: str, endpoint: str = None) -> list[Chunk]:
chunks = self.get_object_nodes(cli, cid, oid, endpoint)
return [Chunk(**chunk) for chunk in chunks["data_objects"]]
@reporter.step("Get parity chunk ")
def get_parity_chunk_object(self, cli: FrostfsCli, cid: str, oid: str, endpoint: str = None) -> Chunk:
chunks = self.get_object_nodes(cli, cid, oid, endpoint)["data_objects"]
return Chunk(**chunks[-1])
@reporter.step("Get data chunk ")
def get_data_chunk_object(self, cli: FrostfsCli, cid: str, oid: str, endpoint: str = None) -> Chunk:
chunks = self.get_object_nodes(cli, cid, oid, endpoint)["data_objects"]
return Chunk(**chunks[0])
@reporter.step("Search node without chunks ")
def search_node_not_chunks(self, chunks: list[Chunk], local_cli: FrostfsCli, endpoint: str = None) -> list[ClusterNode]:
if not endpoint:
self.cluster.default_rpc_endpoint
netmap = parse_netmap_output(local_cli.netmap.snapshot(endpoint).stdout)
chunks_node_key = []
for chunk in chunks:
chunks_node_key.extend(chunk.confirmed_nodes)
for node_info in netmap.copy():
if node_info.node_id in chunks_node_key and node_info in netmap:
netmap.remove(node_info)
result = []
for node_info in netmap:
for cluster_node in self.cluster.cluster_nodes:
if node_info.node == cluster_node.host_ip:
result.append(cluster_node)
return result
@reporter.step("Create container, policy={policy}")
def create_container(self, user_cli: FrostfsCli, endpoint: str, policy: str) -> str:
return user_cli.container.create(endpoint, policy=policy, await_mode=True).stdout.split(" ")[1].strip().split("\n")[0]
@reporter.step("Search node chunk {chunk}")
def get_chunk_node(self, frostfs_cli: FrostfsCli, chunk: Chunk) -> tuple[ClusterNode, NodeNetmapInfo]:
netmap = parse_netmap_output(frostfs_cli.netmap.snapshot(self.cluster.default_rpc_endpoint).stdout)
for node_info in netmap:
if node_info.node_id in chunk.confirmed_nodes:
for cluster_node in self.cluster.cluster_nodes:
if cluster_node.host_ip == node_info.node:
return (cluster_node, node_info)
@reporter.step("Check replication chunks={total_chunks} chunks ")
def check_replication(self, total_chunks: int, local_cli: FrostfsCli, cid: str, oid: str) -> bool:
object_nodes_info = local_cli.object.nodes(self.cluster.default_rpc_endpoint, cid, oid=oid, json=True).stdout
object_nodes_info = json.loads(object_nodes_info)
return len(object_nodes_info["data_objects"]) == total_chunks
@allure.title("Disable Policer on all nodes")
@pytest.fixture()
def disable_policer(
self,
cluster_state_controller: ClusterStateController,
) -> None:
with reporter.step(f"Disable policer for nodes"):
cluster_state_controller.manager(ConfigStateManager).set_on_all_nodes(
service_type=StorageNode, values={"policer": {"unsafe_disable": True}}
)
yield
with reporter.step(f"Enable policer for nodes"):
cluster_state_controller.start_stopped_hosts()
cluster_state_controller.manager(ConfigStateManager).revert_all()
@allure.title("Create container with EC policy (size={object_size.value})")
def test_create_container_with_ec_policy(
self,
default_user: User,
frostfs_local_cli: FrostfsCli,
object_size: ObjectSize,
) -> None:
test_file = generate_file(object_size.value)
rep_count = 3
if object_size.name == "complex":
rep_count *= 4
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 2.1")
with reporter.step("Put object in container."):
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check replication chunks."):
assert self.check_replication(rep_count, frostfs_local_cli, cid, oid)
@allure.title("Lose node with chunk data")
@pytest.mark.failover
def test_lose_node_with_data_chunk(
self,
frostfs_local_cli: FrostfsCli,
default_user: User,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
disable_policer: None,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 3.1")
with reporter.step("Put object in container."):
test_file = generate_file(simple_object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check chunk replication on 4 nodes."):
assert self.check_replication(4, frostfs_local_cli, cid, oid)
with reporter.step("Search node data chunk"):
chunk = self.get_data_chunk_object(frostfs_local_cli, cid, oid)
chunk_node = self.get_chunk_node(frostfs_local_cli, chunk)[0]
with reporter.step("Stop node with data chunk."):
cluster_state_controller.stop_node_host(chunk_node, "hard")
with reporter.step("Get object"):
node = list(set(self.cluster.cluster_nodes) - {chunk_node})[0]
get_object(default_user.wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step("Start stopped node, and check replication chunks."):
cluster_state_controller.start_node_host(chunk_node)
assert self.check_replication(4, frostfs_local_cli, cid, oid)
@allure.title("Lose node with chunk parity")
@pytest.mark.failover
def test_lose_node_with_parity_chunk(
self,
frostfs_local_cli: FrostfsCli,
default_user: User,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
disable_policer: None,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 3.1")
with reporter.step("Put object in container."):
test_file = generate_file(simple_object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check chunk replication on 4 nodes."):
assert self.check_replication(4, frostfs_local_cli, cid, oid)
with reporter.step("Search node with parity chunk"):
chunk = self.get_parity_chunk_object(frostfs_local_cli, cid, oid)
chunk_node = self.get_chunk_node(frostfs_local_cli, chunk)[0]
with reporter.step("Stop node parity chunk."):
cluster_state_controller.stop_node_host(chunk_node, "hard")
with reporter.step("Get object, expect success."):
node = list(set(self.cluster.cluster_nodes) - {chunk_node})[0]
get_object(default_user.wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step("Start stoped node, and check replication chunks."):
cluster_state_controller.start_node_host(chunk_node)
assert self.check_replication(4, frostfs_local_cli, cid, oid)
@allure.title("Lose nodes with chunk data and parity")
@pytest.mark.failover
def test_lose_nodes_data_chunk_and_parity(
self,
frostfs_local_cli: FrostfsCli,
default_user: User,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
disable_policer: None,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 3.1")
with reporter.step("Put object in container."):
test_file = generate_file(simple_object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check count chunks, expect 4."):
assert self.check_replication(4, frostfs_local_cli, cid, oid)
with reporter.step("Search node data chunk and node parity chunk"):
data_chunk = self.get_data_chunk_object(frostfs_local_cli, cid, oid)
node_data_chunk = self.get_chunk_node(frostfs_local_cli, data_chunk)[0]
parity_chunk = self.get_parity_chunk_object(frostfs_local_cli, cid, oid)
node_parity_chunk = self.get_chunk_node(frostfs_local_cli, parity_chunk)[0]
with reporter.step("Stop node with data chunk."):
cluster_state_controller.stop_node_host(node_data_chunk, "hard")
with reporter.step("Get object"):
node = list(set(self.cluster.cluster_nodes) - {node_data_chunk, node_parity_chunk})[0]
get_object(default_user.wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step("Start stopped host and check chunks."):
cluster_state_controller.start_node_host(node_data_chunk)
assert self.check_replication(4, frostfs_local_cli, cid, oid)
with reporter.step("Stop node with parity chunk and one all node."):
cluster_state_controller.stop_node_host(node_data_chunk, "hard")
cluster_state_controller.stop_node_host(node_parity_chunk, "hard")
with reporter.step("Get object, expect error."):
with pytest.raises(RuntimeError):
get_object(default_user.wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
with reporter.step("Start stopped nodes and check replication chunk."):
cluster_state_controller.start_stopped_hosts()
assert self.check_replication(4, frostfs_local_cli, cid, oid)
@allure.title("Policer work with chunk")
@pytest.mark.failover
def test_work_policer_with_nodes(
self,
simple_object_size: ObjectSize,
frostfs_local_cli: FrostfsCli,
default_user: User,
cluster_state_controller: ClusterStateController,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 2.1")
with reporter.step("Put object on container."):
test_file = generate_file(simple_object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check count chunks nodes on 3."):
assert self.check_replication(3, frostfs_local_cli, cid, oid)
with reporter.step("Stop node with chunk."):
data_chunk = self.get_data_chunk_object(frostfs_local_cli, cid, oid)
first_all_chunks = self.get_all_chunks_object(frostfs_local_cli, cid, oid)
node_data_chunk = self.get_chunk_node(frostfs_local_cli, data_chunk)[0]
cluster_state_controller.stop_node_host(node_data_chunk, "hard")
with reporter.step("Check replication chunk with different node."):
alive_endpoint = list(set(self.cluster.cluster_nodes) - {node_data_chunk})[0].storage_node.get_rpc_endpoint()
node = self.search_node_not_chunks(first_all_chunks, frostfs_local_cli, endpoint=alive_endpoint)[0]
second_all_chunks = self.get_all_chunks_object(frostfs_local_cli, cid, oid, node.storage_node.get_rpc_endpoint())
with reporter.step("Check that oid no change."):
oid_chunk_check = [chunk for chunk in second_all_chunks if data_chunk.object_id == chunk.object_id]
assert len(oid_chunk_check) > 0
with reporter.step("Start stopped host, and check delete 4 chunk."):
cluster_state_controller.start_node_host(node_data_chunk)
all_chunks_after_start_node = self.get_all_chunks_object(frostfs_local_cli, cid, oid)
assert len(all_chunks_after_start_node) == 3
@allure.title("EC X.Y combinations (nodes={node_count},policy={ec_policy},size={object_size.name})")
def test_create_container_with_difference_count_nodes(
self,
node_count: int,
ec_policy: str,
object_size: ObjectSize,
default_user: User,
frostfs_local_cli: FrostfsCli,
) -> None:
with reporter.step("Create container."):
expected_chunks = int(ec_policy.split(" ")[1].split(".")[0]) + int(ec_policy.split(" ")[1].split(".")[1])
if "complex" in object_size.name:
expected_chunks *= 4
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, ec_policy)
with reporter.step("Put object in container."):
test_file = generate_file(object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check count object chunks."):
chunks = self.get_all_chunks_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
assert len(chunks) == expected_chunks
with reporter.step("get object and check hash."):
file_with_node = get_object(default_user.wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
assert get_file_hash(test_file) == get_file_hash(file_with_node)
@allure.title("Request PUT with copies_number flag")
def test_put_object_with_copies_number(
self,
frostfs_local_cli: FrostfsCli,
default_user: User,
simple_object_size: ObjectSize,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 2.1")
with reporter.step("Put object in container with copies number = 1"):
test_file = generate_file(simple_object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint, copies_number=1)
with reporter.step("Check that count chunks > 1."):
chunks = self.get_all_chunks_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
assert len(chunks) > 1
@allure.title("Request PUT and 1 node off")
@pytest.mark.failover
def test_put_object_with_off_cnr_node(
self,
frostfs_local_cli: FrostfsCli,
cluster_state_controller: ClusterStateController,
default_user: User,
simple_object_size: ObjectSize,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 3.1")
with reporter.step("Stop one node in container nodes"):
cluster_state_controller.stop_node_host(self.cluster.cluster_nodes[1], "hard")
with reporter.step("Put object in container, expect error."):
test_file = generate_file(simple_object_size.value)
with pytest.raises(RuntimeError, match="put single object on client"):
put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
@allure.title("Request DELETE (size={object_size.name})")
@pytest.mark.failover
def test_delete_object_in_ec_cnr(
self,
default_user: User,
frostfs_local_cli: FrostfsCli,
object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 2.1")
with reporter.step("Put object in container."):
test_file = generate_file(object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check object chunks nodes."):
chunks_object = self.get_all_chunks_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
replication_count = 3 if object_size.name == "simple" else 3 * 4
assert len(chunks_object) == replication_count
with reporter.step("Delete object"):
frostfs_local_cli.object.delete(self.cluster.default_rpc_endpoint, cid, oid)
with reporter.step("Check that delete all chunks."):
for chunk in chunks_object:
with pytest.raises(RuntimeError, match="object already removed"):
frostfs_local_cli.object.head(self.cluster.default_rpc_endpoint, cid, chunk.object_id)
with reporter.step("Put second object."):
oid_second = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check second object chunks nodes."):
chunks_second_object = self.get_all_chunks_object(frostfs_local_cli, cid, oid_second, self.cluster.default_rpc_endpoint)
assert len(chunks_second_object) == replication_count
with reporter.step("Stop nodes with chunk."):
chunk_node = self.get_chunk_node(frostfs_local_cli, chunks_second_object[0])
cluster_state_controller.stop_node_host(chunk_node[0], "hard")
with reporter.step("Delete second object"):
frostfs_local_cli.object.delete(self.cluster.default_rpc_endpoint, cid, oid_second)
with reporter.step("Check that delete all chunk second object."):
for chunk in chunks_second_object:
with pytest.raises(RuntimeError, match="object already removed"):
frostfs_local_cli.object.head(self.cluster.default_rpc_endpoint, cid, chunk.object_id)
@allure.title("Request LOCK (size={object_size.name})")
@pytest.mark.failover
def test_lock_object_in_ec_cnr(
self,
frostfs_local_cli: FrostfsCli,
object_size: ObjectSize,
default_user: User,
cluster_state_controller: ClusterStateController,
) -> None:
with reporter.step("Create container."):
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, "EC 2.1")
with reporter.step("Put object in container."):
test_file = generate_file(object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check object chunks nodes."):
chunks_object = self.get_all_chunks_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
replication_count = 3 if object_size.name == "simple" else 3 * 4
assert len(chunks_object) == replication_count
with reporter.step("Put LOCK in object."):
epoch = frostfs_local_cli.netmap.epoch(self.cluster.default_rpc_endpoint).stdout.strip()
frostfs_local_cli.object.lock(self.cluster.default_rpc_endpoint, cid, oid, expire_at=int(epoch) + 5).stdout
with reporter.step("Check LOCK in object"):
chunks = frostfs_local_cli.object.head(self.cluster.default_rpc_endpoint, cid, oid, raw=True).stdout.strip().split(" ")
oids_chunks = [chunk.strip() for chunk in chunks if len(chunk) > 35]
for chunk_id in oids_chunks:
with pytest.raises(RuntimeError, match="could not delete objects"):
frostfs_local_cli.object.delete(self.cluster.default_rpc_endpoint, cid, chunk_id)
with reporter.step("Stop chunk node."):
chunk_node = self.get_chunk_node(frostfs_local_cli, chunks_object[0])
cluster_state_controller.stop_node_host(chunk_node[0], "hard")
cluster_state_controller.start_node_host(chunk_node[0])
with reporter.step("Check LOCK in object."):
chunks = self.get_all_chunks_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
for chunk_id in oids_chunks:
with pytest.raises(RuntimeError, match="could not delete objects"):
frostfs_local_cli.object.delete(self.cluster.default_rpc_endpoint, cid, chunk_id)
@allure.title("Output MaxEC* params in frostfscli (type={type_shards})")
@pytest.mark.parametrize("type_shards", ["Maximum count of data shards", "Maximum count of parity shards"])
def test_maxec_info_with_output_cli(self, frostfs_local_cli: FrostfsCli, type_shards: str) -> None:
with reporter.step("Get and check params"):
net_info = frostfs_local_cli.netmap.netinfo(self.cluster.default_rpc_endpoint).stdout
assert type_shards in net_info
@allure.title("Change MaxEC*Count params")
def test_change_max_data_shards_params(
self,
frostfs_remote_adm: FrostfsAdm,
frostfs_local_cli: FrostfsCli,
restore_network_config: None,
) -> None:
with reporter.step("Get now params MaxECDataCount and MaxECParityCount"):
node_netinfo = NetmapParser.netinfo(frostfs_local_cli.netmap.netinfo(self.cluster.default_rpc_endpoint).stdout)
with reporter.step("Change params"):
frostfs_remote_adm.morph.set_config(set_key_value='"MaxECDataCount=5" "MaxECParityCount=3"')
with reporter.step("Get update params"):
update_net_info = NetmapParser.netinfo(frostfs_local_cli.netmap.netinfo(self.cluster.default_rpc_endpoint).stdout)
with reporter.step("Check old and new params difference"):
assert (
update_net_info.maximum_count_of_data_shards not in node_netinfo.maximum_count_of_data_shards
and update_net_info.maximum_count_of_parity_shards not in node_netinfo.maximum_count_of_parity_shards
)
@allure.title("Check maximum count data and parity shards")
def test_change_over_max_parity_shards_params(
self,
frostfs_remote_adm: FrostfsAdm,
) -> None:
with reporter.step("Change over maximum params shards count."):
with pytest.raises(RuntimeError, match="MaxECDataCount and MaxECParityCount must be <= 256"):
frostfs_remote_adm.morph.set_config(set_key_value='"MaxECDataCount=130" "MaxECParityCount=130"')
@allure.title("Create container with EC policy and SELECT (SELECT={select})")
@pytest.mark.parametrize("select", [2, 4])
def test_create_container_with_select(
self,
select: int,
frostfs_local_cli: FrostfsCli,
) -> None:
with reporter.step("Create container"):
policy = f"EC 1.1 CBF 1 SELECT {select} FROM *"
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, policy)
with reporter.step("Check container nodes decomposed"):
container_nodes = frostfs_local_cli.container.search_node(self.cluster.default_rpc_endpoint, cid).stdout.strip().split("\n")[1:]
assert len(container_nodes) == select
@allure.title("Create container with EC policy and CBF (CBF={cbf})")
@pytest.mark.parametrize("cbf, expected_nodes", [(1, 2), (2, 4)])
def test_create_container_with_cbf(
self,
cbf: int,
expected_nodes: int,
frostfs_local_cli: FrostfsCli,
) -> None:
with reporter.step("Create container."):
policy = f"EC 1.1 CBF {cbf}"
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, policy)
with reporter.step("Check expected container nodes."):
container_nodes = frostfs_local_cli.container.search_node(self.cluster.default_rpc_endpoint, cid).stdout.strip().split("\n")[1:]
assert len(container_nodes) == expected_nodes
@allure.title("Create container with EC policy and FILTER")
def test_create_container_with_filter(
self,
default_user: User,
frostfs_local_cli: FrostfsCli,
simple_object_size: ObjectSize,
) -> None:
with reporter.step("Create Container."):
policy = "EC 1.1 IN RUS SELECT 2 FROM RU AS RUS FILTER Country EQ Russia AS RU"
cid = self.create_container(frostfs_local_cli, self.cluster.default_rpc_endpoint, policy)
with reporter.step("Put object in container."):
test_file = generate_file(simple_object_size.value)
oid = put_object(default_user.wallet, test_file, cid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Check object is decomposed exclusively on Russian nodes"):
data_chunk = self.get_data_chunk_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
parity_chunk = self.get_parity_chunk_object(frostfs_local_cli, cid, oid, self.cluster.default_rpc_endpoint)
node_data_chunk = self.get_chunk_node(frostfs_local_cli, data_chunk)
node_parity_chunk = self.get_chunk_node(frostfs_local_cli, parity_chunk)
for node in [node_data_chunk[1], node_parity_chunk[1]]:
assert "Russia" in node.country

View file

@ -1,16 +1,15 @@
import logging
import random
from time import sleep
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import head_object, put_object
from frostfs_testlib.steps.cli.container import create_container, delete_container
from frostfs_testlib.steps.cli.object import delete_object, head_object, put_object
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file
@ -25,10 +24,15 @@ WAIT_FOR_REPLICATION = 60
@pytest.mark.failover
@pytest.mark.replication
class TestReplication(ClusterTestBase):
@pytest.fixture(autouse=True)
def start_stopped_nodes_after_test(self, cluster_state_controller: ClusterStateController):
yield
cluster_state_controller.start_stopped_hosts()
@allure.title("Replication (obj_size={object_size})")
def test_replication(
self,
default_wallet: WalletInfo,
default_wallet: str,
client_shell: Shell,
cluster: Cluster,
object_size: ObjectSize,
@ -52,7 +56,7 @@ class TestReplication(ClusterTestBase):
file_path = generate_file(object_size.value)
with reporter.step("Put object"):
with allure.step("Put object"):
oid = put_object(
wallet=default_wallet,
path=file_path,
@ -65,7 +69,7 @@ class TestReplication(ClusterTestBase):
)
cluster_state_controller.start_node_host(node_for_rep)
with reporter.step(f"Wait for replication."):
with allure.step(f"Wait for replication."):
object_nodes = wait_object_replication(
cid=cid,
oid=oid,
@ -74,7 +78,7 @@ class TestReplication(ClusterTestBase):
nodes=self.cluster.storage_nodes,
)
with reporter.step("Check attributes"):
with allure.step("Check attributes"):
for node in object_nodes:
header_info = head_object(
wallet=default_wallet,
@ -93,19 +97,18 @@ class TestReplication(ClusterTestBase):
f"expected attribute value: {attribute_value}"
)
# TODO: Research why this fails
# with reporter.step("Cleanup"):
# delete_object(
# wallet=default_wallet,
# cid=cid,
# oid=oid,
# shell=client_shell,
# endpoint=cluster.default_rpc_endpoint,
# )
with allure.step("Cleanup"):
delete_object(
wallet=default_wallet,
cid=cid,
oid=oid,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
)
# delete_container(
# wallet=default_wallet,
# cid=cid,
# shell=client_shell,
# endpoint=cluster.default_rpc_endpoint,
# )
delete_container(
wallet=default_wallet,
cid=cid,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
)

View file

@ -2,7 +2,6 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.acl import (
bearer_token_base64_from_file,
@ -23,6 +22,7 @@ logger = logging.getLogger("NeoLogger")
@pytest.mark.http_gate
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@pytest.mark.http_put
class Test_http_bearer(ClusterTestBase):
PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
@ -44,7 +44,7 @@ class Test_http_bearer(ClusterTestBase):
@pytest.fixture(scope="class")
def eacl_deny_for_others(self, user_container: str) -> None:
with reporter.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"):
with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"):
eacl = EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT)
set_eacl(
self.wallet,
@ -57,7 +57,7 @@ class Test_http_bearer(ClusterTestBase):
@pytest.fixture(scope="class")
def bearer_token_no_limit_for_others(self, user_container: str) -> str:
with reporter.step(f"Create bearer token for {EACLRole.OTHERS} with all operations allowed"):
with allure.step(f"Create bearer token for {EACLRole.OTHERS} with all operations allowed"):
bearer = form_bearertoken_file(
self.wallet,
user_container,
@ -69,7 +69,7 @@ class Test_http_bearer(ClusterTestBase):
bearer_signed = f"{bearer}_signed"
sign_bearer(
shell=self.shell,
wallet=self.wallet,
wallet_path=self.wallet,
eacl_rules_file_from=bearer,
eacl_rules_file_to=bearer_signed,
json=False,
@ -88,6 +88,7 @@ class Test_http_bearer(ClusterTestBase):
error_pattern="access to object operation denied",
)
@pytest.mark.skip("Temp disable for v0.37")
def test_put_with_bearer_when_eacl_restrict(
self,
object_size: ObjectSize,
@ -98,7 +99,7 @@ class Test_http_bearer(ClusterTestBase):
eacl_deny_for_others
bearer = bearer_token_no_limit_for_others
file_path = generate_file(object_size.value)
with reporter.step(f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"):
with allure.step(f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"):
headers = [f" -H 'Authorization: Bearer {bearer}'"]
oid = upload_via_http_gate_curl(
cid=user_container,
@ -113,5 +114,5 @@ class Test_http_bearer(ClusterTestBase):
cid=user_container,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
)

View file

@ -1,6 +1,7 @@
import os
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node
@ -26,11 +27,11 @@ OBJECT_NOT_FOUND_ERROR = "not found"
@allure.link(
"https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#frostfs-http-gateway",
"https://github.com/TrueCloudLab/frostfs-http-gw#frostfs-http-gateway",
name="frostfs-http-gateway",
)
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@pytest.mark.sanity
@pytest.mark.http_gate
class TestHttpGate(ClusterTestBase):
@ -50,7 +51,7 @@ class TestHttpGate(ClusterTestBase):
Steps:
1. Create simple and large objects.
2. Put objects using gRPC (frostfs-cli).
3. Download objects using HTTP gate (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading).
3. Download objects using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading).
4. Get objects using gRPC (frostfs-cli).
5. Compare hashes for got objects.
6. Compare hashes for got and original objects.
@ -68,7 +69,7 @@ class TestHttpGate(ClusterTestBase):
file_path_simple = generate_file(simple_object_size.value)
file_path_large = generate_file(complex_object_size.value)
with reporter.step("Put objects using gRPC"):
with allure.step("Put objects using gRPC"):
oid_simple = put_object_to_random_node(
wallet=self.wallet,
path=file_path_simple,
@ -92,21 +93,23 @@ class TestHttpGate(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.link(
"https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#frostfs-http-gateway",
"https://github.com/TrueCloudLab/frostfs-http-gw#frostfs-http-gateway",
name="frostfs-http-gateway",
)
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
class TestHttpPut(ClusterTestBase):
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@allure.title("Put over HTTP, Get over HTTP")
@pytest.mark.smoke
def test_put_http_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
@ -115,8 +118,8 @@ class TestHttpPut(ClusterTestBase):
Steps:
1. Create simple and large objects.
2. Upload objects using HTTP (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading).
3. Download objects using HTTP gate (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading).
2. Upload objects using HTTP (https://github.com/TrueCloudLab/frostfs-http-gw#uploading).
3. Download objects using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading).
4. Compare hashes for got and original objects.
Expected result:
@ -132,7 +135,7 @@ class TestHttpPut(ClusterTestBase):
file_path_simple = generate_file(simple_object_size.value)
file_path_large = generate_file(complex_object_size.value)
with reporter.step("Put objects using HTTP"):
with allure.step("Put objects using HTTP"):
oid_simple = upload_via_http_gate(
cid=cid, path=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint
)
@ -148,13 +151,15 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.link(
"https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#by-attributes",
"https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes",
name="download by attributes",
)
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@allure.title("Put over HTTP, Get over HTTP with {id} header")
@pytest.mark.parametrize(
"attributes,id",
@ -172,7 +177,7 @@ class TestHttpPut(ClusterTestBase):
Steps:
1. Create simple and large objects.
2. Upload objects using HTTP with particular attributes in the header.
3. Download objects by attributes using HTTP gate (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#by-attributes).
3. Download objects by attributes using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes).
4. Compare hashes for got and original objects.
Expected result:
@ -187,7 +192,7 @@ class TestHttpPut(ClusterTestBase):
)
file_path = generate_file(simple_object_size.value)
with reporter.step("Put objects using HTTP with attribute"):
with allure.step("Put objects using HTTP with attribute"):
headers = attr_into_header(attributes)
oid = upload_via_http_gate(
cid=cid,
@ -201,7 +206,8 @@ class TestHttpPut(ClusterTestBase):
file_name=file_path,
cid=cid,
attrs=attributes,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Expiration-Epoch in HTTP header (epoch_gap={epoch_gap})")
@ -226,19 +232,25 @@ class TestHttpPut(ClusterTestBase):
valid_until = min_valid_epoch + gap_until
headers = {"X-Attribute-System-Expiration-Epoch": str(valid_until)}
with reporter.step("Put objects using HTTP with attribute Expiration-Epoch"):
with allure.step("Put objects using HTTP with attribute Expiration-Epoch"):
oid = upload_via_http_gate(
cid=cid,
path=file_path,
headers=headers,
endpoint=http_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
if get_epoch(self.shell, self.cluster) + 1 <= valid_until:
oids_to_be_valid.append(oid)
else:
oids_to_be_expired.append(oid)
with reporter.step("This object can be got"):
get_via_http_gate(cid=cid, oid=oid, node=self.cluster.cluster_nodes[0])
with allure.step("This object can be got"):
get_via_http_gate(
cid=cid,
oid=oid,
endpoint=http_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
self.tick_epoch()
@ -246,16 +258,22 @@ class TestHttpPut(ClusterTestBase):
wait_for_gc_pass_on_storage_nodes()
for oid in oids_to_be_expired:
with reporter.step(f"{oid} shall be expired and cannot be got"):
with allure.step(f"{oid} shall be expired and cannot be got"):
try_to_get_object_and_expect_error(
cid=cid,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern=OBJECT_NOT_FOUND_ERROR,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
for oid in oids_to_be_valid:
with reporter.step(f"{oid} shall be valid and can be got"):
get_via_http_gate(cid=cid, oid=oid, node=self.cluster.cluster_nodes[0])
with allure.step(f"{oid} shall be valid and can be got"):
get_via_http_gate(
cid=cid,
oid=oid,
endpoint=http_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Zip in HTTP header")
def test_zip_in_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
@ -286,9 +304,14 @@ class TestHttpPut(ClusterTestBase):
endpoint=self.cluster.default_http_gate_endpoint,
)
dir_path = get_via_zip_http_gate(cid=cid, prefix=common_prefix, node=self.cluster.cluster_nodes[0])
dir_path = get_via_zip_http_gate(
cid=cid,
prefix=common_prefix,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("Verify hashes"):
with allure.step("Verify hashes"):
assert get_file_hash(f"{dir_path}/file1") == get_file_hash(file_path_simple)
assert get_file_hash(f"{dir_path}/file2") == get_file_hash(file_path_large)
@ -309,7 +332,7 @@ class TestHttpPut(ClusterTestBase):
file_path = generate_file(complex_object_size.value)
with reporter.step("Put objects using HTTP"):
with allure.step("Put objects using HTTP"):
oid_gate = upload_via_http_gate(cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint)
oid_curl = upload_via_http_gate_curl(
cid=cid,
@ -324,7 +347,8 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
verify_object_hash(
oid=oid_curl,
@ -333,10 +357,12 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
object_getter=get_via_http_curl,
)
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@allure.title("Put/Get over HTTP using Curl utility")
def test_put_http_get_http_curl(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
@ -352,7 +378,7 @@ class TestHttpPut(ClusterTestBase):
file_path_simple = generate_file(simple_object_size.value)
file_path_large = generate_file(complex_object_size.value)
with reporter.step("Put objects using curl utility"):
with allure.step("Put objects using curl utility"):
oid_simple = upload_via_http_gate_curl(
cid=cid, filepath=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint
)
@ -370,6 +396,7 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
object_getter=get_via_http_curl,
)

View file

@ -3,7 +3,6 @@ import os
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import (
create_container,
@ -30,6 +29,7 @@ logger = logging.getLogger("NeoLogger")
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
class Test_http_headers(ClusterTestBase):
PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
obj1_keys = ["Writer", "Chapter1", "Chapter2"]
@ -69,7 +69,7 @@ class Test_http_headers(ClusterTestBase):
)
storage_object = StorageObjectInfo(cid, storage_object_id)
storage_object.size = os.path.getsize(file_path)
storage_object.wallet = wallet
storage_object.wallet_file_path = wallet
storage_object.file_path = file_path
storage_object.attributes = attributes
@ -88,7 +88,7 @@ class Test_http_headers(ClusterTestBase):
storage_object_1 = storage_objects_with_attributes[0]
with reporter.step(
with allure.step(
f'Download object#1 via wget with attributes Chapter2: {storage_object_1.attributes["Chapter2"]} and compare hashes'
):
get_object_by_attr_and_verify_hashes(
@ -96,7 +96,8 @@ class Test_http_headers(ClusterTestBase):
file_name=storage_object_1.file_path,
cid=storage_object_1.cid,
attrs={"Chapter2": storage_object_1.attributes["Chapter2"]},
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Get object2 with different attributes, then delete object2 and get object1")
@ -114,7 +115,7 @@ class Test_http_headers(ClusterTestBase):
storage_object_1 = storage_objects_with_attributes[0]
storage_object_2 = storage_objects_with_attributes[1]
with reporter.step(
with allure.step(
f'Download object#2 via wget with attributes [chapter2={storage_object_2.attributes["chapter2"]}] / [Ch@pter1={storage_object_2.attributes["Ch@pter1"]}] and compare hashes'
):
selected_attributes_object2 = [
@ -127,9 +128,10 @@ class Test_http_headers(ClusterTestBase):
file_name=storage_object_2.file_path,
cid=storage_object_2.cid,
attrs=attributes,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("Delete object#2 and verify is the container deleted"):
with allure.step("Delete object#2 and verify is the container deleted"):
delete_object(
wallet=self.wallet,
cid=storage_object_2.cid,
@ -140,12 +142,13 @@ class Test_http_headers(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=storage_object_2.cid,
oid=storage_object_2.oid,
node=self.cluster.cluster_nodes[0],
error_pattern=OBJECT_ALREADY_REMOVED_ERROR,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
storage_objects_with_attributes.remove(storage_object_2)
with reporter.step(
with allure.step(
f'Download object#1 with attributes [Writer={storage_object_1.attributes["Writer"]}] and compare hashes'
):
key_value_pair = {"Writer": storage_object_1.attributes["Writer"]}
@ -154,7 +157,8 @@ class Test_http_headers(ClusterTestBase):
file_name=storage_object_1.file_path,
cid=storage_object_1.cid,
attrs=key_value_pair,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("[NEGATIVE] Put object and get right after container is deleted")
@ -171,7 +175,7 @@ class Test_http_headers(ClusterTestBase):
"""
storage_object_1 = storage_objects_with_attributes[0]
with reporter.step(
with allure.step(
"[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]"
):
file_path_3 = generate_file(storage_object_1.size)
@ -186,7 +190,7 @@ class Test_http_headers(ClusterTestBase):
headers=headers,
error_pattern=error_pattern,
)
with reporter.step("Delete container and verify container deletion"):
with allure.step("Delete container and verify container deletion"):
delete_container(
wallet=self.wallet,
cid=storage_object_1.cid,
@ -204,14 +208,15 @@ class Test_http_headers(ClusterTestBase):
assert storage_object_1.cid not in list_containers(
self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
with reporter.step("[Negative] Try to download (wget) object via wget with attributes [peace=peace]"):
with allure.step("[Negative] Try to download (wget) object via wget with attributes [peace=peace]"):
request = f"/get/{storage_object_1.cid}/peace/peace"
error_pattern = "404 Not Found"
try_to_get_object_via_passed_request_and_expect_error(
cid=storage_object_1.cid,
oid="",
node=self.cluster.cluster_nodes[0],
error_pattern=error_pattern,
attrs=attrs_obj3,
http_request_path=request,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)

View file

@ -2,19 +2,14 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.steps.http.http_gate import (
assert_hashes_are_equal,
get_object_by_attr_and_verify_hashes,
get_via_http_gate,
try_to_get_object_via_passed_request_and_expect_error,
verify_object_hash,
)
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
@ -35,24 +30,24 @@ class Test_http_object(ClusterTestBase):
@allure.title("Put over gRPC, Get over HTTP with attributes (obj_size={object_size})")
def test_object_put_get_attributes(self, object_size: ObjectSize):
"""
Test that object can be put using gRPC interface and got using HTTP.
Test that object can be put using gRPC interface and get using HTTP.
Steps:
1. Create an object;
2. Put object(s) using gRPC (frostfs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
3. Download the object using HTTP gate (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading);
4. Compare hashes of the original and the downloaded object;
5. [Negative] Try to the get the object with the specified attributes and `get` request: [get/$CID/chapter1/peace];
6. Download the object with the specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace];
7. Compare hashes of the original and the downloaded object;
8. [Negative] Try to the get the object via `get_by_attribute` request: [get_by_attribute/$CID/$OID];
1. Create object;
2. Put objects using gRPC (frostfs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
3. Download object using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading);
4. Compare hashes between original and downloaded object;
5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace];
6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace];
7. Compare hashes between original and downloaded object;
8. [Negative] Try to the get object via `get_by_attribute` request: [get_by_attribute/$CID/$OID];
Expected result:
Hashes must be the same.
"""
with reporter.step("Create public container"):
with allure.step("Create public container"):
cid = create_container(
self.wallet,
shell=self.shell,
@ -74,7 +69,7 @@ class Test_http_object(ClusterTestBase):
key_value1 = obj_key1 + "=" + obj_value1
key_value2 = obj_key2 + "=" + obj_value2
with reporter.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"):
with allure.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"):
oid = put_object_to_random_node(
wallet=self.wallet,
path=file_path,
@ -83,7 +78,7 @@ class Test_http_object(ClusterTestBase):
cluster=self.cluster,
attributes=f"{key_value1},{key_value2}",
)
with reporter.step("Get object and verify hashes [ get/$CID/$OID ]"):
with allure.step("Get object and verify hashes [ get/$CID/$OID ]"):
verify_object_hash(
oid=oid,
file_name=file_path,
@ -91,69 +86,41 @@ class Test_http_object(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("[Negative] try to get object: [get/$CID/chapter1/peace]"):
with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"):
attrs = {obj_key1: obj_value1, obj_key2: obj_value2}
request = f"/get/{cid}/{obj_key1}/{obj_value1}"
expected_err_msg = "Failed to get object via HTTP gate:"
try_to_get_object_via_passed_request_and_expect_error(
cid=cid,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern=expected_err_msg,
http_request_path=request,
attrs=attrs,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("Download the object with attribute [get_by_attribute/$CID/chapter1/peace]"):
with allure.step(
"Download the object with attribute [get_by_attribute/$CID/chapter1/peace]"
):
get_object_by_attr_and_verify_hashes(
oid=oid,
file_name=file_path,
cid=cid,
attrs=attrs,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
request = f"/get_by_attribute/{cid}/{oid}"
try_to_get_object_via_passed_request_and_expect_error(
cid=cid,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern=expected_err_msg,
http_request_path=request,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Put over s3, Get over HTTP with bucket name and key")
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
def test_object_put_get_bucketname_key(self, object_size: ObjectSize, s3_client: S3ClientWrapper):
"""
Test that object can be put using s3-gateway interface and got via HTTP with bucket name and object key.
Steps:
1. Create an object;
2. Create a bucket via s3;
3. Put the object via s3;
4. Download the object using HTTP gate with the bucket name and the object key;
5. Compare hashes of the original and the downloaded objects;
Expected result:
Hashes must be the same.
"""
file_path = generate_file(object_size.value)
object_key = s3_helper.object_key_from_file_path(file_path)
bucket = s3_client.create_bucket(acl="public-read-write")
s3_client.put_object(bucket=bucket, filepath=file_path, key=object_key)
obj_s3 = s3_client.get_object(bucket=bucket, key=object_key)
request = f"/get/{bucket}/{object_key}"
obj_http = get_via_http_gate(
cid=None,
oid=None,
node=self.cluster.cluster_nodes[0],
request_path=request,
)
with reporter.step("Verify hashes"):
assert_hashes_are_equal(file_path, obj_http, obj_s3)

View file

@ -2,7 +2,6 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.http.http_gate import upload_via_http_gate_curl, verify_object_hash
@ -15,6 +14,7 @@ logger = logging.getLogger("NeoLogger")
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
class Test_http_streaming(ClusterTestBase):
PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
@ -31,13 +31,13 @@ class Test_http_streaming(ClusterTestBase):
Steps:
1. Create big object;
2. Put object using curl with pipe (streaming);
3. Download object using HTTP gate (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading);
3. Download object using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading);
4. Compare hashes between original and downloaded object;
Expected result:
Hashes must be the same.
"""
with reporter.step("Create public container and verify container creation"):
with allure.step("Create public container and verify container creation"):
cid = create_container(
self.wallet,
shell=self.shell,
@ -45,11 +45,11 @@ class Test_http_streaming(ClusterTestBase):
rule=self.PLACEMENT_RULE,
basic_acl=PUBLIC_ACL,
)
with reporter.step("Allocate big object"):
with allure.step("Allocate big object"):
# Generate file
file_path = generate_file(complex_object_size.value)
with reporter.step("Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"):
with allure.step("Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"):
oid = upload_via_http_gate_curl(
cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint
)
@ -60,5 +60,6 @@ class Test_http_streaming(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)

View file

@ -5,7 +5,6 @@ from typing import Optional
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
@ -34,6 +33,7 @@ SYSTEM_EXPIRATION_RFC3339 = "System-Expiration-RFC3339"
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
class Test_http_system_header(ClusterTestBase):
PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
@ -123,7 +123,8 @@ class Test_http_system_header(ClusterTestBase):
cid=user_container,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
head = head_object(
wallet=self.wallet,
@ -138,7 +139,7 @@ class Test_http_system_header(ClusterTestBase):
def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)})
file_path = generate_file(simple_object_size.value)
with reporter.step("Put object using HTTP with attribute Expiration-Epoch where epoch is expired"):
with allure.step("Put object using HTTP with attribute Expiration-Epoch where epoch is expired"):
upload_via_http_gate_curl(
cid=user_container,
filepath=file_path,
@ -151,9 +152,7 @@ class Test_http_system_header(ClusterTestBase):
def test_unable_put_negative_duration(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"})
file_path = generate_file(simple_object_size.value)
with reporter.step(
"Put object using HTTP with attribute System-Expiration-Duration where duration is negative"
):
with allure.step("Put object using HTTP with attribute System-Expiration-Duration where duration is negative"):
upload_via_http_gate_curl(
cid=user_container,
filepath=file_path,
@ -166,7 +165,7 @@ class Test_http_system_header(ClusterTestBase):
def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"})
file_path = generate_file(simple_object_size.value)
with reporter.step(
with allure.step(
"Put object using HTTP with attribute System-Expiration-Timestamp where duration is in the past"
):
upload_via_http_gate_curl(
@ -192,6 +191,7 @@ class Test_http_system_header(ClusterTestBase):
)
@allure.title("Priority of attributes epoch>duration (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_epoch_duration(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self.tick_epoch()
epoch_count = 1
@ -201,33 +201,35 @@ class Test_http_system_header(ClusterTestBase):
)
attributes = {SYSTEM_EXPIRATION_EPOCH: expected_epoch, SYSTEM_EXPIRATION_DURATION: "1m"}
file_path = generate_file(object_size.value)
with reporter.step(
with allure.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
):
oid, head_info = self.oid_header_info_for_object(
file_path=file_path, attributes=attributes, user_container=user_container
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"):
with allure.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1):
self.tick_epoch()
assert (
get_epoch(self.shell, self.cluster) == expected_epoch + 1
), f"Epochs should be equal: {get_epoch(self.shell, self.cluster)} != {expected_epoch + 1}"
with reporter.step("Check object deleted because it expires-on epoch"):
with allure.step("Check object deleted because it expires-on epoch"):
wait_for_epochs_align(self.shell, self.cluster)
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
@allure.title("Priority of attributes duration>timestamp (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_dur_timestamp(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self.tick_epoch()
epoch_count = 2
@ -240,33 +242,35 @@ class Test_http_system_header(ClusterTestBase):
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=1),
}
file_path = generate_file(object_size.value)
with reporter.step(
with allure.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
):
oid, head_info = self.oid_header_info_for_object(
file_path=file_path, attributes=attributes, user_container=user_container
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"):
with allure.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1):
self.tick_epoch()
assert (
get_epoch(self.shell, self.cluster) == expected_epoch + 1
), f"Epochs should be equal: {get_epoch(self.shell, self.cluster)} != {expected_epoch + 1}"
with reporter.step("Check object deleted because it expires-on epoch"):
with allure.step("Check object deleted because it expires-on epoch"):
wait_for_epochs_align(self.shell, self.cluster)
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
@allure.title("Priority of attributes timestamp>Expiration-RFC (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_timestamp_rfc(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self.tick_epoch()
epoch_count = 2
@ -281,27 +285,28 @@ class Test_http_system_header(ClusterTestBase):
),
}
file_path = generate_file(object_size.value)
with reporter.step(
with allure.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
):
oid, head_info = self.oid_header_info_for_object(
file_path=file_path, attributes=attributes, user_container=user_container
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"):
with allure.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1):
self.tick_epoch()
assert (
get_epoch(self.shell, self.cluster) == expected_epoch + 1
), f"Epochs should be equal: {get_epoch(self.shell, self.cluster)} != {expected_epoch + 1}"
with reporter.step("Check object deleted because it expires-on epoch"):
with allure.step("Check object deleted because it expires-on epoch"):
wait_for_epochs_align(self.shell, self.cluster)
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@ -329,7 +334,7 @@ class Test_http_system_header(ClusterTestBase):
)
}
file_path = generate_file(object_size.value)
with reporter.step(
with allure.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
):
oid, head_info = self.oid_header_info_for_object(
@ -338,7 +343,7 @@ class Test_http_system_header(ClusterTestBase):
user_container=user_container,
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"):
with allure.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1):
self.tick_epoch()
# check that {EXPIRATION_EXPIRATION_RFC} absents in header output
@ -346,13 +351,14 @@ class Test_http_system_header(ClusterTestBase):
get_epoch(self.shell, self.cluster) == expected_epoch + 1
), f"Epochs should be equal: {get_epoch(self.shell, self.cluster)} != {expected_epoch + 1}"
with reporter.step("Check object deleted because it expires-on epoch"):
with allure.step("Check object deleted because it expires-on epoch"):
wait_for_epochs_align(self.shell, self.cluster)
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):

View file

@ -1,9 +1,6 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import S3_BUCKET_DOES_NOT_ALLOW_ACL
from frostfs_testlib.resources.s3_acl_grants import PRIVATE_GRANTS, PUBLIC_READ_GRANTS, PUBLIC_READ_WRITE_GRANTS
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file
@ -13,51 +10,52 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.s3_gate
class TestS3GateACL:
@allure.title("Object ACL (s3_client={s3_client})")
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
def test_s3_object_ACL(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into bucket"):
with allure.step("Put object into bucket, Check ACL is empty"):
s3_client.put_object(bucket, file_path)
obj_acl = s3_client.get_object_acl(bucket, file_name)
assert obj_acl == [], f"Expected ACL is empty, got {obj_acl}"
with reporter.step("Verify private ACL is default"):
object_grants = s3_client.get_object_acl(bucket, file_name)
s3_helper.verify_acl_permissions(object_grants, PRIVATE_GRANTS)
with allure.step("Put object ACL = public-read"):
s3_client.put_object_acl(bucket, file_name, "public-read")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
with reporter.step("Verify put object ACL is restricted"):
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
object_grants = s3_client.put_object_acl(bucket, file_name, acl="public-read")
with allure.step("Put object ACL = private"):
s3_client.put_object_acl(bucket, file_name, "private")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
@allure.title("Create Bucket with different ACL (s3_client={s3_client})")
def test_s3_create_bucket_with_ACL(self, s3_client: S3ClientWrapper):
with reporter.step("Create bucket with ACL private"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="private")
bucket_grants = s3_client.get_bucket_acl(bucket)
s3_helper.verify_acl_permissions(bucket_grants, PRIVATE_GRANTS)
with reporter.step("Create bucket with ACL public-read"):
read_bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read")
bucket_grants = s3_client.get_bucket_acl(read_bucket)
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_GRANTS)
with reporter.step("Create bucket with ACL public-read-write"):
public_rw_bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_grants = s3_client.get_bucket_acl(public_rw_bucket)
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_WRITE_GRANTS)
with allure.step("Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
s3_client.put_object_acl(
bucket,
file_name,
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@allure.title("Bucket ACL (s3_client={s3_client})")
@pytest.mark.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
def test_s3_bucket_ACL(self, s3_client: S3ClientWrapper):
with reporter.step("Create bucket with public-read-write ACL"):
with allure.step("Create bucket with ACL = public-read-write"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_grants = s3_client.get_bucket_acl(bucket)
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_WRITE_GRANTS)
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
with reporter.step("Change bucket ACL to private"):
with allure.step("Change bucket ACL to private"):
s3_client.put_bucket_acl(bucket, acl="private")
bucket_grants = s3_client.get_bucket_acl(bucket)
s3_helper.verify_acl_permissions(bucket_grants, PRIVATE_GRANTS)
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
with reporter.step("Change bucket ACL to public-read"):
s3_client.put_bucket_acl(bucket, acl="public-read")
bucket_grants = s3_client.get_bucket_acl(bucket)
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_GRANTS)
with allure.step("Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"):
s3_client.put_bucket_acl(
bucket,
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")

View file

@ -2,8 +2,7 @@ from datetime import datetime, timedelta
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file
@ -12,84 +11,62 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.s3_gate
@pytest.mark.s3_gate_bucket
class TestS3GateBucket:
@allure.title("Bucket API (s3_client={s3_client})")
def test_s3_buckets(
self,
s3_client: S3ClientWrapper,
simple_object_size: ObjectSize,
):
"""
Test base S3 Bucket API (Create/List/Head/Delete).
"""
@allure.title("Create Bucket with different ACL (s3_client={s3_client})")
def test_s3_create_bucket_with_ACL(self, s3_client: S3ClientWrapper):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with allure.step("Create bucket with ACL private"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="private")
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
with reporter.step("Create buckets"):
bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
s3_helper.set_bucket_versioning(s3_client, bucket_1, VersioningStatus.ENABLED)
bucket_2 = s3_client.create_bucket()
with allure.step("Create bucket with ACL = public-read"):
bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read")
bucket_acl_1 = s3_client.get_bucket_acl(bucket_1)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
with reporter.step("Check buckets are presented in the system"):
buckets = s3_client.list_buckets()
assert bucket_1 in buckets, f"Expected bucket {bucket_1} is in the list"
assert bucket_2 in buckets, f"Expected bucket {bucket_2} is in the list"
with allure.step("Create bucket with ACL public-read-write"):
bucket_2 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_acl_2 = s3_client.get_bucket_acl(bucket_2)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
with reporter.step("Bucket must be empty"):
for bucket in (bucket_1, bucket_2):
with reporter.step("Verify default list command"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Create bucket with ACL = authenticated-read"):
bucket_3 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="authenticated-read")
bucket_acl_3 = s3_client.get_bucket_acl(bucket_3)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers")
with reporter.step("Verify V2 list command"):
objects_list = s3_client.list_objects_v2(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
@allure.title("Create Bucket with different ACL by grant (s3_client={s3_client})")
def test_s3_create_bucket_with_grands(self, s3_client: S3ClientWrapper):
with reporter.step("Check buckets are visible with S3 head command"):
s3_client.head_bucket(bucket_1)
s3_client.head_bucket(bucket_2)
with allure.step("Create bucket with --grant-read"):
bucket = s3_client.create_bucket(
object_lock_enabled_for_bucket=True,
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
with reporter.step("Check we can put/list object with S3 commands"):
version_id = s3_client.put_object(bucket_1, file_path)
s3_client.head_object(bucket_1, file_name)
with allure.step("Create bucket with --grant-wtite"):
bucket_1 = s3_client.create_bucket(
object_lock_enabled_for_bucket=True,
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
bucket_acl_1 = s3_client.get_bucket_acl(bucket_1)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
bucket_objects = s3_client.list_objects(bucket_1)
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
with reporter.step("Try to delete not empty bucket and get error"):
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
s3_client.delete_bucket(bucket_1)
s3_client.head_bucket(bucket_1)
with reporter.step("Delete empty bucket_2"):
s3_client.delete_bucket(bucket_2)
with reporter.step("Check bucket_2 is deleted"):
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_bucket(bucket_2)
buckets = s3_client.list_buckets()
assert bucket_1 in buckets, f"Expected bucket {bucket_1} is in the list"
assert bucket_2 not in buckets, f"Expected bucket {bucket_2} is not in the list"
with reporter.step("Delete object from bucket_1"):
s3_client.delete_object(bucket_1, file_name, version_id)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=[])
with reporter.step("Delete bucket_1"):
s3_client.delete_bucket(bucket_1)
with reporter.step("Check bucket_1 deleted"):
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_bucket(bucket_1)
with allure.step("Create bucket with --grant-full-control"):
bucket_2 = s3_client.create_bucket(
object_lock_enabled_for_bucket=True,
grant_full_control="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
bucket_acl_2 = s3_client.get_bucket_acl(bucket_2)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
@allure.title("Create bucket with object lock (s3_client={s3_client})")
def test_s3_bucket_object_lock(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Create bucket with --no-object-lock-enabled-for-bucket"):
with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False)
date_obj = datetime.utcnow() + timedelta(days=1)
with pytest.raises(Exception, match=r".*Object Lock configuration does not exist for this bucket.*"):
@ -101,7 +78,7 @@ class TestS3GateBucket:
object_lock_mode="COMPLIANCE",
object_lock_retain_until_date=date_obj.strftime("%Y-%m-%dT%H:%M:%S"),
)
with reporter.step("Create bucket with --object-lock-enabled-for-bucket"):
with allure.step("Create bucket with --object-lock-enabled-for-bucket"):
bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
date_obj_1 = datetime.utcnow() + timedelta(days=1)
s3_client.put_object(
@ -121,21 +98,21 @@ class TestS3GateBucket:
file_name_2 = s3_helper.object_key_from_file_path(file_path_2)
bucket = s3_client.create_bucket()
with reporter.step("Put two objects into bucket"):
with allure.step("Put two objects into bucket"):
s3_client.put_object(bucket, file_path_1)
s3_client.put_object(bucket, file_path_2)
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name_1, file_name_2])
with reporter.step("Try to delete not empty bucket and get error"):
with allure.step("Try to delete not empty bucket and get error"):
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
s3_client.delete_bucket(bucket)
with reporter.step("Delete object in bucket"):
with allure.step("Delete object in bucket"):
s3_client.delete_object(bucket, file_name_1)
s3_client.delete_object(bucket, file_name_2)
s3_helper.check_objects_in_bucket(s3_client, bucket, [])
with reporter.step("Delete empty bucket"):
with allure.step("Delete empty bucket"):
s3_client.delete_bucket(bucket)
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_bucket(bucket)

View file

@ -0,0 +1,516 @@
import logging
import os
from random import choice, choices
import allure
import pytest
from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.epoch import tick_epoch
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import (
generate_file,
generate_file_with_content,
get_file_content,
get_file_hash,
split_file,
)
logger = logging.getLogger("NeoLogger")
@allure.link("https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway")
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_base
class TestS3Gate:
@allure.title("Bucket API (s3_client={s3_client})")
def test_s3_buckets(
self,
s3_client: S3ClientWrapper,
client_shell: Shell,
cluster: Cluster,
simple_object_size: ObjectSize,
):
"""
Test base S3 Bucket API (Create/List/Head/Delete).
"""
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with allure.step("Create buckets"):
bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
s3_helper.set_bucket_versioning(s3_client, bucket_1, VersioningStatus.ENABLED)
bucket_2 = s3_client.create_bucket()
with allure.step("Check buckets are presented in the system"):
buckets = s3_client.list_buckets()
assert bucket_1 in buckets, f"Expected bucket {bucket_1} is in the list"
assert bucket_2 in buckets, f"Expected bucket {bucket_2} is in the list"
with allure.step("Bucket must be empty"):
for bucket in (bucket_1, bucket_2):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Check buckets are visible with S3 head command"):
s3_client.head_bucket(bucket_1)
s3_client.head_bucket(bucket_2)
with allure.step("Check we can put/list object with S3 commands"):
version_id = s3_client.put_object(bucket_1, file_path)
s3_client.head_object(bucket_1, file_name)
bucket_objects = s3_client.list_objects(bucket_1)
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
with allure.step("Try to delete not empty bucket and get error"):
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
s3_client.delete_bucket(bucket_1)
s3_client.head_bucket(bucket_1)
with allure.step(f"Delete empty bucket {bucket_2}"):
s3_client.delete_bucket(bucket_2)
tick_epoch(client_shell, cluster)
with allure.step(f"Check bucket {bucket_2} deleted"):
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_bucket(bucket_2)
buckets = s3_client.list_buckets()
assert bucket_1 in buckets, f"Expected bucket {bucket_1} is in the list"
assert bucket_2 not in buckets, f"Expected bucket {bucket_2} is not in the list"
with allure.step(f"Delete object from {bucket_1}"):
s3_client.delete_object(bucket_1, file_name, version_id)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=[])
with allure.step(f"Delete bucket {bucket_1}"):
s3_client.delete_bucket(bucket_1)
tick_epoch(client_shell, cluster)
with allure.step(f"Check bucket {bucket_1} deleted"):
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_bucket(bucket_1)
@allure.title("Object API (obj_size={object_size}, s3_client={s3_client})")
@pytest.mark.parametrize(
"object_size",
["simple", "complex"],
indirect=True,
)
def test_s3_api_object(
self,
s3_client: S3ClientWrapper,
object_size: ObjectSize,
two_buckets: tuple[str, str],
):
"""
Test base S3 Object API (Put/Head/List) for simple and complex objects.
"""
file_path = generate_file(object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
bucket_1, bucket_2 = two_buckets
for bucket in (bucket_1, bucket_2):
with allure.step("Bucket must be empty"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
s3_client.put_object(bucket, file_path)
s3_client.head_object(bucket, file_name)
bucket_objects = s3_client.list_objects(bucket)
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
with allure.step("Check object's attributes"):
for attrs in (["ETag"], ["ObjectSize", "StorageClass"]):
s3_client.get_object_attributes(bucket, file_name, attrs)
@allure.title("Sync directory (s3_client={s3_client})")
def test_s3_sync_dir(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks sync directory with AWS CLI utility.
"""
file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1")
file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2")
key_to_path = {"test_file_1": file_path_1, "test_file_2": file_path_2}
if not isinstance(s3_client, AwsCliClient):
pytest.skip("This test is not supported with boto3 client")
generate_file_with_content(simple_object_size.value, file_path=file_path_1)
generate_file_with_content(simple_object_size.value, file_path=file_path_2)
s3_client.sync(bucket=bucket, dir_path=os.path.dirname(file_path_1))
with allure.step("Check objects are synced"):
objects = s3_client.list_objects(bucket)
with allure.step("Check these are the same objects"):
assert set(key_to_path.keys()) == set(objects), f"Expected all objects saved. Got {objects}"
for obj_key in objects:
got_object = s3_client.get_object(bucket, obj_key)
assert get_file_hash(got_object) == get_file_hash(
key_to_path.get(obj_key)
), "Expected hashes are the same"
@allure.title("Object versioning (s3_client={s3_client})")
def test_s3_api_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks basic versioning functionality for S3 bucket.
"""
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple)
generate_file_with_content(simple_object_size.value, file_path=file_name_simple, content=version_2_content)
version_id_2 = s3_client.put_object(bucket, file_name_simple)
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {
version_id_1,
version_id_2,
}, f"Expected object has versions: {version_id_1, version_id_2}"
with allure.step("Show information about particular version"):
for version_id in (version_id_1, version_id_2):
response = s3_client.head_object(bucket, obj_key, version_id=version_id)
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
assert response.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with allure.step("Check object's attributes"):
for version_id in (version_id_1, version_id_2):
got_attrs = s3_client.get_object_attributes(bucket, obj_key, ["ETag"], version_id=version_id)
if got_attrs:
assert got_attrs.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
with allure.step("Delete object and check it was deleted"):
response = s3_client.delete_object(bucket, obj_key)
version_id_delete = response.get("VersionId")
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_object(bucket, obj_key)
with allure.step("Get content for all versions and check it is correct"):
for version, content in (
(version_id_2, version_2_content),
(version_id_1, version_1_content),
):
file_name = s3_client.get_object(bucket, obj_key, version_id=version)
got_content = get_file_content(file_name)
assert got_content == content, f"Expected object content is\n{content}\nGot\n{got_content}"
with allure.step("Restore previous object version"):
s3_client.delete_object(bucket, obj_key, version_id=version_id_delete)
file_name = s3_client.get_object(bucket, obj_key)
got_content = get_file_content(file_name)
assert (
got_content == version_2_content
), f"Expected object content is\n{version_2_content}\nGot\n{got_content}"
@pytest.mark.s3_gate_multipart
@allure.title("Object Multipart API (s3_client={s3_client})")
def test_s3_api_multipart(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
Upload part/List parts/Complete multipart upload).
"""
parts_count = 3
file_name_large = generate_file(simple_object_size.value * 1024 * 6 * parts_count) # 5Mb - min part
object_key = s3_helper.object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count)
parts = []
uploads = s3_client.list_multipart_uploads(bucket)
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
with allure.step("Create and abort multipart upload"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
uploads = s3_client.list_multipart_uploads(bucket)
assert uploads, f"Expected there one upload in bucket {bucket}"
assert uploads[0].get("Key") == object_key, f"Expected correct key {object_key} in upload {uploads}"
assert uploads[0].get("UploadId") == upload_id, f"Expected correct UploadId {upload_id} in upload {uploads}"
s3_client.abort_multipart_upload(bucket, object_key, upload_id)
uploads = s3_client.list_multipart_uploads(bucket)
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
with allure.step("Create new multipart upload and upload several parts"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
for part_id, file_path in enumerate(part_files, start=1):
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
parts.append((part_id, etag))
with allure.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
uploads = s3_client.list_multipart_uploads(bucket)
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
with allure.step("Check we can get whole object from bucket"):
got_object = s3_client.get_object(bucket, object_key)
assert get_file_hash(got_object) == get_file_hash(file_name_large)
self.check_object_attributes(s3_client, bucket, object_key, parts_count)
@allure.title("Bucket tagging API (s3_client={s3_client})")
def test_s3_api_bucket_tagging(self, s3_client: S3ClientWrapper, bucket: str):
"""
Test checks S3 Bucket tagging API (Put tag/Get tag).
"""
key_value_pair = [("some-key", "some-value"), ("some-key-2", "some-value-2")]
s3_client.put_bucket_tagging(bucket, key_value_pair)
s3_helper.check_tags_by_bucket(s3_client, bucket, key_value_pair)
s3_client.delete_bucket_tagging(bucket)
s3_helper.check_tags_by_bucket(s3_client, bucket, [])
@allure.title("Object tagging API (s3_client={s3_client})")
def test_s3_api_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
"""
key_value_pair_bucket = [("some-key", "some-value"), ("some-key-2", "some-value-2")]
key_value_pair_obj = [
("some-key-obj", "some-value-obj"),
("some-key--obj2", "some-value--obj2"),
]
key_value_pair_obj_new = [("some-key-obj-new", "some-value-obj-new")]
file_name_simple = generate_file(simple_object_size.value)
obj_key = s3_helper.object_key_from_file_path(file_name_simple)
s3_client.put_bucket_tagging(bucket, key_value_pair_bucket)
s3_client.put_object(bucket, file_name_simple)
for tags in (key_value_pair_obj, key_value_pair_obj_new):
s3_client.put_object_tagging(bucket, obj_key, tags)
s3_helper.check_tags_by_object(
s3_client,
bucket,
obj_key,
tags,
)
s3_client.delete_object_tagging(bucket, obj_key)
s3_helper.check_tags_by_object(s3_client, bucket, obj_key, [])
@allure.title("Delete object & delete objects (s3_client={s3_client})")
def test_s3_api_delete(
self,
s3_client: S3ClientWrapper,
two_buckets: tuple[str, str],
simple_object_size: ObjectSize,
complex_object_size: ObjectSize,
):
"""
Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one.
From second bucket some objects deleted all at once.
"""
max_obj_count = 20
max_delete_objects = 17
put_objects = []
file_paths = []
obj_sizes = [simple_object_size, complex_object_size]
bucket_1, bucket_2 = two_buckets
with allure.step(f"Generate {max_obj_count} files"):
for _ in range(max_obj_count):
file_paths.append(generate_file(choice(obj_sizes).value))
for bucket in (bucket_1, bucket_2):
with allure.step(f"Bucket {bucket} must be empty as it just created"):
objects_list = s3_client.list_objects_v2(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
for file_path in file_paths:
s3_client.put_object(bucket, file_path)
put_objects.append(s3_helper.object_key_from_file_path(file_path))
with allure.step(f"Check all objects put in bucket {bucket} successfully"):
bucket_objects = s3_client.list_objects_v2(bucket)
assert set(put_objects) == set(
bucket_objects
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
with allure.step("Delete some objects from bucket_1 one by one"):
objects_to_delete_b1 = choices(put_objects, k=max_delete_objects)
for obj in objects_to_delete_b1:
s3_client.delete_object(bucket_1, obj)
with allure.step("Check deleted objects are not visible in bucket bucket_1"):
bucket_objects = s3_client.list_objects_v2(bucket_1)
assert set(put_objects).difference(set(objects_to_delete_b1)) == set(
bucket_objects
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
for object_key in objects_to_delete_b1:
with pytest.raises(Exception, match="The specified key does not exist"):
s3_client.get_object(bucket_1, object_key)
with allure.step("Delete some objects from bucket_2 at once"):
objects_to_delete_b2 = choices(put_objects, k=max_delete_objects)
s3_client.delete_objects(bucket_2, objects_to_delete_b2)
with allure.step("Check deleted objects are not visible in bucket bucket_2"):
objects_list = s3_client.list_objects_v2(bucket_2)
assert set(put_objects).difference(set(objects_to_delete_b2)) == set(
objects_list
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
for object_key in objects_to_delete_b2:
with pytest.raises(Exception, match="The specified key does not exist"):
s3_client.get_object(bucket_2, object_key)
@allure.title("Copy object to the same bucket (s3_client={s3_client})")
def test_s3_copy_same_bucket(
self,
s3_client: S3ClientWrapper,
bucket: str,
complex_object_size: ObjectSize,
simple_object_size: ObjectSize,
):
"""
Test object can be copied to the same bucket.
#TODO: delete after test_s3_copy_object will be merge
"""
file_path_simple = generate_file(simple_object_size.value)
file_path_large = generate_file(complex_object_size.value)
file_name_simple = s3_helper.object_key_from_file_path(file_path_simple)
file_name_large = s3_helper.object_key_from_file_path(file_path_large)
bucket_objects = [file_name_simple, file_name_large]
with allure.step("Bucket must be empty"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Put objects into bucket"):
for file_path in (file_path_simple, file_path_large):
s3_client.put_object(bucket, file_path)
with allure.step("Copy one object into the same bucket"):
copy_obj_path = s3_client.copy_object(bucket, file_name_simple)
bucket_objects.append(copy_obj_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
with allure.step("Check copied object has the same content"):
got_copied_file = s3_client.get_object(bucket, copy_obj_path)
assert get_file_hash(file_path_simple) == get_file_hash(got_copied_file), "Hashes must be the same"
with allure.step("Delete one object from bucket"):
s3_client.delete_object(bucket, file_name_simple)
bucket_objects.remove(file_name_simple)
s3_helper.check_objects_in_bucket(
s3_client,
bucket,
expected_objects=bucket_objects,
unexpected_objects=[file_name_simple],
)
@allure.title("Copy object to another bucket (s3_client={s3_client})")
def test_s3_copy_to_another_bucket(
self,
s3_client: S3ClientWrapper,
two_buckets: tuple[str, str],
complex_object_size: ObjectSize,
simple_object_size: ObjectSize,
):
"""
Test object can be copied to another bucket.
#TODO: delete after test_s3_copy_object will be merge
"""
file_path_simple = generate_file(simple_object_size.value)
file_path_large = generate_file(complex_object_size.value)
file_name_simple = s3_helper.object_key_from_file_path(file_path_simple)
file_name_large = s3_helper.object_key_from_file_path(file_path_large)
bucket_1_objects = [file_name_simple, file_name_large]
bucket_1, bucket_2 = two_buckets
with allure.step("Buckets must be empty"):
for bucket in (bucket_1, bucket_2):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Put objects into one bucket"):
for file_path in (file_path_simple, file_path_large):
s3_client.put_object(bucket_1, file_path)
with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, file_name_large, bucket=bucket_2)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with allure.step("Check copied object has the same content"):
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
assert get_file_hash(file_path_large) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
with allure.step("Delete one object from first bucket"):
s3_client.delete_object(bucket_1, file_name_simple)
bucket_1_objects.remove(file_name_simple)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with allure.step("Delete one object from second bucket and check it is empty"):
s3_client.delete_object(bucket_2, copy_obj_path_b2)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[])
def check_object_attributes(self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int):
if not isinstance(s3_client, AwsCliClient):
logger.warning("Attributes check is not supported for boto3 implementation")
return
with allure.step("Check object's attributes"):
obj_parts = s3_client.get_object_attributes(bucket, object_key, ["ObjectParts"], full_output=False)
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
assert len(obj_parts.get("Parts")) == parts_count, f"Expected Parts cunt is {parts_count}"
with allure.step("Check object's attribute max-parts"):
max_parts = 2
obj_parts = s3_client.get_object_attributes(
bucket,
object_key,
["ObjectParts"],
max_parts=max_parts,
full_output=False,
)
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
assert obj_parts.get("MaxParts") == max_parts, f"Expected MaxParts is {parts_count}"
assert len(obj_parts.get("Parts")) == max_parts, f"Expected Parts count is {parts_count}"
with allure.step("Check object's attribute part-number-marker"):
part_number_marker = 3
obj_parts = s3_client.get_object_attributes(
bucket,
object_key,
["ObjectParts"],
part_number=part_number_marker,
full_output=False,
)
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
assert (
obj_parts.get("PartNumberMarker") == part_number_marker
), f"Expected PartNumberMarker is {part_number_marker}"
assert len(obj_parts.get("Parts")) == 1, f"Expected Parts count is {parts_count}"

View file

@ -3,197 +3,190 @@ from datetime import datetime, timedelta
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
@allure.title("[Module] Create bucket with object_lock_enabled_for_bucket")
@pytest.fixture(scope="module")
def bucket_w_lock(s3_client: S3ClientWrapper):
return s3_client.create_bucket(object_lock_enabled_for_bucket=True)
@allure.title("[Module] Create bucket without object_lock_enabled_for_bucket")
@pytest.fixture(scope="module")
def bucket_no_lock(s3_client: S3ClientWrapper):
return s3_client.create_bucket(object_lock_enabled_for_bucket=False)
@pytest.mark.s3_gate
@pytest.mark.s3_gate_locking
@pytest.mark.parametrize("version_id", [None, "second"])
class TestS3GateLocking:
@allure.title("Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})")
def test_s3_object_locking(
self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize
):
def test_s3_object_locking(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2
with reporter.step("Put several versions of object into bucket"):
s3_client.put_object(bucket_w_lock, file_path)
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
with allure.step("Put several versions of object into bucket"):
s3_client.put_object(bucket, file_path)
file_name_1 = generate_file_with_content(simple_object_size.value, file_path=file_path)
version_id_2 = s3_client.put_object(bucket_w_lock, file_name_1)
version_id_2 = s3_client.put_object(bucket, file_name_1)
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name])
if version_id:
version_id = version_id_2
with reporter.step(f"Put retention period {retention_period}min to object {file_name}"):
with allure.step(f"Put retention period {retention_period}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period)
retention = {
"Mode": "COMPLIANCE",
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "COMPLIANCE", date_obj, "OFF")
s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with reporter.step(f"Put legal hold to object {file_name}"):
s3_client.put_object_legal_hold(bucket_w_lock, file_name, "ON", version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "COMPLIANCE", date_obj, "ON")
with allure.step(f"Put legal hold to object {file_name}"):
s3_client.put_object_legal_hold(bucket, file_name, "ON", version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
with reporter.step("Fail with deleting object with legal hold and retention period"):
with allure.step("Fail with deleting object with legal hold and retention period"):
if version_id:
with pytest.raises(Exception):
# An error occurred (AccessDenied) when calling the DeleteObject operation (reached max retries: 0): Access Denied.
s3_client.delete_object(bucket_w_lock, file_name, version_id)
s3_client.delete_object(bucket, file_name, version_id)
with reporter.step("Check retention period is no longer set on the uploaded object"):
with allure.step("Check retention period is no longer set on the uploaded object"):
time.sleep((retention_period + 1) * 60)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "COMPLIANCE", date_obj, "ON")
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
with reporter.step("Fail with deleting object with legal hold and retention period"):
with allure.step("Fail with deleting object with legal hold and retention period"):
if version_id:
with pytest.raises(Exception):
# An error occurred (AccessDenied) when calling the DeleteObject operation (reached max retries: 0): Access Denied.
s3_client.delete_object(bucket_w_lock, file_name, version_id)
s3_client.delete_object(bucket, file_name, version_id)
else:
s3_client.delete_object(bucket_w_lock, file_name, version_id)
s3_client.delete_object(bucket, file_name, version_id)
@allure.title("Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})")
def test_s3_mode_compliance(
self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize
):
def test_s3_mode_compliance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2
retention_period_1 = 1
with reporter.step("Put object into bucket"):
obj_version = s3_client.put_object(bucket_w_lock, file_path)
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
with allure.step("Put object into bucket"):
obj_version = s3_client.put_object(bucket, file_path)
if version_id:
version_id = obj_version
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name])
with reporter.step(f"Put retention period {retention_period}min to object {file_name}"):
with allure.step(f"Put retention period {retention_period}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period)
retention = {
"Mode": "COMPLIANCE",
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "COMPLIANCE", date_obj, "OFF")
s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with reporter.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = {
"Mode": "COMPLIANCE",
"RetainUntilDate": date_obj,
}
with pytest.raises(Exception):
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
s3_client.put_object_retention(bucket, file_name, retention, version_id)
@allure.title("Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})")
def test_s3_mode_governance(
self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize
):
def test_s3_mode_governance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 3
retention_period_1 = 2
retention_period_2 = 5
with reporter.step("Put object into bucket"):
obj_version = s3_client.put_object(bucket_w_lock, file_path)
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
with allure.step("Put object into bucket"):
obj_version = s3_client.put_object(bucket, file_path)
if version_id:
version_id = obj_version
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name])
with reporter.step(f"Put retention period {retention_period}min to object {file_name}"):
with allure.step(f"Put retention period {retention_period}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period)
retention = {
"Mode": "GOVERNANCE",
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "GOVERNANCE", date_obj, "OFF")
s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
with reporter.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = {
"Mode": "GOVERNANCE",
"RetainUntilDate": date_obj,
}
with pytest.raises(Exception):
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
s3_client.put_object_retention(bucket, file_name, retention, version_id)
with reporter.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = {
"Mode": "GOVERNANCE",
"RetainUntilDate": date_obj,
}
with pytest.raises(Exception):
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
s3_client.put_object_retention(bucket, file_name, retention, version_id)
with reporter.step(f"Put new retention period {retention_period_2}min to object {file_name}"):
with allure.step(f"Put new retention period {retention_period_2}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_2)
retention = {
"Mode": "GOVERNANCE",
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id, True)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "GOVERNANCE", date_obj, "OFF")
s3_client.put_object_retention(bucket, file_name, retention, version_id, True)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
@allure.title(
"[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})"
)
def test_s3_legal_hold(
self, s3_client: S3ClientWrapper, bucket_no_lock: str, version_id: str, simple_object_size: ObjectSize
):
def test_s3_legal_hold(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into bucket"):
obj_version = s3_client.put_object(bucket_no_lock, file_path)
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False)
with allure.step("Put object into bucket"):
obj_version = s3_client.put_object(bucket, file_path)
if version_id:
version_id = obj_version
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name])
with reporter.step(f"Put legal hold to object {file_name}"):
with allure.step(f"Put legal hold to object {file_name}"):
with pytest.raises(Exception):
s3_client.put_object_legal_hold(bucket_no_lock, file_name, "ON", version_id)
s3_client.put_object_legal_hold(bucket, file_name, "ON", version_id)
@pytest.mark.s3_gate
class TestS3GateLockingBucket:
@allure.title("Bucket Lock (s3_client={s3_client})")
def test_s3_bucket_lock(self, s3_client: S3ClientWrapper, bucket_w_lock: str, simple_object_size: ObjectSize):
def test_s3_bucket_lock(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
configuration = {"Rule": {"DefaultRetention": {"Mode": "COMPLIANCE", "Days": 1}}}
with reporter.step("PutObjectLockConfiguration with ObjectLockEnabled=False"):
s3_client.put_object_lock_configuration(bucket_w_lock, configuration)
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
with reporter.step("PutObjectLockConfiguration with ObjectLockEnabled=True"):
with allure.step("PutObjectLockConfiguration with ObjectLockEnabled=False"):
s3_client.put_object_lock_configuration(bucket, configuration)
with allure.step("PutObjectLockConfiguration with ObjectLockEnabled=True"):
configuration["ObjectLockEnabled"] = "Enabled"
s3_client.put_object_lock_configuration(bucket_w_lock, configuration)
s3_client.put_object_lock_configuration(bucket, configuration)
with reporter.step("GetObjectLockConfiguration"):
config = s3_client.get_object_lock_configuration(bucket_w_lock)
with allure.step("GetObjectLockConfiguration"):
config = s3_client.get_object_lock_configuration(bucket)
configuration["Rule"]["DefaultRetention"]["Years"] = 0
assert config == configuration, f"Configurations must be equal {configuration}"
with reporter.step("Put object into bucket"):
s3_client.put_object(bucket_w_lock, file_path)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "COMPLIANCE", None, "OFF", 1)
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1)

View file

@ -1,13 +1,10 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash, split_file
PART_SIZE = 5 * 1024 * 1024
@ -21,7 +18,7 @@ class TestS3GateMultipart(ClusterTestBase):
@allure.title("Object Multipart API (s3_client={s3_client}, bucket versioning = {versioning_status})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], indirect=True)
def test_s3_object_multipart(
self, s3_client: S3ClientWrapper, bucket: str, default_wallet: WalletInfo, versioning_status: str
self, s3_client: S3ClientWrapper, bucket: str, default_wallet: str, versioning_status: str
):
parts_count = 5
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
@ -29,13 +26,12 @@ class TestS3GateMultipart(ClusterTestBase):
part_files = split_file(file_name_large, parts_count)
parts = []
with reporter.step(f"Get related container_id for bucket"):
with allure.step(f"Get related container_id for bucket"):
for cluster_node in self.cluster.cluster_nodes:
container_id = search_container_by_name(bucket, cluster_node)
if container_id:
break
with reporter.step("Upload first part"):
with allure.step("Upload first part"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
uploads = s3_client.list_multipart_uploads(bucket)
etag = s3_client.upload_part(bucket, object_key, upload_id, 1, part_files[0])
@ -43,37 +39,38 @@ class TestS3GateMultipart(ClusterTestBase):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
with reporter.step("Upload last parts"):
with allure.step("Upload last parts"):
for part_id, file_path in enumerate(part_files[1:], start=2):
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
parts.append((part_id, etag))
with reporter.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
with reporter.step("Complete multipart upload"):
response = s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
version_id = None
if versioning_status == VersioningStatus.ENABLED:
version_id = response["VersionId"]
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
with reporter.step("There should be no multipart uploads"):
with allure.step("Check upload list is empty"):
uploads = s3_client.list_multipart_uploads(bucket)
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
with reporter.step("Check we can get whole object from bucket"):
with allure.step("Check we can get whole object from bucket"):
got_object = s3_client.get_object(bucket, object_key)
assert get_file_hash(got_object) == get_file_hash(file_name_large)
with reporter.step("Delete the object"):
s3_client.delete_object(bucket, object_key, version_id)
if version_id:
with allure.step("Delete the object version"):
s3_client.delete_object(bucket, object_key, version_id)
else:
with allure.step("Delete the object"):
s3_client.delete_object(bucket, object_key)
with reporter.step("There should be no objects in bucket"):
with allure.step("List objects in the bucket, expect to be empty"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with reporter.step("There should be no objects in container"):
with allure.step("List objects in the container via rpc, expect to be empty"):
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
@ -82,7 +79,7 @@ class TestS3GateMultipart(ClusterTestBase):
def test_s3_abort_multipart(
self,
s3_client: S3ClientWrapper,
default_wallet: WalletInfo,
default_wallet: str,
bucket: str,
simple_object_size: ObjectSize,
complex_object_size: ObjectSize,
@ -93,44 +90,38 @@ class TestS3GateMultipart(ClusterTestBase):
files_count = len(to_upload)
upload_key = "multipart_abort"
with reporter.step("Get related container_id for bucket"):
for cluster_node in self.cluster.cluster_nodes:
container_id = search_container_by_name(bucket, cluster_node)
if container_id:
break
with allure.step(f"Get related container_id for bucket '{bucket}'"):
container_id = search_container_by_name(
default_wallet, bucket, self.shell, self.cluster.default_rpc_endpoint
)
with reporter.step("Create multipart upload"):
with allure.step("Create multipart upload"):
upload_id = s3_client.create_multipart_upload(bucket, upload_key)
with reporter.step(f"Upload {files_count} parts to multipart upload"):
with allure.step(f"Upload {files_count} files to multipart upload"):
for i, file in enumerate(to_upload, 1):
s3_client.upload_part(bucket, upload_key, upload_id, i, file)
with reporter.step(f"There should be {files_count} objects in bucket"):
with allure.step(f"Check that we have {files_count} files in bucket"):
parts = s3_client.list_parts(bucket, upload_key, upload_id)
assert len(parts) == files_count, f"Expected {files_count} parts, got\n{parts}"
with reporter.step(f"There should be {files_count} objects in container"):
with allure.step(f"Check that we have {files_count} files in container '{container_id}'"):
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == files_count, f"Expected {files_count} objects in container, got\n{objects}"
with reporter.step("Abort multipart upload"):
with allure.step("Abort multipart upload"):
s3_client.abort_multipart_upload(bucket, upload_key, upload_id)
uploads = s3_client.list_multipart_uploads(bucket)
assert not uploads, f"Expected no uploads in bucket {bucket}"
with reporter.step("There should be no objects in bucket"):
with allure.step("Check that we have no files in bucket since upload was aborted"):
with pytest.raises(Exception, match=self.NO_SUCH_UPLOAD):
s3_client.list_parts(bucket, upload_key, upload_id)
with reporter.step("There should be no objects in container"):
@wait_for_success(120, 10)
def check_no_objects():
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
check_no_objects()
with allure.step("Check that we have no files in container since upload was aborted"):
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
@allure.title("Upload Part Copy (s3_client={s3_client})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
@ -142,29 +133,27 @@ class TestS3GateMultipart(ClusterTestBase):
parts = []
objs = []
with reporter.step(f"Put {parts_count} objects in bucket"):
with allure.step(f"Put {parts_count} objects in bucket"):
for part in part_files:
s3_client.put_object(bucket, part)
objs.append(s3_helper.object_key_from_file_path(part))
s3_helper.check_objects_in_bucket(s3_client, bucket, objs)
with reporter.step("Create multipart upload object"):
with allure.step("Create multipart upload object"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
uploads = s3_client.list_multipart_uploads(bucket)
assert len(uploads) == 1, f"Expected one upload in bucket {bucket}"
assert uploads[0].get("Key") == object_key, f"Expected correct key {object_key} in upload {uploads}"
assert uploads[0].get("UploadId") == upload_id, f"Expected correct UploadId {upload_id} in upload {uploads}"
assert uploads, f"Expected there are uploads in bucket {bucket}"
with reporter.step("Upload parts to multipart upload"):
with allure.step("Upload parts to multipart upload"):
for part_id, obj_key in enumerate(objs, start=1):
etag = s3_client.upload_part_copy(bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}")
parts.append((part_id, etag))
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
with reporter.step("Complete multipart upload"):
with allure.step("Complete multipart upload"):
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
with reporter.step("Get whole object from bucket"):
with allure.step("Check we can get whole object from bucket"):
got_object = s3_client.get_object(bucket, object_key)
assert get_file_hash(got_object) == get_file_hash(file_name_large)

View file

@ -1,28 +1,20 @@
import os
import random
import string
import uuid
from datetime import datetime, timedelta
from random import choices, sample
from typing import Literal
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
from frostfs_testlib.resources.error_patterns import S3_BUCKET_DOES_NOT_ALLOW_ACL, S3_MALFORMED_XML_REQUEST
from frostfs_testlib.resources.s3_acl_grants import PRIVATE_GRANTS
from frostfs_testlib.resources.error_patterns import S3_MALFORMED_XML_REQUEST
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import (
TestFile,
concat_files,
generate_file,
generate_file_with_content,
get_file_hash,
)
from frostfs_testlib.utils.file_utils import concat_files, generate_file, generate_file_with_content, get_file_hash
@pytest.mark.s3_gate
@ -35,44 +27,11 @@ class TestS3GateObject:
public_key = wallet_utils.get_wallet_public_key(second_wallet, DEFAULT_WALLET_PASS)
yield public_key
@allure.title("Object API (obj_size={object_size}, s3_client={s3_client})")
@pytest.mark.parametrize(
"object_size",
["simple", "complex"],
indirect=True,
)
def test_s3_api_object(
self,
s3_client: S3ClientWrapper,
object_size: ObjectSize,
bucket: str,
):
"""
Test base S3 Object API (Put/Head/List) for simple and complex objects.
"""
with reporter.step("Prepare object to upload"):
test_file = generate_file(object_size.value)
file_name = s3_helper.object_key_from_file_path(test_file)
with reporter.step("Put object to bucket"):
s3_client.put_object(bucket, test_file)
with reporter.step("Head object from bucket"):
s3_client.head_object(bucket, file_name)
with reporter.step("Verify object in list"):
bucket_objects = s3_client.list_objects(bucket)
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
with reporter.step("Check object's attributes"):
for attrs in (["ETag"], ["ObjectSize", "StorageClass"]):
s3_client.get_object_attributes(bucket, file_name, attrs)
@allure.title("Copy object (s3_client={s3_client})")
def test_s3_copy_object(
self,
s3_client: S3ClientWrapper,
two_buckets: list[str],
two_buckets: tuple[str, str],
simple_object_size: ObjectSize,
):
file_path = generate_file(simple_object_size.value)
@ -81,10 +40,13 @@ class TestS3GateObject:
bucket_1, bucket_2 = two_buckets
with reporter.step("Put object into one bucket"):
objects_list = s3_client.list_objects(bucket_1)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("Put object into one bucket"):
s3_client.put_object(bucket_1, file_path)
with reporter.step("Copy one object into the same bucket"):
with allure.step("Copy one object into the same bucket"):
copy_obj_path = s3_client.copy_object(bucket_1, file_name)
bucket_1_objects.append(copy_obj_path)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, bucket_1_objects)
@ -92,22 +54,22 @@ class TestS3GateObject:
objects_list = s3_client.list_objects(bucket_2)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with reporter.step("Copy object from first bucket into second"):
with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, file_name, bucket=bucket_2)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with reporter.step("Check copied object has the same content"):
with allure.step("Check copied object has the same content"):
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
assert get_file_hash(file_path) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
with reporter.step("Delete one object from first bucket"):
with allure.step("Delete one object from first bucket"):
s3_client.delete_object(bucket_1, file_name)
bucket_1_objects.remove(file_name)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with reporter.step("Copy one object into the same bucket"):
with allure.step("Copy one object into the same bucket"):
with pytest.raises(Exception):
s3_client.copy_object(bucket_1, file_name)
@ -115,7 +77,7 @@ class TestS3GateObject:
def test_s3_copy_version_object(
self,
s3_client: S3ClientWrapper,
two_buckets: list[str],
two_buckets: tuple[str, str],
simple_object_size: ObjectSize,
):
version_1_content = "Version 1"
@ -125,50 +87,47 @@ class TestS3GateObject:
bucket_1, bucket_2 = two_buckets
s3_helper.set_bucket_versioning(s3_client, bucket_1, VersioningStatus.ENABLED)
with reporter.step("Put object into bucket"):
with allure.step("Put object into bucket"):
s3_client.put_object(bucket_1, file_name_simple)
bucket_1_objects = [obj_key]
s3_helper.check_objects_in_bucket(s3_client, bucket_1, [obj_key])
with reporter.step("Copy one object into the same bucket"):
with allure.step("Copy one object into the same bucket"):
copy_obj_path = s3_client.copy_object(bucket_1, obj_key)
bucket_1_objects.append(copy_obj_path)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, bucket_1_objects)
s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED)
with reporter.step("Copy object from first bucket into second"):
with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, obj_key, bucket=bucket_2)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with reporter.step("Delete one object from first bucket and check object in bucket"):
with allure.step("Delete one object from first bucket and check object in bucket"):
s3_client.delete_object(bucket_1, obj_key)
bucket_1_objects.remove(obj_key)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
with reporter.step("Copy one object into the same bucket"):
with allure.step("Copy one object into the same bucket"):
with pytest.raises(Exception):
s3_client.copy_object(bucket_1, obj_key)
@allure.title("Copy with acl (s3_client={s3_client})")
def test_s3_copy_acl(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
file_path = generate_file_with_content(simple_object_size.value)
file_name = os.path.basename(file_path)
version_1_content = "Version 1"
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name])
with allure.step("Put several versions of object into bucket"):
s3_client.put_object(bucket, file_name_simple)
s3_helper.check_objects_in_bucket(s3_client, bucket, [obj_key])
with reporter.step("[NEGATIVE] Copy object with public-read-write ACL"):
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
copy_path = s3_client.copy_object(bucket, file_name, acl="public-read-write")
with reporter.step("Copy object with private ACL"):
copy_path = s3_client.copy_object(bucket, file_name, acl="private")
object_grants = s3_client.get_object_acl(bucket, copy_path)
s3_helper.verify_acl_permissions(object_grants, PRIVATE_GRANTS)
with allure.step("Copy object and check acl attribute"):
copy_obj_path = s3_client.copy_object(bucket, obj_key, acl="public-read-write")
obj_acl = s3_client.get_object_acl(bucket, copy_obj_path)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
@allure.title("Copy object with metadata (s3_client={s3_client})")
def test_s3_copy_metadate(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
@ -179,25 +138,25 @@ class TestS3GateObject:
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put object into bucket"):
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path, metadata=object_metadata)
bucket_1_objects = [file_name]
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects)
with reporter.step("Copy one object"):
with allure.step("Copy one object"):
copy_obj_path = s3_client.copy_object(bucket, file_name)
bucket_1_objects.append(copy_obj_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects)
obj_head = s3_client.head_object(bucket, copy_obj_path)
assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
with reporter.step("Copy one object with metadata"):
with allure.step("Copy one object with metadata"):
copy_obj_path = s3_client.copy_object(bucket, file_name, metadata_directive="COPY")
bucket_1_objects.append(copy_obj_path)
obj_head = s3_client.head_object(bucket, copy_obj_path)
assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
with reporter.step("Copy one object with new metadata"):
with allure.step("Copy one object with new metadata"):
object_metadata_1 = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
copy_obj_path = s3_client.copy_object(
bucket,
@ -218,13 +177,13 @@ class TestS3GateObject:
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
s3_client.put_object(bucket, file_path)
s3_client.put_object_tagging(bucket, file_name_simple, tags=object_tagging)
bucket_1_objects = [file_name_simple]
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects)
with reporter.step("Copy one object without tag"):
with allure.step("Copy one object without tag"):
copy_obj_path = s3_client.copy_object(bucket, file_name_simple)
got_tags = s3_client.get_object_tagging(bucket, copy_obj_path)
assert got_tags, f"Expected tags, got {got_tags}"
@ -232,7 +191,7 @@ class TestS3GateObject:
for tag in expected_tags:
assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
with reporter.step("Copy one object with tag"):
with allure.step("Copy one object with tag"):
copy_obj_path_1 = s3_client.copy_object(bucket, file_name_simple, tagging_directive="COPY")
got_tags = s3_client.get_object_tagging(bucket, copy_obj_path_1)
assert got_tags, f"Expected tags, got {got_tags}"
@ -240,7 +199,7 @@ class TestS3GateObject:
for tag in expected_tags:
assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
with reporter.step("Copy one object with new tag"):
with allure.step("Copy one object with new tag"):
tag_key = "tag1"
tag_value = uuid.uuid4()
new_tag = f"{tag_key}={tag_value}"
@ -271,12 +230,14 @@ class TestS3GateObject:
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple)
file_name_1 = generate_file_with_content(simple_object_size.value, file_name_simple, version_2_content)
file_name_1 = generate_file_with_content(
simple_object_size.value, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_client.put_object(bucket, file_name_1)
with reporter.step("Check bucket shows all versions"):
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {
@ -284,26 +245,26 @@ class TestS3GateObject:
version_id_2,
}, f"Object should have versions: {version_id_1, version_id_2}"
with reporter.step("Delete 1 version of object"):
with allure.step("Delete 1 version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_1)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
with reporter.step("Delete second version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id_2)
with allure.step("Delete second version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_2)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert not obj_versions, "Expected object not found"
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
with reporter.step("Put new object into bucket"):
file_name_complex = generate_file(complex_object_size.value)
obj_key = os.path.basename(file_name_complex)
s3_client.put_object(bucket, file_name_complex)
with allure.step("Put new object into bucket"):
file_name_simple = generate_file(complex_object_size.value)
obj_key = os.path.basename(file_name_simple)
s3_client.put_object(bucket, file_name_simple)
with reporter.step("Delete last object"):
with allure.step("Delete last object"):
delete_obj = s3_client.delete_object(bucket, obj_key)
versions = s3_client.list_objects_versions(bucket, True)
assert versions.get("DeleteMarkers", None), "Expected delete Marker"
@ -320,28 +281,34 @@ class TestS3GateObject:
obj_key = os.path.basename(file_name_1)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_1)
file_name_2 = generate_file_with_content(simple_object_size.value, file_name_1, version_2_content)
file_name_2 = generate_file_with_content(
simple_object_size.value, file_path=file_name_1, content=version_2_content
)
version_id_2 = s3_client.put_object(bucket, file_name_2)
file_name_3 = generate_file_with_content(simple_object_size.value, file_name_1, version_3_content)
file_name_3 = generate_file_with_content(
simple_object_size.value, file_path=file_name_1, content=version_3_content
)
version_id_3 = s3_client.put_object(bucket, file_name_3)
file_name_4 = generate_file_with_content(simple_object_size.value, file_name_1, version_4_content)
file_name_4 = generate_file_with_content(
simple_object_size.value, file_path=file_name_1, content=version_4_content
)
version_id_4 = s3_client.put_object(bucket, file_name_4)
version_ids = {version_id_1, version_id_2, version_id_3, version_id_4}
with reporter.step("Check bucket shows all versions"):
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
with reporter.step("Delete two objects from bucket one by one"):
version_to_delete_b1 = random.sample([version_id_1, version_id_2, version_id_3, version_id_4], k=2)
with allure.step("Delete two objects from bucket one by one"):
version_to_delete_b1 = sample([version_id_1, version_id_2, version_id_3, version_id_4], k=2)
version_to_save = list(set(version_ids) - set(version_to_delete_b1))
for ver in version_to_delete_b1:
s3_client.delete_object(bucket, obj_key, ver)
with reporter.step("Check bucket shows all versions"):
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == obj_key]
assert obj_versions.sort() == version_to_save.sort(), f"Object should have versions: {version_to_save}"
@ -354,22 +321,22 @@ class TestS3GateObject:
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple)
file_name_1 = generate_file_with_content(
simple_object_size.value, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_client.put_object(bucket, file_name_1)
with reporter.step("Get first version of object"):
with allure.step("Get first version of object"):
object_1 = s3_client.get_object(bucket, obj_key, version_id_1, full_output=True)
assert object_1.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
with reporter.step("Get second version of object"):
with allure.step("Get second version of object"):
object_2 = s3_client.get_object(bucket, obj_key, version_id_2, full_output=True)
assert object_2.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
with reporter.step("Get object"):
with allure.step("Get object"):
object_3 = s3_client.get_object(bucket, obj_key, full_output=True)
assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
@ -385,12 +352,12 @@ class TestS3GateObject:
file_name = s3_helper.object_key_from_file_path(file_path)
file_hash = get_file_hash(file_path)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_path)
file_name_1 = generate_file_with_content(simple_object_size.value, file_path=file_path)
version_id_2 = s3_client.put_object(bucket, file_name_1)
with reporter.step("Get first version of object"):
with allure.step("Get first version of object"):
object_1_part_1 = s3_client.get_object(
bucket,
file_name,
@ -418,7 +385,7 @@ class TestS3GateObject:
con_file = concat_files([object_1_part_1, object_1_part_2, object_1_part_3])
assert get_file_hash(con_file) == file_hash, "Hashes must be the same"
with reporter.step("Get second version of object"):
with allure.step("Get second version of object"):
object_2_part_1 = s3_client.get_object(
bucket,
file_name,
@ -443,7 +410,7 @@ class TestS3GateObject:
con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3])
assert get_file_hash(con_file_1) == get_file_hash(file_name_1), "Hashes must be the same"
with reporter.step("Get object"):
with allure.step("Get object"):
object_3_part_1 = s3_client.get_object(
bucket, file_name, object_range=[0, int(simple_object_size.value / 3)]
)
@ -483,7 +450,7 @@ class TestS3GateObject:
objects_in_bucket = []
objects_count = 3
with reporter.step(f"Put {objects_count} into bucket"):
with allure.step(f"Put {objects_count} into bucket"):
for _ in range(objects_count):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
@ -492,15 +459,15 @@ class TestS3GateObject:
# Extend deletion list to 1001 elements with same keys for test speed
objects_to_delete = self.copy_extend_list(objects_in_bucket, 1001)
with reporter.step("Send delete request with 1001 objects and expect error"):
with allure.step("Send delete request with 1001 objects and expect error"):
with pytest.raises(Exception, match=S3_MALFORMED_XML_REQUEST):
s3_client.delete_objects(bucket, objects_to_delete)
with reporter.step("Send delete request with 1000 objects without error"):
with allure.step("Send delete request with 1000 objects without error"):
with expect_not_raises():
s3_client.delete_objects(bucket, objects_to_delete[:1000])
@allure.title("Object head is unloaded with the correct version (s3_client={s3_client})")
@allure.title("Copy object with metadata (s3_client={s3_client})")
@pytest.mark.smoke
def test_s3_head_object(
self,
@ -514,12 +481,12 @@ class TestS3GateObject:
file_name = s3_helper.object_key_from_file_path(file_path)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_path, metadata=object_metadata)
file_name_1 = generate_file_with_content(simple_object_size.value, file_path=file_path)
version_id_2 = s3_client.put_object(bucket, file_name_1)
with reporter.step("Get head of first version of object"):
with allure.step("Get head of first version of object"):
response = s3_client.head_object(bucket, file_name)
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
@ -527,7 +494,7 @@ class TestS3GateObject:
assert response.get("VersionId") == version_id_2, f"Expected VersionId is {version_id_2}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with reporter.step("Get head ob first version of object"):
with allure.step("Get head ob first version of object"):
response = s3_client.head_object(bucket, file_name, version_id=version_id_1)
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
@ -550,11 +517,11 @@ class TestS3GateObject:
file_name_2 = s3_helper.object_key_from_file_path(file_path_2)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
s3_client.put_object(bucket, file_path_1)
s3_client.put_object(bucket, file_path_2)
with reporter.step("Get list of object"):
with allure.step("Get list of object"):
if list_type == "v1":
list_obj = s3_client.list_objects(bucket)
elif list_type == "v2":
@ -564,7 +531,7 @@ class TestS3GateObject:
list_obj.sort() == [file_name, file_name_2].sort()
), f"bucket should have object key {file_name, file_name_2}"
with reporter.step("Delete object"):
with allure.step("Delete object"):
delete_obj = s3_client.delete_object(bucket, file_name)
if list_type == "v1":
list_obj_1 = s3_client.list_objects(bucket, full_output=True)
@ -595,7 +562,7 @@ class TestS3GateObject:
tag_2 = f"{tag_key_2}={tag_value_2}"
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.SUSPENDED)
with reporter.step("Put first object into bucket"):
with allure.step("Put first object into bucket"):
s3_client.put_object(bucket, file_path_1, metadata=object_1_metadata, tagging=tag_1)
obj_head = s3_client.head_object(bucket, file_name)
assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
@ -603,7 +570,7 @@ class TestS3GateObject:
assert got_tags, f"Expected tags, got {got_tags}"
assert got_tags == [{"Key": tag_key_1, "Value": str(tag_value_1)}], "Tags must be the same"
with reporter.step("Rewrite file into bucket"):
with allure.step("Rewrite file into bucket"):
file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_2, metadata=object_2_metadata, tagging=tag_2)
obj_head = s3_client.head_object(bucket, file_name)
@ -622,7 +589,7 @@ class TestS3GateObject:
tag_value_3 = uuid.uuid4()
tag_3 = f"{tag_key_3}={tag_value_3}"
with reporter.step("Put third object into bucket"):
with allure.step("Put third object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3)
obj_head_3 = s3_client.head_object(bucket, file_name_3)
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same"
@ -630,7 +597,7 @@ class TestS3GateObject:
assert got_tags_3, f"Expected tags, got {got_tags_3}"
assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
with reporter.step("Put new version of file into bucket"):
with allure.step("Put new version of file into bucket"):
file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_3)
version_id_2 = s3_client.put_object(bucket, file_path_4)
versions = s3_client.list_objects_versions(bucket)
@ -642,13 +609,13 @@ class TestS3GateObject:
got_tags_4 = s3_client.get_object_tagging(bucket, file_name_3)
assert not got_tags_4, "No tags expected"
with reporter.step("Get object"):
with allure.step("Get object"):
object_3 = s3_client.get_object(bucket, file_name_3, full_output=True)
assert object_3.get("VersionId") == version_id_2, f"get object with version {version_id_2}"
object_3 = s3_client.get_object(bucket, file_name_3)
assert get_file_hash(file_path_4) == get_file_hash(object_3), "Hashes must be the same"
with reporter.step("Get first version of object"):
with allure.step("Get first version of object"):
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1, full_output=True)
assert object_4.get("VersionId") == version_id_1, f"get object with version {version_id_1}"
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1)
@ -670,38 +637,71 @@ class TestS3GateObject:
simple_object_size: ObjectSize,
second_wallet_public_key: str,
):
file_path = generate_file(complex_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus[bucket_versioning])
file_path_1 = generate_file(complex_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path_1)
if bucket_versioning == "ENABLED":
status = VersioningStatus.ENABLED
elif bucket_versioning == "SUSPENDED":
status = VersioningStatus.SUSPENDED
s3_helper.set_bucket_versioning(s3_client, bucket, status)
with reporter.step("Put object with acl private"):
s3_client.put_object(bucket, file_path, acl="private")
object_grants = s3_client.get_object_acl(bucket, file_name)
s3_helper.verify_acl_permissions(object_grants, PRIVATE_GRANTS)
object = s3_client.get_object(bucket, file_name)
assert get_file_hash(file_path) == get_file_hash(object), "Hashes must be the same"
with allure.step("Put object with acl private"):
s3_client.put_object(bucket, file_path_1, acl="private")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
object_1 = s3_client.get_object(bucket, file_name)
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
with reporter.step("[NEGATIVE] Put object with acl public-read"):
generate_file_with_content(simple_object_size.value, file_path)
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
s3_client.put_object(bucket, file_path, acl="public-read")
with allure.step("Put object with acl public-read"):
file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_2, acl="public-read")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
object_2 = s3_client.get_object(bucket, file_name)
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
with reporter.step("[NEGATIVE] Put object with acl public-read-write"):
generate_file_with_content(simple_object_size.value, file_path)
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
s3_client.put_object(bucket, file_path, acl="public-read-write")
with allure.step("Put object with acl public-read-write"):
file_path_3 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_3, acl="public-read-write")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
object_3 = s3_client.get_object(bucket, file_name)
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
with reporter.step("[NEGATIVE] Put object with --grant-full-control id=mycanonicaluserid"):
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
s3_client.put_object(bucket, file_path, grant_full_control=f"id={second_wallet_public_key}")
with allure.step("Put object with acl authenticated-read"):
file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_4, acl="authenticated-read")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
object_4 = s3_client.get_object(bucket, file_name)
assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same"
with reporter.step(
"[NEGATIVE] Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
s3_client.put_object(
bucket, file_path, grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers"
)
file_path_5 = generate_file(complex_object_size.value)
file_name_5 = s3_helper.object_key_from_file_path(file_path_5)
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
generate_file_with_content(simple_object_size.value, file_path=file_path_5)
s3_client.put_object(
bucket,
file_path_5,
grant_full_control=f"id={second_wallet_public_key}",
)
obj_acl = s3_client.get_object_acl(bucket, file_name_5)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
object_5 = s3_client.get_object(bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
with allure.step("Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
generate_file_with_content(simple_object_size.value, file_path=file_path_5)
s3_client.put_object(
bucket,
file_path_5,
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
obj_acl = s3_client.get_object_acl(bucket, file_name_5)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
object_6 = s3_client.get_object(bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_6), "Hashes must be the same"
@allure.title("Put object with lock-mode (s3_client={s3_client})")
def test_s3_put_object_lock_mode(
@ -716,7 +716,7 @@ class TestS3GateObject:
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"):
with allure.step("Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"):
date_obj = datetime.utcnow() + timedelta(days=1)
s3_client.put_object(
bucket,
@ -727,7 +727,7 @@ class TestS3GateObject:
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
with reporter.step(
with allure.step(
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
):
date_obj = datetime.utcnow() + timedelta(days=2)
@ -740,7 +740,7 @@ class TestS3GateObject:
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with reporter.step(
with allure.step(
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
):
date_obj = datetime.utcnow() + timedelta(days=3)
@ -754,7 +754,7 @@ class TestS3GateObject:
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
with reporter.step("Put object with lock-mode"):
with allure.step("Put object with lock-mode"):
with pytest.raises(
Exception,
match=r".*must both be supplied*",
@ -762,7 +762,7 @@ class TestS3GateObject:
# x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied
s3_client.put_object(bucket, file_path_1, object_lock_mode="COMPLIANCE")
with reporter.step("Put object with lock-mode and past date"):
with allure.step("Put object with lock-mode and past date"):
date_obj = datetime.utcnow() - timedelta(days=3)
with pytest.raises(
Exception,
@ -776,72 +776,7 @@ class TestS3GateObject:
object_lock_retain_until_date=date_obj,
)
@allure.title("Delete object & delete objects (s3_client={s3_client})")
def test_s3_api_delete(
self,
s3_client: S3ClientWrapper,
two_buckets: list[str],
simple_object_size: ObjectSize,
complex_object_size: ObjectSize,
):
"""
Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one.
From second bucket some objects deleted all at once.
"""
max_obj_count = 20
max_delete_objects = 17
put_objects = []
file_paths = []
obj_sizes = [simple_object_size, complex_object_size]
bucket_1, bucket_2 = two_buckets
with reporter.step(f"Generate {max_obj_count} files"):
for _ in range(max_obj_count):
test_file = generate_file(random.choice(obj_sizes).value)
file_paths.append(test_file)
put_objects.append(s3_helper.object_key_from_file_path(test_file.path))
for i, bucket in enumerate([bucket_1, bucket_2], 1):
with reporter.step(f"Put {max_obj_count} objects into bucket_{i}"):
for file_path in file_paths:
s3_client.put_object(bucket, file_path)
with reporter.step(f"Check all objects put in bucket_{i} successfully"):
bucket_objects = s3_client.list_objects_v2(bucket)
assert set(put_objects) == set(
bucket_objects
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
with reporter.step("Delete some objects from bucket_1 one by one"):
objects_to_delete_b1 = random.sample(put_objects, k=max_delete_objects)
for obj in objects_to_delete_b1:
s3_client.delete_object(bucket_1, obj)
with reporter.step("Check deleted objects are not visible in bucket bucket_1"):
bucket_objects = s3_client.list_objects_v2(bucket_1)
assert set(put_objects).difference(set(objects_to_delete_b1)) == set(
bucket_objects
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
for object_key in objects_to_delete_b1:
with pytest.raises(Exception, match="The specified key does not exist"):
s3_client.get_object(bucket_1, object_key)
with reporter.step("Delete some objects from bucket_2 at once"):
objects_to_delete_b2 = random.sample(put_objects, k=max_delete_objects)
s3_client.delete_objects(bucket_2, objects_to_delete_b2)
with reporter.step("Check deleted objects are not visible in bucket bucket_2"):
objects_list = s3_client.list_objects_v2(bucket_2)
assert set(put_objects).difference(set(objects_to_delete_b2)) == set(
objects_list
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
for object_key in objects_to_delete_b2:
with pytest.raises(Exception, match="The specified key does not exist"):
s3_client.get_object(bucket_2, object_key)
@allure.title("Sync directory (sync_type={sync_type}, s3_client={s3_client})")
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
@pytest.mark.parametrize("sync_type", ["sync", "cp"])
def test_s3_sync_dir(
self,
@ -850,25 +785,38 @@ class TestS3GateObject:
bucket: str,
simple_object_size: ObjectSize,
):
test_file_1 = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1"))
test_file_2 = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2"))
file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1")
file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2")
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
key_to_path = {"test_file_1": test_file_1.path, "test_file_2": test_file_2.path}
key_to_path = {"test_file_1": file_path_1, "test_file_2": file_path_2}
generate_file_with_content(simple_object_size.value, test_file_1)
generate_file_with_content(simple_object_size.value, test_file_2)
if not isinstance(s3_client, AwsCliClient):
pytest.skip("This test is not supported with boto3 client")
generate_file_with_content(simple_object_size.value, file_path=file_path_1)
generate_file_with_content(simple_object_size.value, file_path=file_path_2)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
# TODO: return ACL, when https://github.com/nspcc-dev/neofs-s3-gw/issues/685 will be closed
if sync_type == "sync":
s3_client.sync(bucket, os.path.dirname(test_file_1), metadata=object_metadata)
s3_client.sync(
bucket=bucket,
dir_path=os.path.dirname(file_path_1),
# acl="public-read-write",
metadata=object_metadata,
)
elif sync_type == "cp":
s3_client.cp(bucket, os.path.dirname(test_file_1), metadata=object_metadata)
s3_client.cp(
bucket=bucket,
dir_path=os.path.dirname(file_path_1),
# acl="public-read-write",
metadata=object_metadata,
)
with reporter.step("Check objects are synced"):
with allure.step("Check objects are synced"):
objects = s3_client.list_objects(bucket)
assert set(key_to_path.keys()) == set(objects), f"Expected all abjects saved. Got {objects}"
with reporter.step("Check these are the same objects"):
with allure.step("Check these are the same objects"):
for obj_key in objects:
got_object = s3_client.get_object(bucket, obj_key)
assert get_file_hash(got_object) == get_file_hash(
@ -876,41 +824,42 @@ class TestS3GateObject:
), "Expected hashes are the same"
obj_head = s3_client.head_object(bucket, obj_key)
assert obj_head.get("Metadata") == object_metadata, f"Metadata of object is {object_metadata}"
object_grants = s3_client.get_object_acl(bucket, obj_key)
s3_helper.verify_acl_permissions(object_grants, PRIVATE_GRANTS)
# Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved
# obj_acl = s3_client.get_object_acl(bucket, obj_key)
# s3_helper.assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers")
@allure.title("Put 10 nested level object (s3_client={s3_client})")
def test_s3_put_10_folder(
self,
s3_client: S3ClientWrapper,
bucket: str,
temp_directory,
simple_object_size: ObjectSize,
):
key_characters_sample = string.ascii_letters + string.digits + "._-"
path = "/".join(["".join(choices(string.ascii_letters, k=3)) for _ in range(10)])
file_path_1 = os.path.join(temp_directory, path, "test_file_1")
generate_file_with_content(simple_object_size.value, file_path=file_path_1)
file_name = s3_helper.object_key_from_file_path(file_path_1)
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with reporter.step("Put object"):
test_file = generate_file(simple_object_size.value)
obj_key = (
"/"
+ "/".join(["".join(random.choices(key_characters_sample, k=5)) for _ in range(10)])
+ "/test_file_1"
)
s3_client.put_object(bucket, test_file, obj_key)
with reporter.step("Check object can be downloaded"):
s3_client.get_object(bucket, obj_key)
with reporter.step("Check object listing"):
s3_helper.check_objects_in_bucket(s3_client, bucket, [obj_key])
with allure.step("Put object"):
s3_client.put_object(bucket, file_path_1)
s3_helper.check_objects_in_bucket(s3_client, bucket, [file_name])
@allure.title("Delete non-existing object from empty bucket (s3_client={s3_client})")
def test_s3_delete_non_existing_object(self, s3_client: S3ClientWrapper, bucket: str):
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
objects_list = s3_client.list_objects_versions(bucket)
with allure.step("Check that bucket is empty"):
assert not objects_list, f"Expected empty bucket, got {objects_list}"
obj_key = "fake_object_key"
with reporter.step("Delete non-existing object"):
with allure.step("Delete non-existing object"):
delete_obj = s3_client.delete_object(bucket, obj_key)
# there should be no objects or delete markers in the bucket
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
objects_list = s3_client.list_objects_versions(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
@ -918,22 +867,25 @@ class TestS3GateObject:
@allure.title("Delete the same object twice (s3_client={s3_client})")
def test_s3_delete_twice(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
objects_list = s3_client.list_objects(bucket)
with allure.step("Check that bucket is empty"):
assert not objects_list, f"Expected empty bucket, got {objects_list}"
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put object into one bucket"):
with allure.step("Put object into one bucket"):
s3_client.put_object(bucket, file_path)
with reporter.step("Delete the object from the bucket"):
with allure.step("Delete the object from the bucket"):
delete_object = s3_client.delete_object(bucket, file_name)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name}
assert obj_versions, f"Object versions were not found {versions}"
assert obj_versions, f"Object versions were not found {objects_list}"
assert "DeleteMarker" in delete_object.keys(), "Delete markers not found"
with reporter.step("Delete the object from the bucket again"):
with allure.step("Delete the object from the bucket again"):
delete_object_2nd_attempt = s3_client.delete_object(bucket, file_name)
versions_2nd_attempt = s3_client.list_objects_versions(bucket)

View file

@ -1,16 +1,12 @@
import json
import os
import allure
import pytest
from botocore.exceptions import ClientError
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.storage_policy import get_simple_object_copies
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file
@ -20,15 +16,13 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True)
class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
def test_s3_bucket_location(
self, default_wallet: WalletInfo, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
):
def test_s3_bucket_location(self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path_1 = generate_file(simple_object_size.value)
file_name_1 = s3_helper.object_key_from_file_path(file_path_1)
file_path_2 = generate_file(simple_object_size.value)
file_name_2 = s3_helper.object_key_from_file_path(file_path_2)
with reporter.step("Create two buckets with different bucket configuration"):
with allure.step("Create two buckets with different bucket configuration"):
bucket_1 = s3_client.create_bucket(location_constraint="complex")
s3_helper.set_bucket_versioning(s3_client, bucket_1, VersioningStatus.ENABLED)
bucket_2 = s3_client.create_bucket(location_constraint="rep-3")
@ -38,28 +32,30 @@ class TestS3GatePolicy(ClusterTestBase):
bucket_1 in list_buckets and bucket_2 in list_buckets
), f"Expected two buckets {bucket_1, bucket_2}, got {list_buckets}"
with reporter.step("Check head buckets"):
with allure.step("Check head buckets"):
with expect_not_raises():
s3_client.head_bucket(bucket_1)
s3_client.head_bucket(bucket_2)
with reporter.step("Put objects into buckets"):
with allure.step("Put objects into buckets"):
version_id_1 = s3_client.put_object(bucket_1, file_path_1)
version_id_2 = s3_client.put_object(bucket_2, file_path_2)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, [file_name_1])
s3_helper.check_objects_in_bucket(s3_client, bucket_2, [file_name_2])
with reporter.step("Check bucket location"):
with allure.step("Check bucket location"):
bucket_loc_1 = s3_client.get_bucket_location(bucket_1)
bucket_loc_2 = s3_client.get_bucket_location(bucket_2)
assert bucket_loc_1 == "complex"
assert bucket_loc_2 == "rep-3"
with reporter.step("Check object policy"):
for cluster_node in self.cluster.cluster_nodes:
cid_1 = search_container_by_name(name=bucket_1, node=cluster_node)
if cid_1:
break
with allure.step("Check object policy"):
cid_1 = search_container_by_name(
default_wallet,
bucket_1,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
copies_1 = get_simple_object_copies(
wallet=default_wallet,
cid=cid_1,
@ -68,10 +64,12 @@ class TestS3GatePolicy(ClusterTestBase):
nodes=self.cluster.storage_nodes,
)
assert copies_1 == 1
for cluster_node in self.cluster.cluster_nodes:
cid_2 = search_container_by_name(name=bucket_2, node=cluster_node)
if cid_2:
break
cid_2 = search_container_by_name(
default_wallet,
bucket_2,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
copies_2 = get_simple_object_copies(
wallet=default_wallet,
cid=cid_2,
@ -83,56 +81,50 @@ class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket with unexisting location constraint (s3_client={s3_client})")
def test_s3_bucket_wrong_location(self, s3_client: S3ClientWrapper):
with reporter.step("Create bucket with unenxisting location constraint policy"):
with allure.step("Create bucket with unenxisting location constraint policy"):
with pytest.raises(Exception):
s3_client.create_bucket(location_constraint="UNEXISTING LOCATION CONSTRAINT")
@allure.title("Bucket policy (s3_client={s3_client})")
def test_s3_bucket_policy(self, s3_client: S3ClientWrapper, bucket: str):
with reporter.step("Create bucket"):
def test_s3_bucket_policy(self, s3_client: S3ClientWrapper):
with allure.step("Create bucket with default policy"):
bucket = s3_client.create_bucket()
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("GetBucketPolicy"):
with pytest.raises((RuntimeError, ClientError)):
s3_client.get_bucket_policy(bucket)
with allure.step("GetBucketPolicy"):
s3_client.get_bucket_policy(bucket)
with reporter.step("Put new policy"):
with allure.step("Put new policy"):
custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json"
custom_policy = {
"Version": "2012-10-17",
"Version": "2008-10-17",
"Id": "aaaa-bbbb-cccc-dddd",
"Statement": [
{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": "*",
"Principal": {"AWS": "*"},
"Action": ["s3:GetObject"],
"Resource": [f"arn:aws:s3:::{bucket}/*"],
}
],
}
s3_client.put_bucket_policy(bucket, custom_policy)
with reporter.step("GetBucketPolicy"):
returned_policy = json.loads(s3_client.get_bucket_policy(bucket))
assert returned_policy == custom_policy, "Wrong policy was received"
with reporter.step("Delete the policy"):
s3_client.delete_bucket_policy(bucket)
with reporter.step("GetBucketPolicy"):
with pytest.raises((RuntimeError, ClientError)):
s3_client.get_bucket_policy(bucket)
with allure.step("GetBucketPolicy"):
policy_1 = s3_client.get_bucket_policy(bucket)
print(policy_1)
@allure.title("Bucket CORS (s3_client={s3_client})")
def test_s3_cors(self, s3_client: S3ClientWrapper, bucket: str):
with reporter.step("Create bucket without cors"):
def test_s3_cors(self, s3_client: S3ClientWrapper):
with allure.step("Create bucket without cors"):
bucket = s3_client.create_bucket()
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with pytest.raises(Exception):
bucket_cors = s3_client.get_bucket_cors(bucket)
with reporter.step("Put bucket cors"):
with allure.step("Put bucket cors"):
cors = {
"CORSRules": [
{
@ -154,7 +146,7 @@ class TestS3GatePolicy(ClusterTestBase):
bucket_cors = s3_client.get_bucket_cors(bucket)
assert bucket_cors == cors.get("CORSRules"), f"Expected CORSRules must be {cors.get('CORSRules')}"
with reporter.step("delete bucket cors"):
with allure.step("delete bucket cors"):
s3_client.delete_bucket_cors(bucket)
with pytest.raises(Exception):

View file

@ -4,7 +4,6 @@ from typing import Tuple
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -28,76 +27,76 @@ class TestS3GateTagging:
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with reporter.step("Put with 3 tags object into bucket"):
with allure.step("Put with 3 tags object into bucket"):
tag_1 = "Tag1=Value1"
s3_client.put_object(bucket, file_path, tagging=tag_1)
got_tags = s3_client.get_object_tagging(bucket, file_name)
assert got_tags, f"Expected tags, got {got_tags}"
assert got_tags == [{"Key": "Tag1", "Value": "Value1"}], "Tags must be the same"
with reporter.step("Put 10 new tags for object"):
with allure.step("Put 10 new tags for object"):
tags_2 = self.create_tags(10)
s3_client.put_object_tagging(bucket, file_name, tags=tags_2)
s3_helper.check_tags_by_object(s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")])
with reporter.step("Put 10 extra new tags for object"):
with allure.step("Put 10 extra new tags for object"):
tags_3 = self.create_tags(10)
s3_client.put_object_tagging(bucket, file_name, tags=tags_3)
s3_helper.check_tags_by_object(s3_client, bucket, file_name, tags_3, tags_2)
with reporter.step("Copy one object with tag"):
with allure.step("Copy one object with tag"):
copy_obj_path_1 = s3_client.copy_object(bucket, file_name, tagging_directive="COPY")
s3_helper.check_tags_by_object(s3_client, bucket, copy_obj_path_1, tags_3, tags_2)
with reporter.step("Put 11 new tags to object and expect an error"):
with allure.step("Put 11 new tags to object and expect an error"):
tags_4 = self.create_tags(11)
with pytest.raises(Exception, match=r".*Object tags cannot be greater than 10*"):
# An error occurred (BadRequest) when calling the PutObjectTagging operation: Object tags cannot be greater than 10
s3_client.put_object_tagging(bucket, file_name, tags=tags_4)
with reporter.step("Put empty tag"):
with allure.step("Put empty tag"):
tags_5 = []
s3_client.put_object_tagging(bucket, file_name, tags=tags_5)
s3_helper.check_tags_by_object(s3_client, bucket, file_name, [])
with reporter.step("Put 10 object tags"):
with allure.step("Put 10 object tags"):
tags_6 = self.create_tags(10)
s3_client.put_object_tagging(bucket, file_name, tags=tags_6)
s3_helper.check_tags_by_object(s3_client, bucket, file_name, tags_6)
with reporter.step("Delete tags by delete-object-tagging"):
with allure.step("Delete tags by delete-object-tagging"):
s3_client.delete_object_tagging(bucket, file_name)
s3_helper.check_tags_by_object(s3_client, bucket, file_name, [])
@allure.title("Bucket tagging (s3_client={s3_client})")
def test_s3_bucket_tagging(self, s3_client: S3ClientWrapper, bucket: str):
with reporter.step("Put 10 bucket tags"):
with allure.step("Put 10 bucket tags"):
tags_1 = self.create_tags(10)
s3_client.put_bucket_tagging(bucket, tags_1)
s3_helper.check_tags_by_bucket(s3_client, bucket, tags_1)
with reporter.step("Put new 10 bucket tags"):
with allure.step("Put new 10 bucket tags"):
tags_2 = self.create_tags(10)
s3_client.put_bucket_tagging(bucket, tags_2)
s3_helper.check_tags_by_bucket(s3_client, bucket, tags_2, tags_1)
with reporter.step("Put 11 new tags to bucket and expect an error"):
with allure.step("Put 11 new tags to bucket and expect an error"):
tags_3 = self.create_tags(11)
with pytest.raises(Exception, match=r".*Object tags cannot be greater than 10.*"):
# An error occurred (BadRequest) when calling the PutBucketTagging operation (reached max retries: 0): Object tags cannot be greater than 10
s3_client.put_bucket_tagging(bucket, tags_3)
with reporter.step("Put empty tag"):
with allure.step("Put empty tag"):
tags_4 = []
s3_client.put_bucket_tagging(bucket, tags_4)
s3_helper.check_tags_by_bucket(s3_client, bucket, tags_4)
with reporter.step("Put new 10 bucket tags"):
with allure.step("Put new 10 bucket tags"):
tags_5 = self.create_tags(10)
s3_client.put_bucket_tagging(bucket, tags_5)
s3_helper.check_tags_by_bucket(s3_client, bucket, tags_5, tags_2)
with reporter.step("Delete tags by delete-bucket-tagging"):
with allure.step("Delete tags by delete-bucket-tagging"):
s3_client.delete_bucket_tagging(bucket)
s3_helper.check_tags_by_bucket(s3_client, bucket, [])

View file

@ -1,12 +1,9 @@
import os
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content, get_file_content
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
@pytest.mark.s3_gate
@ -18,69 +15,6 @@ class TestS3GateVersioning:
with pytest.raises(Exception):
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.SUSPENDED)
@allure.title("Object versioning (s3_client={s3_client})")
def test_s3_api_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks basic versioning functionality for S3 bucket.
"""
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple)
generate_file_with_content(simple_object_size.value, file_path=file_name_simple, content=version_2_content)
version_id_2 = s3_client.put_object(bucket, file_name_simple)
with reporter.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {
version_id_1,
version_id_2,
}, f"Expected object has versions: {version_id_1, version_id_2}"
with reporter.step("Show information about particular version"):
for version_id in (version_id_1, version_id_2):
response = s3_client.head_object(bucket, obj_key, version_id=version_id)
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
assert response.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with reporter.step("Check object's attributes"):
for version_id in (version_id_1, version_id_2):
got_attrs = s3_client.get_object_attributes(bucket, obj_key, ["ETag"], version_id=version_id)
if got_attrs:
assert got_attrs.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
with reporter.step("Delete object and check it was deleted"):
response = s3_client.delete_object(bucket, obj_key)
version_id_delete = response.get("VersionId")
with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_object(bucket, obj_key)
with reporter.step("Get content for all versions and check it is correct"):
for version, content in (
(version_id_2, version_2_content),
(version_id_1, version_1_content),
):
file_name = s3_client.get_object(bucket, obj_key, version_id=version)
got_content = get_file_content(file_name)
assert got_content == content, f"Expected object content is\n{content}\nGot\n{got_content}"
with reporter.step("Restore previous object version"):
s3_client.delete_object(bucket, obj_key, version_id=version_id_delete)
file_name = s3_client.get_object(bucket, obj_key)
got_content = get_file_content(file_name)
assert (
got_content == version_2_content
), f"Expected object content is\n{version_2_content}\nGot\n{got_content}"
@allure.title("Enable and disable versioning without object_lock (s3_client={s3_client})")
def test_s3_version(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
@ -89,7 +23,7 @@ class TestS3GateVersioning:
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.SUSPENDED)
with reporter.step("Put object into bucket"):
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
objects_list = s3_client.list_objects(bucket)
assert objects_list == bucket_objects, f"Expected list with single objects in bucket, got {objects_list}"
@ -103,26 +37,26 @@ class TestS3GateVersioning:
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_path)
file_name_1 = generate_file_with_content(simple_object_size.value, file_path=file_path)
version_id_2 = s3_client.put_object(bucket, file_name_1)
with reporter.step("Check bucket shows all versions"):
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == file_name]
assert (
obj_versions.sort() == [version_id_1, version_id_2, "null"].sort()
), f"Expected object has versions: {version_id_1, version_id_2, 'null'}"
with reporter.step("Get object"):
with allure.step("Get object"):
object_1 = s3_client.get_object(bucket, file_name, full_output=True)
assert object_1.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
with reporter.step("Get first version of object"):
with allure.step("Get first version of object"):
object_2 = s3_client.get_object(bucket, file_name, version_id_1, full_output=True)
assert object_2.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
with reporter.step("Get second version of object"):
with allure.step("Get second version of object"):
object_3 = s3_client.get_object(bucket, file_name, version_id_2, full_output=True)
assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"

View file

@ -1,42 +1,72 @@
import logging
from re import fullmatch
import os
from http import HTTPStatus
from re import fullmatch, match
import allure
import pytest
from frostfs_testlib import reporter
import requests
from frostfs_testlib.hosting import Hosting
from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.utils.env_utils import read_env_properties, save_env_properties
from frostfs_testlib.utils.version_utils import get_remote_binaries_versions
from pytest import FixtureRequest
logger = logging.getLogger("NeoLogger")
VERSION_REGEX = r"^([a-zA-Z0-9]*/)?\d+\.\d+\.\d+(-.*)?(?<!dirty)"
VERSION_ERROR_MSG = "{name} [{host}]: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}"
def _check_version_format(version):
return fullmatch(VERSION_REGEX, version)
@allure.title("Check binaries versions")
@pytest.mark.check_binaries
def test_binaries_versions(hosting: Hosting):
def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
"""
Compare binaries versions from external source (url) and deployed on servers.
"""
with reporter.step("Get binaries versions from servers"):
versions_by_host = get_remote_binaries_versions(hosting)
with allure.step("Get binaries versions from servers"):
got_versions = get_remote_binaries_versions(hosting)
exсeptions = []
environment_dir = request.config.getoption("--alluredir") or ASSETS_DIR
env_file = os.path.join(environment_dir, "environment.properties")
env_properties = read_env_properties(env_file)
last_host, versions_on_last_host = versions_by_host.popitem()
for name, version in versions_on_last_host.items():
for host, versions_on_host in versions_by_host.items():
if versions_on_host[name] != version:
exсeptions.append(f"Binary of {name} has inconsistent version {versions_on_host[name]} on host {host}")
if not _check_version_format(versions_on_host[name]):
exсeptions.append(VERSION_ERROR_MSG.format(name=name, host=host, version=version))
# compare versions from servers and file
exeptions = []
additional_env_properties = {}
if not _check_version_format(version):
exсeptions.append(VERSION_ERROR_MSG.format(name=name, host=last_host, version=version))
for binary, version in got_versions.items():
if not fullmatch(r"^\d+\.\d+\.\d+(-.*)?(?<!dirty)", version):
exeptions.append(f"{binary}: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}")
assert not exсeptions, "\n".join(exсeptions)
# If some binary was not listed in the env properties file, let's add it
# so that we have full information about versions in allure report
if env_properties and binary not in env_properties:
additional_env_properties[binary] = version
if env_properties and additional_env_properties:
save_env_properties(env_file, additional_env_properties)
# create clear beautiful error with aggregation info
if exeptions:
msg = "\n".join(exeptions)
raise AssertionError(f"Found binaries with unexpected versions:\n{msg}")
@allure.step("Download versions info from {url}")
def download_versions_info(url: str) -> dict:
binaries_to_version = {}
response = requests.get(url)
assert response.status_code == HTTPStatus.OK, f"Got {response.status_code} code. Content {response.json()}"
content = response.text
assert content, f"Expected file with content, got {response}"
for line in content.split("\n"):
m = match("(.*)=(.*)", line)
if not m:
logger.warning(f"Could not get binary/version from {line}")
continue
bin_name, bin_version = m.group(1), m.group(2)
binaries_to_version[bin_name] = bin_version
return binaries_to_version

View file

@ -1,28 +1,26 @@
from datetime import datetime
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
@pytest.fixture(scope="module")
def owner_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create user wallet which owns containers and objects"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
def owner_wallet(wallet_factory: WalletFactory) -> WalletInfo:
"""
Returns wallet which owns containers and objects
"""
return wallet_factory.create_wallet()
@pytest.fixture(scope="module")
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create user wallet which will use objects from owner via static session"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
def user_wallet(wallet_factory: WalletFactory) -> WalletInfo:
"""
Returns wallet which will use objects from owner via static session
"""
return wallet_factory.create_wallet()
@pytest.fixture(scope="module")
def stranger_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create stranger user wallet which should fail to obtain data"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
def stranger_wallet(wallet_factory: WalletFactory) -> WalletInfo:
"""
Returns stranger wallet which should fail to obtain data
"""
return wallet_factory.create_wallet()

View file

@ -2,13 +2,12 @@ import random
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import DEFAULT_WALLET_PASS
from frostfs_testlib.resources.error_patterns import SESSION_NOT_FOUND
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import delete_object, put_object, put_object_to_random_node
from frostfs_testlib.steps.session_token import create_session_token
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import generate_file
@ -18,7 +17,7 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.session_token
class TestDynamicObjectSession(ClusterTestBase):
@allure.title("Object Operations with Session Token (obj_size={object_size})")
def test_object_session_token(self, default_wallet: WalletInfo, object_size: ObjectSize):
def test_object_session_token(self, default_wallet: str, object_size: ObjectSize):
"""
Test how operations over objects are executed with a session token
@ -32,21 +31,25 @@ class TestDynamicObjectSession(ClusterTestBase):
with a session token
"""
with reporter.step("Init wallet"):
with allure.step("Init wallet"):
wallet = default_wallet
address = wallet_utils.get_last_address_from_wallet(wallet, "")
with reporter.step("Nodes Settlements"):
session_token_node, container_node, non_container_node = random.sample(self.cluster.storage_nodes, 3)
with allure.step("Nodes Settlements"):
session_token_node, container_node, non_container_node = random.sample(
self.cluster.storage_nodes, 3
)
with reporter.step("Create Session Token"):
with allure.step("Create Session Token"):
session_token = create_session_token(
shell=self.shell,
owner=default_wallet.get_address(),
wallet=default_wallet,
owner=address,
wallet_path=wallet,
wallet_password=DEFAULT_WALLET_PASS,
rpc_endpoint=session_token_node.get_rpc_endpoint(),
)
with reporter.step("Create Private Container"):
with allure.step("Create Private Container"):
un_locode = container_node.get_un_locode()
locode = "SPB" if un_locode == "RU LED" else un_locode.split()[1]
placement_policy = (
@ -61,7 +64,7 @@ class TestDynamicObjectSession(ClusterTestBase):
rule=placement_policy,
)
with reporter.step("Put Objects"):
with allure.step("Put Objects"):
file_path = generate_file(object_size.value)
oid = put_object_to_random_node(
wallet=wallet,
@ -78,7 +81,7 @@ class TestDynamicObjectSession(ClusterTestBase):
cluster=self.cluster,
)
with reporter.step("Node not in container but granted a session token"):
with allure.step("Node not in container but granted a session token"):
put_object(
wallet=wallet,
path=file_path,
@ -96,7 +99,7 @@ class TestDynamicObjectSession(ClusterTestBase):
session=session_token,
)
with reporter.step("Node in container and not granted a session token"):
with allure.step("Node in container and not granted a session token"):
with pytest.raises(Exception, match=SESSION_NOT_FOUND):
put_object(
wallet=wallet,
@ -116,7 +119,7 @@ class TestDynamicObjectSession(ClusterTestBase):
session=session_token,
)
with reporter.step("Node not in container and not granted a session token"):
with allure.step("Node not in container and not granted a session token"):
with pytest.raises(Exception, match=SESSION_NOT_FOUND):
put_object(
wallet=wallet,

View file

@ -2,7 +2,6 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import (
EXPIRED_SESSION_TOKEN,
MALFORMED_REQUEST,
@ -50,8 +49,8 @@ RANGE_OFFSET_FOR_COMPLEX_OBJECT = 200
@pytest.fixture(scope="module")
def storage_containers(owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> list[str]:
cid = create_container(owner_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
other_cid = create_container(owner_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
other_cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
yield [cid, other_cid]
@ -70,11 +69,11 @@ def storage_objects(
file_path = generate_file(object_size.value)
storage_objects = []
with reporter.step("Put objects"):
with allure.step("Put objects"):
# upload couple objects
for _ in range(3):
storage_object_id = put_object_to_random_node(
wallet=owner_wallet,
wallet=owner_wallet.path,
path=file_path,
cid=storage_containers[0],
shell=client_shell,
@ -83,7 +82,7 @@ def storage_objects(
storage_object = StorageObjectInfo(storage_containers[0], storage_object_id)
storage_object.size = object_size.value
storage_object.wallet = owner_wallet
storage_object.wallet_file_path = owner_wallet.path
storage_object.file_path = file_path
storage_objects.append(storage_object)
@ -93,7 +92,7 @@ def storage_objects(
delete_objects(storage_objects, client_shell, cluster)
@reporter.step("Get ranges for test")
@allure.step("Get ranges for test")
def get_ranges(storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str) -> list[str]:
"""
Returns ranges to test range/hash methods via static session
@ -163,7 +162,7 @@ class TestObjectStaticSession(ClusterTestBase):
for node in self.cluster.storage_nodes:
for storage_object in storage_objects[0:2]:
method_under_test(
wallet=user_wallet,
wallet=user_wallet.path,
cid=storage_object.cid,
oid=storage_object.oid,
shell=self.shell,
@ -193,10 +192,10 @@ class TestObjectStaticSession(ClusterTestBase):
ranges_to_test = get_ranges(storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint)
for range_to_test in ranges_to_test:
with reporter.step(f"Check range {range_to_test}"):
with allure.step(f"Check range {range_to_test}"):
with expect_not_raises():
method_under_test(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
shell=self.shell,
@ -219,7 +218,7 @@ class TestObjectStaticSession(ClusterTestBase):
cid = storage_objects[0].cid
expected_object_ids = [storage_object.oid for storage_object in storage_objects[0:2]]
actual_object_ids = search_object(
user_wallet,
user_wallet.path,
cid,
self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -240,7 +239,7 @@ class TestObjectStaticSession(ClusterTestBase):
"""
with pytest.raises(Exception, match=UNRELATED_OBJECT):
head_object(
user_wallet,
user_wallet.path,
storage_objects[2].cid,
storage_objects[2].oid,
self.shell,
@ -262,7 +261,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=UNRELATED_KEY):
head_object(
stranger_wallet,
stranger_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -284,7 +283,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=WRONG_VERB):
get_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -307,7 +306,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=UNRELATED_CONTAINER):
get_object_from_random_node(
user_wallet,
user_wallet.path,
storage_containers[1],
storage_object.oid,
self.shell,
@ -341,7 +340,7 @@ class TestObjectStaticSession(ClusterTestBase):
signed_token_file = sign_session_token(self.shell, session_token_file, stranger_wallet)
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
head_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -375,7 +374,7 @@ class TestObjectStaticSession(ClusterTestBase):
signed_token_file = sign_session_token(self.shell, session_token_file, owner_wallet)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(
user_wallet,
user_wallet.path,
container,
storage_object.oid,
self.shell,
@ -407,7 +406,7 @@ class TestObjectStaticSession(ClusterTestBase):
)
with pytest.raises(Exception, match=INVALID_SIGNATURE):
head_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -433,7 +432,7 @@ class TestObjectStaticSession(ClusterTestBase):
object_id = storage_objects[0].oid
expiration = Lifetime(epoch + 1, epoch, epoch)
with reporter.step("Create session token"):
with allure.step("Create session token"):
token_expire_at_next_epoch = get_object_signed_token(
owner_wallet,
user_wallet,
@ -445,10 +444,10 @@ class TestObjectStaticSession(ClusterTestBase):
expiration,
)
with reporter.step("Object should be available with session token after token creation"):
with allure.step("Object should be available with session token after token creation"):
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -456,11 +455,11 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_expire_at_next_epoch,
)
with reporter.step("Object should be available at last epoch before session token expiration"):
with allure.step("Object should be available at last epoch before session token expiration"):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -468,11 +467,11 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_expire_at_next_epoch,
)
with reporter.step("Object should NOT be available after session token expiration epoch"):
with allure.step("Object should NOT be available after session token expiration epoch"):
self.tick_epoch()
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -499,7 +498,7 @@ class TestObjectStaticSession(ClusterTestBase):
object_id = storage_objects[0].oid
expiration = Lifetime(epoch + 2, epoch + 1, epoch)
with reporter.step("Create session token"):
with allure.step("Create session token"):
token_start_at_next_epoch = get_object_signed_token(
owner_wallet,
user_wallet,
@ -511,10 +510,10 @@ class TestObjectStaticSession(ClusterTestBase):
expiration,
)
with reporter.step("Object should NOT be available with session token after token creation"):
with allure.step("Object should NOT be available with session token after token creation"):
with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -522,11 +521,11 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch,
)
with reporter.step("Object should be available with session token starting from token nbf epoch"):
with allure.step("Object should be available with session token starting from token nbf epoch"):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -534,11 +533,11 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch,
)
with reporter.step("Object should be available at last epoch before session token expiration"):
with allure.step("Object should be available at last epoch before session token expiration"):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -546,11 +545,11 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch,
)
with reporter.step("Object should NOT be available after session token expiration epoch"):
with allure.step("Object should NOT be available after session token expiration epoch"):
self.tick_epoch()
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -589,7 +588,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -610,7 +609,7 @@ class TestObjectStaticSession(ClusterTestBase):
storage_object = storage_objects[0]
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
delete_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -632,7 +631,7 @@ class TestObjectStaticSession(ClusterTestBase):
storage_object = storage_objects[0]
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
put_object_to_random_node(
user_wallet,
user_wallet.path,
storage_object.file_path,
storage_object.cid,
self.shell,
@ -671,7 +670,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,

View file

@ -1,5 +1,5 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.acl import create_eacl, set_eacl, wait_for_cache_expired
@ -41,9 +41,9 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
Validate static session with create operation
"""
with reporter.step("Create container with static session token"):
with allure.step("Create container with static session token"):
cid = create_container(
user_wallet,
user_wallet.path,
session_token=static_sessions[ContainerVerb.CREATE],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -51,12 +51,14 @@ class TestSessionTokenContainer(ClusterTestBase):
)
container_info: dict[str, str] = get_container(
owner_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
owner_wallet.path, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert container_info["ownerID"] == owner_wallet.get_address()
assert cid not in list_containers(user_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid in list_containers(owner_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid not in list_containers(
user_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid in list_containers(owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
def test_static_session_token_container_create_with_other_verb(
self,
@ -66,11 +68,11 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
Validate static session without create operation
"""
with reporter.step("Try create container with static session token without PUT rule"):
with allure.step("Try create container with static session token without PUT rule"):
for verb in [verb for verb in ContainerVerb if verb != ContainerVerb.CREATE]:
with pytest.raises(Exception):
create_container(
user_wallet,
user_wallet.path,
session_token=static_sessions[verb],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -85,10 +87,10 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
Validate static session with create operation for other wallet
"""
with reporter.step("Try create container with static session token without PUT rule"):
with allure.step("Try create container with static session token without PUT rule"):
with pytest.raises(Exception):
create_container(
stranger_wallet,
stranger_wallet.path,
session_token=static_sessions[ContainerVerb.CREATE],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -104,16 +106,16 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
Validate static session with delete operation
"""
with reporter.step("Create container"):
with allure.step("Create container"):
cid = create_container(
owner_wallet,
owner_wallet.path,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
wait_for_creation=False,
)
with reporter.step("Delete container with static session token"):
with allure.step("Delete container with static session token"):
delete_container(
wallet=user_wallet,
wallet=user_wallet.path,
cid=cid,
session_token=static_sessions[ContainerVerb.DELETE],
shell=self.shell,
@ -121,7 +123,9 @@ class TestSessionTokenContainer(ClusterTestBase):
await_mode=True,
)
assert cid not in list_containers(owner_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid not in list_containers(
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
@pytest.mark.sanity
def test_static_session_token_container_set_eacl(
@ -135,20 +139,20 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
Validate static session with set eacl operation
"""
with reporter.step("Create container"):
with allure.step("Create container"):
cid = create_container(
owner_wallet,
owner_wallet.path,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
file_path = generate_file(simple_object_size.value)
assert can_put_object(stranger_wallet, cid, file_path, self.shell, self.cluster)
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
with reporter.step("Deny all operations for other via eACL"):
with allure.step("Deny all operations for other via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -157,4 +161,4 @@ class TestSessionTokenContainer(ClusterTestBase):
)
wait_for_cache_expired()
assert not can_put_object(stranger_wallet, cid, file_path, self.shell, self.cluster)
assert not can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)

View file

@ -1,85 +1,126 @@
import json
import pathlib
import re
from dataclasses import dataclass
from io import StringIO
import allure
import pytest
from frostfs_testlib import reporter
import yaml
from configobj import ConfigObj
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.resources.wellknown_acl import EACL_PUBLIC_READ_WRITE
from frostfs_testlib.steps.cli.container import create_container, delete_container
from frostfs_testlib.steps.cli.object import delete_object, get_object, get_object_nodes, put_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.shard import Shard
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel, wait_for_success
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.storage.cluster import Cluster, StorageNode
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
BLOBSTOR_PREFIX = "_BLOBSTOR_"
@dataclass
class Blobstor:
path: str
path_type: str
def __eq__(self, other) -> bool:
if not isinstance(other, self.__class__):
raise RuntimeError(f"Only two {self.__class__.__name__} instances can be compared")
return self.path == other.path and self.path_type == other.path_type
def __hash__(self):
return hash((self.path, self.path_type))
@staticmethod
def from_config_object(section: ConfigObj, shard_id: str, blobstor_id: str):
var_prefix = f"{SHARD_PREFIX}{shard_id}{BLOBSTOR_PREFIX}{blobstor_id}"
return Blobstor(section.get(f"{var_prefix}_PATH"), section.get(f"{var_prefix}_TYPE"))
@dataclass
class Shard:
blobstor: list[Blobstor]
metabase: str
writecache: str
def __eq__(self, other) -> bool:
if not isinstance(other, self.__class__):
raise RuntimeError(f"Only two {self.__class__.__name__} instances can be compared")
return (
set(self.blobstor) == set(other.blobstor)
and self.metabase == other.metabase
and self.writecache == other.writecache
)
def __hash__(self):
return hash((self.metabase, self.writecache))
@staticmethod
def _get_blobstor_count_from_section(config_object: ConfigObj, shard_id: int):
pattern = f"{SHARD_PREFIX}{shard_id}{BLOBSTOR_PREFIX}"
blobstors = {key[: len(pattern) + 2] for key in config_object.keys() if pattern in key}
return len(blobstors)
@staticmethod
def from_config_object(config_object: ConfigObj, shard_id: int):
var_prefix = f"{SHARD_PREFIX}{shard_id}"
blobstor_count = Shard._get_blobstor_count_from_section(config_object, shard_id)
blobstors = [
Blobstor.from_config_object(config_object, shard_id, blobstor_id) for blobstor_id in range(blobstor_count)
]
write_cache_enabled = config_object.as_bool(f"{var_prefix}_WRITECACHE_ENABLED")
return Shard(
blobstors,
config_object.get(f"{var_prefix}_METABASE_PATH"),
config_object.get(f"{var_prefix}_WRITECACHE_PATH") if write_cache_enabled else "",
)
@staticmethod
def from_object(shard):
metabase = shard["metabase"]["path"] if "path" in shard["metabase"] else shard["metabase"]
writecache = shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
return Shard(
blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]],
metabase=metabase,
writecache=writecache,
)
def shards_from_yaml(contents: str) -> list[Shard]:
config = yaml.safe_load(contents)
config["storage"]["shard"].pop("default")
return [Shard.from_object(shard) for shard in config["storage"]["shard"].values()]
def shards_from_env(contents: str) -> list[Shard]:
configObj = ConfigObj(StringIO(contents))
pattern = f"{SHARD_PREFIX}\d*"
num_shards = len(set(re.findall(pattern, contents)))
return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)]
@pytest.mark.shard
class TestControlShard(ClusterTestBase):
class TestControlShard:
@staticmethod
@wait_for_success(180, 30)
def get_object_path_and_name_file(oid: str, cid: str, node: ClusterNode) -> tuple[str, str]:
oid_path = f"{oid[0]}/{oid[1]}/{oid[2]}/{oid[3]}"
object_path = None
def get_shards_from_config(node: StorageNode) -> list[Shard]:
config_file = node.get_shard_config_path()
file_type = pathlib.Path(config_file).suffix
contents = node.host.get_shell().exec(f"cat {config_file}").stdout
with reporter.step("Search object file"):
node_shell = node.storage_node.host.get_shell()
data_path = node.storage_node.get_data_directory()
all_datas = node_shell.exec(f"ls -la {data_path}/data | awk '{{ print $9 }}'").stdout.strip()
for data_dir in all_datas.replace(".", "").strip().split("\n"):
check_dir = node_shell.exec(
f" [ -d {data_path}/data/{data_dir}/data/{oid_path} ] && echo 1 || echo 0"
).stdout
if "1" in check_dir:
object_path = f"{data_path}/data/{data_dir}/data/{oid_path}"
object_name = f"{oid[4:]}.{cid}"
break
parser_method = {
".env": shards_from_env,
".yaml": shards_from_yaml,
".yml": shards_from_yaml,
}
assert object_path is not None, f"{oid} object not found in directory - {data_path}/data"
return object_path, object_name
def set_shard_rw_mode(self, node: ClusterNode):
watcher = ShardsWatcher(node)
shards = watcher.get_shards()
for shard in shards:
watcher.set_shard_mode(shard["shard_id"], mode="read-write")
watcher.await_for_all_shards_status(status="read-write")
@pytest.fixture()
@allure.title("Revert all shards mode")
def revert_all_shards_mode(self) -> None:
yield
parallel(self.set_shard_rw_mode, self.cluster.cluster_nodes)
@pytest.fixture()
def oid_cid_node(self, default_wallet: WalletInfo, max_object_size: int) -> tuple[str, str, ClusterNode]:
with reporter.step("Create container, and put object"):
cid = create_container(
wallet=default_wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule="REP 1 CBF 1",
basic_acl=EACL_PUBLIC_READ_WRITE,
)
file = generate_file(round(max_object_size * 0.8))
oid = put_object(
wallet=default_wallet, path=file, cid=cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
with reporter.step("Search node with object"):
nodes = get_object_nodes(cluster=self.cluster, cid=cid, oid=oid, alive_node=self.cluster.cluster_nodes[0])
yield oid, cid, nodes[0]
object_path, object_name = self.get_object_path_and_name_file(oid, cid, nodes[0])
nodes[0].host.get_shell().exec(f"chmod +r {object_path}/{object_name}")
delete_object(
wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
shards = parser_method[file_type](contents)
return shards
@staticmethod
def get_shards_from_cli(node: StorageNode) -> list[Shard]:
@ -89,7 +130,7 @@ class TestControlShard(ClusterTestBase):
cli_config = node.host.get_cli_config("frostfs-cli")
cli = FrostfsCli(node.host.get_shell(), cli_config.exec_path)
cli = FrostfsCli(node.host.get_shell(), cli_config.exec_path, DEFAULT_WALLET_CONFIG)
result = cli.shards.list(
endpoint=control_endpoint,
wallet=wallet_path,
@ -99,50 +140,9 @@ class TestControlShard(ClusterTestBase):
)
return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]
@pytest.fixture()
def change_config_storage(self, cluster_state_controller: ClusterStateController):
with reporter.step("Change threshold error shards"):
cluster_state_controller.manager(ConfigStateManager).set_on_all_nodes(
service_type=StorageNode, values={"storage:shard_ro_error_threshold": "5"}
)
yield
with reporter.step("Restore threshold error shards"):
cluster_state_controller.manager(ConfigStateManager).revert_all()
@allure.title("All shards are available")
def test_control_shard(self, cluster: Cluster):
for storage_node in cluster.storage_nodes:
shards_from_config = storage_node.get_shards()
shards_from_config = self.get_shards_from_config(storage_node)
shards_from_cli = self.get_shards_from_cli(storage_node)
assert set(shards_from_config) == set(shards_from_cli)
@pytest.mark.failover
def test_shard_errors(
self,
default_wallet: WalletInfo,
oid_cid_node: tuple[str, str, ClusterNode],
change_config_storage: None,
revert_all_shards_mode: None,
):
oid, cid, node = oid_cid_node
with reporter.step("Search object in system."):
object_path, object_name = self.get_object_path_and_name_file(*oid_cid_node)
with reporter.step("Block read file"):
node.host.get_shell().exec(f"chmod a-r {object_path}/{object_name}")
with reporter.step("Get object, expect 6 errors"):
for _ in range(6):
with pytest.raises(RuntimeError):
get_object(
wallet=default_wallet,
cid=cid,
oid=oid,
shell=self.shell,
endpoint=node.storage_node.get_rpc_endpoint(),
)
with reporter.step("Check shard status"):
for shard in ShardsWatcher(node).get_shards():
if shard["blobstor"][1]["path"] in object_path:
with reporter.step(f"Shard - {shard['shard_id']} to {node.host_ip}, mode - {shard['mode']}"):
assert shard["mode"] == "degraded-read-only"
break

View file

@ -1,122 +0,0 @@
import os
import shutil
import time
from datetime import datetime, timezone
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.hosting import Host
from frostfs_testlib.testing.cluster_test_base import Cluster
from frostfs_testlib.testing.parallel import parallel
def pytest_generate_tests(metafunc: pytest.Metafunc):
metafunc.fixturenames.append("repo")
metafunc.fixturenames.append("markers")
metafunc.parametrize(
"repo, markers",
[("frostfs-testcases", metafunc.config.option.markexpr)],
)
@pytest.mark.logs_after_session
class TestLogs:
@pytest.mark.order(1000)
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - search errors")
def test_logs_search_errors(self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest):
end_time = datetime.now(timezone.utc)
logs_dir = os.path.join(temp_directory, "logs")
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b"
exclude_filter = r"too many requests"
log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors)
time.sleep(2)
futures = parallel(
self._collect_logs_on_host,
cluster.hosts,
logs_dir,
issues_regex,
session_start_time,
end_time,
exclude_filter,
priority=log_level_priority,
)
hosts_with_problems = [future.result() for future in futures if not future.exception() and future.result() is not None]
if hosts_with_problems:
self._attach_logs(logs_dir)
assert not hosts_with_problems, f"The following hosts contains critical errors in system logs: {', '.join(hosts_with_problems)}"
@pytest.mark.order(1001)
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - identify sensitive data")
def test_logs_identify_sensitive_data(
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
):
end_time = datetime.now(timezone.utc)
logs_dir = os.path.join(temp_directory, "logs")
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
_regex = {
"authorization_basic": r"basic [a-zA-Z0-9=:_\+\/-]{16,100}",
"authorization_bearer": r"bearer [a-zA-Z0-9_\-\.=:_\+\/]{16,100}",
"access_token": r"\"access_token\":\"[0-9a-z]{16}\$[0-9a-f]{32}\"",
"api_token": r"\"api_token\":\"(xox[a-zA-Z]-[a-zA-Z0-9-]+)\"",
"yadro_access_token": r"[a-zA-Z0-9_-]*:[a-zA-Z0-9_\-]+@yadro\.com*",
"SSH_privKey": r"([-]+BEGIN [^\s]+ PRIVATE KEY[-]+[\s]*[^-]*[-]+END [^\s]+ PRIVATE KEY[-]+)",
"possible_Creds": r"(?i)(" r"password\s*[`=:]+\s*[^\s]+|" r"password is\s*[`=:]+\s*[^\s]+|" r"passwd\s*[`=:]+\s*[^\s]+)",
}
issues_regex = "|".join(_regex.values())
exclude_filter = r"COMMAND=\|--\sBoot\s"
time.sleep(2)
futures = parallel(
self._collect_logs_on_host,
cluster.hosts,
logs_dir,
issues_regex,
session_start_time,
end_time,
exclude_filter,
)
hosts_with_problems = [future.result() for future in futures if not future.exception() and future.result() is not None]
if hosts_with_problems:
self._attach_logs(logs_dir)
assert not hosts_with_problems, f"The following hosts contains sensitive data in system logs: {', '.join(hosts_with_problems)}"
def _collect_logs_on_host(
self,
host: Host,
logs_dir: str,
regex: str,
since: datetime,
until: datetime,
exclude_filter: str,
priority: str = None,
):
with reporter.step(f"Get logs from {host.config.address}"):
logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter, priority=priority)
if not logs:
return None
with open(os.path.join(logs_dir, f"{host.config.address}.log"), "w") as file:
file.write(logs)
return host.config.address
def _attach_logs(self, logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
reporter.attach(logs_zip_file_path, "logs.zip")

View file

@ -0,0 +1,73 @@
import os
import shutil
import time
from datetime import datetime
import allure
import pytest
from frostfs_testlib.hosting import Host
from frostfs_testlib.testing.cluster_test_base import Cluster
from frostfs_testlib.testing.parallel import parallel
def pytest_generate_tests(metafunc: pytest.Metafunc):
metafunc.fixturenames.append("repo")
metafunc.fixturenames.append("markers")
metafunc.parametrize(
"repo, markers",
[("frostfs-testcases", metafunc.config.option.markexpr)],
)
class TestLogs:
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'")
@pytest.mark.logs_after_session
@pytest.mark.no_healthcheck
def test_logs_after_session(
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
):
"""
This test automatically added to any test run to check logs from cluster for critical errors.
"""
end_time = datetime.utcnow()
logs_dir = os.path.join(temp_directory, "logs")
os.makedirs(logs_dir)
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas"
exclude_filter = r"too many requests"
time.sleep(2)
futures = parallel(
self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time, exclude_filter
)
hosts_with_problems = [
future.result() for future in futures if not future.exception() and future.result() is not None
]
if hosts_with_problems:
self._attach_logs(logs_dir)
assert (
not hosts_with_problems
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime, exclude_filter: str):
with allure.step(f"Get logs from {host.config.address}"):
logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter)
if not logs:
return None
with open(os.path.join(logs_dir, f"{host.config.address}.log"), "w") as file:
file.write(logs)
return host.config.address
def _attach_logs(self, logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")