Compare commits

..

No commits in common. "e098f632511a613702f21bf1869ca473b01115bd" and "0d7befe9a6fcb6152f3f7d371a893b78d761913d" have entirely different histories.

36 changed files with 925 additions and 708 deletions

View file

@ -8,10 +8,10 @@ hosts:
skip_readiness_check: True
force_transactions: True
services:
- name: frostfs-storage_01
- name: s01
attributes:
container_name: s01
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet01.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet01.json
@ -20,10 +20,12 @@ hosts:
endpoint_data0: s01.frostfs.devenv:8080
control_endpoint: s01.frostfs.devenv:8081
un_locode: "RU MOW"
- name: frostfs-storage_02
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s02
attributes:
container_name: s02
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet02.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet02.json
@ -32,10 +34,12 @@ hosts:
endpoint_data0: s02.frostfs.devenv:8080
control_endpoint: s02.frostfs.devenv:8081
un_locode: "RU LED"
- name: frostfs-storage_03
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s03
attributes:
container_name: s03
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet03.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet03.json
@ -44,10 +48,12 @@ hosts:
endpoint_data0: s03.frostfs.devenv:8080
control_endpoint: s03.frostfs.devenv:8081
un_locode: "SE STO"
- name: frostfs-storage_04
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s04
attributes:
container_name: s04
config_path: /etc/frostfs/storage/config.yml
config_path: ../frostfs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet04.json
local_wallet_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet04.json
@ -56,7 +62,9 @@ hosts:
endpoint_data0: s04.frostfs.devenv:8080
control_endpoint: s04.frostfs.devenv:8081
un_locode: "FI HEL"
- name: frostfs-s3_01
http_hostname: ["no_hostname"]
s3_hostname: ["no_hostname"]
- name: s3-gate01
attributes:
container_name: s3_gate
config_path: ../frostfs-dev-env/services/s3_gate/.s3.env
@ -65,7 +73,7 @@ hosts:
local_wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json
wallet_password: "s3"
endpoint_data0: https://s3.frostfs.devenv:8080
- name: frostfs-http_01
- name: http-gate01
attributes:
container_name: http_gate
config_path: ../frostfs-dev-env/services/http_gate/.http.env
@ -74,7 +82,7 @@ hosts:
local_wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json
wallet_password: "one"
endpoint_data0: http://http.frostfs.devenv
- name: frostfs-ir_01
- name: ir01
attributes:
container_name: ir01
config_path: ../frostfs-dev-env/services/ir/.ir.env
@ -82,7 +90,7 @@ hosts:
local_wallet_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/ir/az.json
wallet_password: "one"
- name: neo-go_01
- name: morph-chain01
attributes:
container_name: morph_chain
config_path: ../frostfs-dev-env/services/morph_chain/protocol.privnet.yml
@ -91,7 +99,7 @@ hosts:
local_wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json
wallet_password: "one"
endpoint_internal0: http://morph-chain.frostfs.devenv:30333
- name: main-chain_01
- name: main-chain01
attributes:
container_name: main_chain
config_path: ../frostfs-dev-env/services/chain/protocol.privnet.yml
@ -100,7 +108,7 @@ hosts:
local_wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json
wallet_password: "one"
endpoint_internal0: http://main-chain.frostfs.devenv:30333
- name: coredns_01
- name: coredns01
attributes:
container_name: coredns
clis:

1
.gitignore vendored
View file

@ -18,7 +18,6 @@ xunit_results.xml
# ignore caches under any path
**/__pycache__
**/.pytest_cache
*.egg-info
# ignore work directories and setup files
.setup

View file

@ -3,7 +3,6 @@ from typing import List, Optional
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.acl import EACLOperation
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from pytest_tests.helpers.object_access import (
can_delete_object,
@ -17,47 +16,57 @@ from pytest_tests.helpers.object_access import (
def check_full_access_to_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
endpoint = cluster.default_rpc_endpoint
assert can_put_object(wallet, cid, file_name, shell, cluster, bearer, xhdr)
assert can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_get_range_hash_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_search_object(wallet, cid, shell, endpoint, oid, bearer, xhdr)
assert can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, xhdr)
assert can_delete_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert can_put_object(wallet, cid, file_name, shell, cluster, bearer, wallet_config, xhdr)
assert can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
assert can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
assert can_get_range_hash_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
assert can_search_object(wallet, cid, shell, endpoint, oid, bearer, wallet_config, xhdr)
assert can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, wallet_config, xhdr)
assert can_delete_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
def check_no_access_to_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
endpoint = cluster.default_rpc_endpoint
assert not can_put_object(wallet, cid, file_name, shell, cluster, bearer, xhdr)
assert not can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_get_range_hash_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_search_object(wallet, cid, shell, endpoint, oid, bearer, xhdr)
assert not can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, xhdr)
assert not can_delete_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
assert not can_put_object(wallet, cid, file_name, shell, cluster, bearer, wallet_config, xhdr)
assert not can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
assert not can_get_range_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
assert not can_get_range_hash_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
assert not can_search_object(wallet, cid, shell, endpoint, oid, bearer, wallet_config, xhdr)
assert not can_get_object(
wallet, cid, oid, file_name, shell, cluster, bearer, wallet_config, xhdr
)
assert not can_delete_object(wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr)
def check_custom_access_to_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
@ -66,6 +75,7 @@ def check_custom_access_to_container(
deny_operations: Optional[List[EACLOperation]] = None,
ignore_operations: Optional[List[EACLOperation]] = None,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
endpoint = cluster.default_rpc_endpoint
@ -73,39 +83,56 @@ def check_custom_access_to_container(
ignore_operations = [op.value for op in ignore_operations or []]
checks: dict = {}
if EACLOperation.PUT.value not in ignore_operations:
checks[EACLOperation.PUT.value] = can_put_object(wallet, cid, file_name, shell, cluster, bearer, xhdr)
checks[EACLOperation.PUT.value] = can_put_object(
wallet, cid, file_name, shell, cluster, bearer, wallet_config, xhdr
)
if EACLOperation.HEAD.value not in ignore_operations:
checks[EACLOperation.HEAD.value] = can_get_head_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
checks[EACLOperation.HEAD.value] = can_get_head_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
if EACLOperation.GET_RANGE.value not in ignore_operations:
checks[EACLOperation.GET_RANGE.value] = can_get_range_of_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
checks[EACLOperation.GET_RANGE.value] = can_get_range_of_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
if EACLOperation.GET_RANGE_HASH.value not in ignore_operations:
checks[EACLOperation.GET_RANGE_HASH.value] = can_get_range_hash_of_object(
wallet, cid, oid, shell, endpoint, bearer, xhdr
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
if EACLOperation.SEARCH.value not in ignore_operations:
checks[EACLOperation.SEARCH.value] = can_search_object(wallet, cid, shell, endpoint, oid, bearer, xhdr)
checks[EACLOperation.SEARCH.value] = can_search_object(
wallet, cid, shell, endpoint, oid, bearer, wallet_config, xhdr
)
if EACLOperation.GET.value not in ignore_operations:
checks[EACLOperation.GET.value] = can_get_object(wallet, cid, oid, file_name, shell, cluster, bearer, xhdr)
checks[EACLOperation.GET.value] = can_get_object(
wallet, cid, oid, file_name, shell, cluster, bearer, wallet_config, xhdr
)
if EACLOperation.DELETE.value not in ignore_operations:
checks[EACLOperation.DELETE.value] = can_delete_object(wallet, cid, oid, shell, endpoint, bearer, xhdr)
checks[EACLOperation.DELETE.value] = can_delete_object(
wallet, cid, oid, shell, endpoint, bearer, wallet_config, xhdr
)
failed_checks = [
f"allowed {action} failed"
for action, success in checks.items()
if not success and action not in deny_operations
] + [f"denied {action} succeeded" for action, success in checks.items() if success and action in deny_operations]
] + [
f"denied {action} succeeded"
for action, success in checks.items()
if success and action in deny_operations
]
assert not failed_checks, ", ".join(failed_checks)
def check_read_only_container(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
):
return check_custom_access_to_container(
@ -115,6 +142,7 @@ def check_read_only_container(
file_name,
deny_operations=[EACLOperation.PUT, EACLOperation.DELETE],
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
cluster=cluster,

View file

@ -14,7 +14,6 @@ from frostfs_testlib.steps.cli.object import (
search_object,
)
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import string_utils
from frostfs_testlib.utils.file_utils import get_file_hash
@ -22,13 +21,14 @@ OPERATION_ERROR_TYPE = RuntimeError
def can_get_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
) -> bool:
with reporter.step("Try get object from container"):
@ -38,6 +38,7 @@ def can_get_object(
cid,
oid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
cluster=cluster,
@ -52,12 +53,13 @@ def can_get_object(
def can_put_object(
wallet: WalletInfo,
wallet: str,
cid: str,
file_name: str,
shell: Shell,
cluster: Cluster,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
attributes: Optional[dict] = None,
) -> bool:
@ -68,6 +70,7 @@ def can_put_object(
file_name,
cid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
attributes=attributes,
shell=shell,
@ -82,12 +85,13 @@ def can_put_object(
def can_delete_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
) -> bool:
with reporter.step("Try delete object from container"):
@ -97,6 +101,7 @@ def can_delete_object(
cid,
oid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -110,12 +115,13 @@ def can_delete_object(
def can_get_head_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
@ -126,6 +132,7 @@ def can_get_head_object(
cid,
oid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -140,12 +147,13 @@ def can_get_head_object(
def can_get_range_of_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
@ -157,6 +165,7 @@ def can_get_range_of_object(
oid,
bearer=bearer,
range_cut="0:10",
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -171,12 +180,13 @@ def can_get_range_of_object(
def can_get_range_hash_of_object(
wallet: WalletInfo,
wallet: str,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
@ -188,6 +198,7 @@ def can_get_range_hash_of_object(
oid,
bearer=bearer,
range_cut="0:10",
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,
@ -202,12 +213,13 @@ def can_get_range_hash_of_object(
def can_search_object(
wallet: WalletInfo,
wallet: str,
cid: str,
shell: Shell,
endpoint: str,
oid: Optional[str] = None,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool:
@ -217,6 +229,7 @@ def can_search_object(
wallet,
cid,
bearer=bearer,
wallet_config=wallet_config,
xhdr=xhdr,
shell=shell,
endpoint=endpoint,

View file

@ -1,10 +1,11 @@
import os
import uuid
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
@ -13,40 +14,54 @@ from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.acl import EACLRole
from frostfs_testlib.storage.dataclasses.frostfs_services import InnerRing, StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import generate_file
OBJECT_COUNT = 5
@dataclass
class Wallets:
wallets: dict[EACLRole, list[WalletInfo]]
class Wallet:
wallet_path: Optional[str] = None
config_path: Optional[str] = None
def get_wallet(self, role: EACLRole = EACLRole.USER) -> WalletInfo:
@dataclass
class Wallets:
wallets: dict[EACLRole, list[Wallet]]
def get_wallet(self, role: EACLRole = EACLRole.USER) -> Wallet:
return self.wallets[role][0]
def get_wallets_list(self, role: EACLRole = EACLRole.USER) -> list[WalletInfo]:
def get_wallets_list(self, role: EACLRole = EACLRole.USER) -> list[Wallet]:
return self.wallets[role]
@pytest.fixture(scope="module")
def wallets(default_wallet: WalletInfo, credentials_provider: CredentialsProvider, cluster: Cluster) -> Wallets:
other_wallets: list = []
for _ in range(2):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
other_wallets.append(credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0]))
def wallets(default_wallet: str, temp_directory: str, cluster: Cluster) -> Wallets:
other_wallets_paths = [os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2)]
for other_wallet_path in other_wallets_paths:
wallet_utils.init_wallet(other_wallet_path, DEFAULT_WALLET_PASS)
ir_node: InnerRing = cluster.ir_nodes[0]
storage_node: StorageNode = cluster.storage_nodes[0]
ir_wallet_path = ir_node.get_wallet_path()
ir_wallet_config = ir_node.get_wallet_config_path()
storage_wallet_path = storage_node.get_wallet_path()
storage_wallet_config = storage_node.get_wallet_config_path()
wallets_collection = Wallets(
wallets={
EACLRole.USER: [default_wallet],
EACLRole.OTHERS: other_wallets,
EACLRole.USER: [Wallet(wallet_path=default_wallet, config_path=DEFAULT_WALLET_CONFIG)],
EACLRole.OTHERS: [
Wallet(wallet_path=other_wallet_path, config_path=DEFAULT_WALLET_CONFIG)
for other_wallet_path in other_wallets_paths
],
EACLRole.SYSTEM: [
WalletInfo.from_node(ir_node),
WalletInfo.from_node(storage_node),
Wallet(wallet_path=ir_wallet_path, config_path=ir_wallet_config),
Wallet(wallet_path=storage_wallet_path, config_path=storage_wallet_config),
],
}
)
@ -55,7 +70,7 @@ def wallets(default_wallet: WalletInfo, credentials_provider: CredentialsProvide
if role == EACLRole.SYSTEM:
continue
for wallet in wallets:
reporter.attach(wallet.path, os.path.basename(wallet.path))
reporter.attach(wallet.wallet_path, os.path.basename(wallet.wallet_path))
return wallets_collection
@ -72,7 +87,7 @@ def eacl_container_with_objects(
user_wallet = wallets.get_wallet()
with reporter.step("Create eACL public container"):
cid = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PUBLIC_ACL,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
@ -81,7 +96,7 @@ def eacl_container_with_objects(
with reporter.step("Add test objects to container"):
objects_oids = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
attributes={"key1": "val1", "key": val, "key2": "abc"},

View file

@ -22,11 +22,11 @@ from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.acl_basic
class TestACLBasic(ClusterTestBase):
@pytest.fixture(scope="function")
def public_container(self, wallets: Wallets):
def public_container(self, wallets):
user_wallet = wallets.get_wallet()
with reporter.step("Create public container"):
cid_public = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PUBLIC_ACL_F,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -35,14 +35,14 @@ class TestACLBasic(ClusterTestBase):
yield cid_public
# with reporter.step('Delete public container'):
# delete_container(user_wallet, cid_public)
# delete_container(user_wallet.wallet_path, cid_public)
@pytest.fixture(scope="function")
def private_container(self, wallets: Wallets):
user_wallet = wallets.get_wallet()
with reporter.step("Create private container"):
cid_private = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PRIVATE_ACL_F,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -51,14 +51,14 @@ class TestACLBasic(ClusterTestBase):
yield cid_private
# with reporter.step('Delete private container'):
# delete_container(user_wallet, cid_private)
# delete_container(user_wallet.wallet_path, cid_private)
@pytest.fixture(scope="function")
def read_only_container(self, wallets: Wallets):
user_wallet = wallets.get_wallet()
with reporter.step("Create public readonly container"):
cid_read_only = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=READONLY_ACL_F,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -67,7 +67,7 @@ class TestACLBasic(ClusterTestBase):
yield cid_read_only
# with reporter.step('Delete public readonly container'):
# delete_container(user_wallet, cid_read_only)
# delete_container(user_wallet.wallet_path, cid_read_only)
@allure.title("Operations with basic ACL on public container (obj_size={object_size})")
def test_basic_acl_public(self, wallets: Wallets, public_container: str, file_path: str):
@ -82,7 +82,7 @@ class TestACLBasic(ClusterTestBase):
# We create new objects for each wallet because check_full_access_to_container
# deletes the object
owner_object_oid = put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -90,7 +90,7 @@ class TestACLBasic(ClusterTestBase):
attributes={"created": "owner"},
)
other_object_oid = put_object_to_random_node(
other_wallet,
other_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -99,7 +99,7 @@ class TestACLBasic(ClusterTestBase):
)
with reporter.step(f"Check {desc} has full access to public container"):
check_full_access_to_container(
wallet,
wallet.wallet_path,
cid,
owner_object_oid,
file_path,
@ -107,7 +107,7 @@ class TestACLBasic(ClusterTestBase):
cluster=self.cluster,
)
check_full_access_to_container(
wallet,
wallet.wallet_path,
cid,
other_object_oid,
file_path,
@ -125,13 +125,13 @@ class TestACLBasic(ClusterTestBase):
cid = private_container
with reporter.step("Add test objects to container"):
owner_object_oid = put_object_to_random_node(
user_wallet, file_path, cid, shell=self.shell, cluster=self.cluster
user_wallet.wallet_path, file_path, cid, shell=self.shell, cluster=self.cluster
)
with reporter.step("Check only owner has full access to private container"):
with reporter.step("Check no one except owner has access to operations with container"):
check_no_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
owner_object_oid,
file_path,
@ -141,7 +141,7 @@ class TestACLBasic(ClusterTestBase):
with reporter.step("Check owner has full access to private container"):
check_full_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
owner_object_oid,
file_path,
@ -160,12 +160,12 @@ class TestACLBasic(ClusterTestBase):
with reporter.step("Add test objects to container"):
object_oid = put_object_to_random_node(
user_wallet, file_path, cid, shell=client_shell, cluster=self.cluster
user_wallet.wallet_path, file_path, cid, shell=client_shell, cluster=self.cluster
)
with reporter.step("Check other has read-only access to operations with container"):
check_read_only_container(
other_wallet,
other_wallet.wallet_path,
cid,
object_oid,
file_path,
@ -175,7 +175,7 @@ class TestACLBasic(ClusterTestBase):
with reporter.step("Check owner has full access to public container"):
check_full_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
object_oid,
file_path,

View file

@ -32,10 +32,11 @@ class TestACLBearer(ClusterTestBase):
with reporter.step(f"Check {role.value} has full access to container without bearer token"):
check_full_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
@ -43,12 +44,12 @@ class TestACLBearer(ClusterTestBase):
with reporter.step(f"Set deny all operations for {role.value} via eACL"):
eacl = [EACLRule(access=EACLAccess.DENY, role=role, operation=op) for op in EACLOperation]
eacl_file = create_eacl(cid, eacl, shell=self.shell)
set_eacl(user_wallet, cid, eacl_file, shell=self.shell, endpoint=endpoint)
set_eacl(user_wallet.wallet_path, cid, eacl_file, shell=self.shell, endpoint=endpoint)
wait_for_cache_expired()
with reporter.step(f"Create bearer token for {role.value} with all operations allowed"):
bearer = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=role) for op in EACLOperation],
shell=self.shell,
@ -57,21 +58,23 @@ class TestACLBearer(ClusterTestBase):
with reporter.step(f"Check {role.value} without token has no access to all operations with container"):
check_no_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step(f"Check {role.value} with token has access to all operations with container"):
check_full_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
bearer=bearer,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
@ -79,21 +82,22 @@ class TestACLBearer(ClusterTestBase):
with reporter.step(f"Set allow all operations for {role.value} via eACL"):
eacl = [EACLRule(access=EACLAccess.ALLOW, role=role, operation=op) for op in EACLOperation]
eacl_file = create_eacl(cid, eacl, shell=self.shell)
set_eacl(user_wallet, cid, eacl_file, shell=self.shell, endpoint=endpoint)
set_eacl(user_wallet.wallet_path, cid, eacl_file, shell=self.shell, endpoint=endpoint)
wait_for_cache_expired()
with reporter.step(f"Check {role.value} without token has access to all operations with container"):
check_full_access_to_container(
deny_wallet,
deny_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
wallet_config=deny_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
@allure.title("BearerToken for compound operations (obj_size={object_size})")
def test_bearer_token_compound_operations(self, wallets: Wallets, eacl_container_with_objects):
def test_bearer_token_compound_operations(self, wallets, eacl_container_with_objects):
endpoint = self.cluster.default_rpc_endpoint
cid, objects_oids, file_path = eacl_container_with_objects
user_wallet = wallets.get_wallet()
@ -132,7 +136,7 @@ class TestACLBearer(ClusterTestBase):
for role, operations in deny_map.items():
eacl_deny += [EACLRule(access=EACLAccess.DENY, role=role, operation=op) for op in operations]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
eacl_table_path=create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -142,27 +146,29 @@ class TestACLBearer(ClusterTestBase):
with reporter.step("Check rule consistency without bearer"):
check_custom_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map[EACLRole.USER],
wallet_config=user_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
check_custom_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map[EACLRole.OTHERS],
wallet_config=other_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
with reporter.step("Check rule consistency using bearer token"):
bearer_user = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.USER)
@ -173,7 +179,7 @@ class TestACLBearer(ClusterTestBase):
)
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
@ -184,22 +190,24 @@ class TestACLBearer(ClusterTestBase):
)
check_custom_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map_with_bearer[EACLRole.USER],
bearer=bearer_user,
wallet_config=user_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)
check_custom_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
objects_oids.pop(),
file_path,
deny_operations=deny_map_with_bearer[EACLRole.OTHERS],
bearer=bearer_other,
wallet_config=other_wallet.config_path,
shell=self.shell,
cluster=self.cluster,
)

View file

@ -27,14 +27,14 @@ from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.acl_extended
class TestEACLContainer(ClusterTestBase):
@pytest.fixture(scope="function")
def eacl_full_placement_container_with_object(self, wallets: Wallets, file_path: str) -> tuple[str, str, str]:
def eacl_full_placement_container_with_object(self, wallets: Wallets, file_path: str) -> str:
user_wallet = wallets.get_wallet()
storage_nodes = self.cluster.storage_nodes
node_count = len(storage_nodes)
with reporter.step("Create eACL public container with full placement rule"):
full_placement_rule = f"REP {node_count} IN X CBF 1 SELECT {node_count} FROM * AS X"
cid = create_container(
wallet=user_wallet,
wallet=user_wallet.wallet_path,
rule=full_placement_rule,
basic_acl=PUBLIC_ACL,
shell=self.shell,
@ -42,7 +42,9 @@ class TestEACLContainer(ClusterTestBase):
)
with reporter.step("Add test object to container"):
oid = put_object_to_random_node(user_wallet, file_path, cid, shell=self.shell, cluster=self.cluster)
oid = put_object_to_random_node(
user_wallet.wallet_path, file_path, cid, shell=self.shell, cluster=self.cluster
)
wait_object_replication(
cid,
oid,
@ -73,7 +75,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step(f"Deny all operations for {deny_role_str} via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -84,7 +86,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step(f"Check only {not_deny_role_str} has full access to container"):
with reporter.step(f"Check {deny_role_str} has not access to any operations with container"):
check_no_access_to_container(
deny_role_wallet,
deny_role_wallet.wallet_path,
cid,
object_oids[0],
file_path,
@ -94,7 +96,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step(f"Check {not_deny_role_wallet} has full access to eACL public container"):
check_full_access_to_container(
not_deny_role_wallet,
not_deny_role_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -105,7 +107,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step(f"Allow all operations for {deny_role_str} via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -115,7 +117,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check all have full access to eACL public container"):
check_full_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -123,7 +125,7 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -143,14 +145,14 @@ class TestEACLContainer(ClusterTestBase):
eacl = [
EACLRule(
access=EACLAccess.ALLOW,
role=other_wallet_allow,
role=other_wallet_allow.wallet_path,
operation=op,
)
for op in EACLOperation
]
eacl += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl, shell=self.shell),
shell=self.shell,
@ -161,7 +163,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check only owner and allowed other have full access to public container"):
with reporter.step("Check other has not access to operations with container"):
check_no_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
object_oids[0],
file_path,
@ -171,7 +173,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check owner has full access to public container"):
check_full_access_to_container(
user_wallet,
user_wallet.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -181,7 +183,7 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check allowed other has full access to public container"):
check_full_access_to_container(
other_wallet_allow,
other_wallet_allow.wallet_path,
cid,
object_oids.pop(),
file_path,
@ -204,7 +206,7 @@ class TestEACLContainer(ClusterTestBase):
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op) for op in EACLOperation]
eacl_deny += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -235,120 +237,134 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check IR and STORAGE rules compliance"):
assert not can_put_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_put_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
object_oids[0],
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_get_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
object_oids[0],
file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_head_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_head_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_search_object(
ir_wallet,
ir_wallet.wallet_path,
cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=ir_wallet.config_path,
)
assert can_search_object(
storage_wallet,
storage_wallet.wallet_path,
cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with reporter.step("Deny all operations for SYSTEM via eACL"):
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(
cid=cid,
@ -364,127 +380,141 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check IR and STORAGE rules compliance with deny eACL"):
assert not can_put_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert not can_put_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_head_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_head_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_search_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_search_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
shell=self.shell,
endpoint=endpoint,
oid=object_oids[0],
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_hash_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_hash_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with reporter.step("Allow all operations for SYSTEM via eACL"):
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(
cid=cid,
@ -500,113 +530,127 @@ class TestEACLContainer(ClusterTestBase):
with reporter.step("Check IR and STORAGE rules compliance with allow eACL"):
assert not can_put_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_put_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=ir_wallet.config_path,
)
assert can_get_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
file_name=file_path,
shell=self.shell,
cluster=self.cluster,
wallet_config=storage_wallet.config_path,
)
assert can_get_head_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_head_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_search_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
shell=self.shell,
oid=object_oids[0],
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_search_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
shell=self.shell,
oid=object_oids[0],
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_get_range_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
assert can_get_range_hash_of_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=ir_wallet,
wallet=ir_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=ir_wallet.config_path,
)
with pytest.raises(AssertionError):
assert can_delete_object(
wallet=storage_wallet,
wallet=storage_wallet.wallet_path,
cid=cid,
oid=object_oids[0],
shell=self.shell,
endpoint=endpoint,
wallet_config=storage_wallet.config_path,
)

View file

@ -64,7 +64,7 @@ class TestEACLFilters(ClusterTestBase):
user_wallet = wallets.get_wallet()
with reporter.step("Create eACL public container"):
cid = create_container(
user_wallet,
user_wallet.wallet_path,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -73,7 +73,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Add test objects to container"):
objects_with_header = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -85,7 +85,7 @@ class TestEACLFilters(ClusterTestBase):
objects_with_other_header = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -97,7 +97,7 @@ class TestEACLFilters(ClusterTestBase):
objects_without_header = [
put_object_to_random_node(
user_wallet,
user_wallet.wallet_path,
file_path,
cid,
shell=self.shell,
@ -110,7 +110,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Delete eACL public container"):
delete_container(
user_wallet,
user_wallet.wallet_path,
cid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -148,7 +148,7 @@ class TestEACLFilters(ClusterTestBase):
for op in EACLOperation
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -171,7 +171,7 @@ class TestEACLFilters(ClusterTestBase):
):
with reporter.step("Check other has full access when sending request without headers"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -181,7 +181,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other has full access when sending request with allowed headers"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -192,7 +192,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other has no access when sending request with denied headers"):
check_no_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -205,14 +205,14 @@ class TestEACLFilters(ClusterTestBase):
"Check other has full access when sending request " "with denied headers and using bearer token"
):
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
oid.pop(),
file_path,
@ -253,7 +253,7 @@ class TestEACLFilters(ClusterTestBase):
for op in self.OBJECT_ATTRIBUTES_FILTER_SUPPORTED_OPERATIONS
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -270,7 +270,7 @@ class TestEACLFilters(ClusterTestBase):
for xhdr in (self.ATTRIBUTE, self.OTHER_ATTRIBUTE, None):
with reporter.step("Check other have full access to objects without attributes"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
objs_without_header.pop(),
file_path,
@ -281,7 +281,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other have full access to objects without deny attribute"):
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
allow_objects.pop(),
file_path,
@ -293,7 +293,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other have no access to objects with deny attribute"):
with pytest.raises(AssertionError):
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
shell=self.shell,
@ -302,7 +302,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
file_path,
@ -313,7 +313,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other have access to objects with deny attribute and using bearer token"):
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(
@ -327,7 +327,7 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
check_full_access_to_container(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects.pop(),
file_path,
@ -340,20 +340,20 @@ class TestEACLFilters(ClusterTestBase):
allow_attribute = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
with reporter.step("Check other can PUT objects without denied attribute"):
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
cluster=self.cluster,
attributes=allow_attribute,
)
assert can_put_object(other_wallet, cid, file_path, shell=self.shell, cluster=self.cluster)
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
deny_attribute = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
with reporter.step("Check other can not PUT objects with denied attribute"):
with pytest.raises(AssertionError):
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -363,7 +363,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other can PUT objects with denied attribute and using bearer token"):
bearer_other_for_put = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(
@ -376,7 +376,7 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -419,7 +419,7 @@ class TestEACLFilters(ClusterTestBase):
for op in self.OBJECT_ATTRIBUTES_FILTER_SUPPORTED_OPERATIONS
]
set_eacl(
user_wallet,
user_wallet.wallet_path,
cid,
create_eacl(cid, eacl, shell=self.shell),
shell=self.shell,
@ -442,7 +442,7 @@ class TestEACLFilters(ClusterTestBase):
oid = objects_without_header.pop()
with pytest.raises(AssertionError):
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
shell=self.shell,
@ -450,7 +450,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
file_path,
@ -458,11 +458,11 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
with pytest.raises(AssertionError):
assert can_put_object(other_wallet, cid, file_path, shell=self.shell, cluster=self.cluster)
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
with reporter.step("Check other can get and put objects without attributes and using bearer token"):
bearer_other = form_bearertoken_file(
user_wallet,
user_wallet.wallet_path,
cid,
[
EACLRule(
@ -476,7 +476,7 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
objects_without_header[0],
shell=self.shell,
@ -484,7 +484,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
objects_without_header[0],
file_path,
@ -493,7 +493,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -504,14 +504,14 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other can get objects with attributes matching the filter"):
oid = allow_objects.pop()
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
file_path,
@ -519,7 +519,7 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,
@ -530,7 +530,7 @@ class TestEACLFilters(ClusterTestBase):
with reporter.step("Check other cannot get objects without attributes matching the filter"):
with pytest.raises(AssertionError):
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
shell=self.shell,
@ -538,7 +538,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
deny_objects[0],
file_path,
@ -547,7 +547,7 @@ class TestEACLFilters(ClusterTestBase):
)
with pytest.raises(AssertionError):
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
attributes=deny_attribute,
@ -560,7 +560,7 @@ class TestEACLFilters(ClusterTestBase):
):
oid = deny_objects.pop()
assert can_get_head_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
shell=self.shell,
@ -568,7 +568,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_get_object(
other_wallet,
other_wallet.wallet_path,
cid,
oid,
file_path,
@ -577,7 +577,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
assert can_put_object(
other_wallet,
other_wallet.wallet_path,
cid,
file_path,
shell=self.shell,

View file

@ -11,7 +11,6 @@ import pytest
import yaml
from dateutil import parser
from frostfs_testlib import plugins, reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting import Hosting
from frostfs_testlib.reporter import AllureHandler, StepsLogger
@ -19,11 +18,12 @@ from frostfs_testlib.resources.common import (
ASSETS_DIR,
COMPLEX_OBJECT_CHUNKS_COUNT,
COMPLEX_OBJECT_TAIL_SIZE,
DEFAULT_WALLET_PASS,
SIMPLE_OBJECT_SIZE,
)
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.steps.cli.container import DEFAULT_PLACEMENT_RULE, DEFAULT_EC_PLACEMENT_RULE
from frostfs_testlib.steps.cli.container import list_containers
from frostfs_testlib.steps.cli.object import get_netmap_netinfo
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage import get_service_registry
@ -31,8 +31,7 @@ from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import wait_for_success
@ -47,7 +46,7 @@ SERVICE_ACTIVE_TIME = 20
# Add logs check test even if it's not fit to mark selectors
def pytest_configure(config: pytest.Config):
markers = config.option.markexpr
if markers != "" and "sanity" not in markers:
if markers != "":
config.option.markexpr = f"logs_after_session or ({markers})"
@ -175,9 +174,9 @@ def require_multiple_interfaces(cluster: Cluster):
@pytest.fixture(scope="session")
def max_object_size(cluster: Cluster, client_shell: Shell) -> int:
storage_node = cluster.storage_nodes[0]
wallet = WalletInfo.from_node(storage_node)
net_info = get_netmap_netinfo(
wallet=wallet,
wallet=storage_node.get_wallet_path(),
wallet_config=storage_node.get_wallet_config_path(),
endpoint=storage_node.get_rpc_endpoint(),
shell=client_shell,
)
@ -210,30 +209,12 @@ def object_size(
return complex_object_size
@pytest.fixture(scope="session")
def rep_placement_policy() -> PlacementPolicy:
return PlacementPolicy("rep", DEFAULT_PLACEMENT_RULE)
@pytest.fixture(scope="session")
def ec_placement_policy() -> PlacementPolicy:
return PlacementPolicy("ec", DEFAULT_EC_PLACEMENT_RULE)
def wallet_factory(temp_directory: str, client_shell: Shell, cluster: Cluster) -> WalletFactory:
return WalletFactory(temp_directory, client_shell, cluster)
# By default we want all tests to be executed with both storage policies.
# This can be overriden in choosen tests if needed.
@pytest.fixture(
scope="session",
params=[pytest.param("rep", marks=pytest.mark.rep), pytest.param("ec", marks=pytest.mark.ec)],
)
def placement_policy(
rep_placement_policy: PlacementPolicy, ec_placement_policy: PlacementPolicy, request: pytest.FixtureRequest
) -> PlacementPolicy:
if request.param == "rep":
return rep_placement_policy
return ec_placement_policy
@pytest.fixture(scope="session")
def cluster(temp_directory: str, hosting: Hosting, client_shell: Shell) -> Cluster:
cluster = Cluster(hosting)
@ -272,11 +253,6 @@ def cluster_state_controller(client_shell: Shell, cluster: Cluster, healthcheck:
yield controller
@pytest.fixture(scope="session")
def credentials_provider(cluster: Cluster) -> CredentialsProvider:
return CredentialsProvider(cluster)
@reporter.step("[Class]: Create S3 client")
@pytest.fixture(
scope="class",
@ -286,20 +262,29 @@ def credentials_provider(cluster: Cluster) -> CredentialsProvider:
],
)
def s3_client(
default_user: User,
default_wallet: str,
client_shell: Shell,
s3_policy: Optional[str],
cluster: Cluster,
auth_container_placement_policy: str,
request: pytest.FixtureRequest,
credentials_provider: CredentialsProvider,
) -> S3ClientWrapper:
node = cluster.cluster_nodes[0]
credentials_provider.S3.provide(default_user, node, s3_policy)
wallet = WalletInfo(path=default_wallet, password=DEFAULT_WALLET_PASS)
(cid, access_key_id, secret_access_key) = s3_helper.init_s3_credentials(
wallet,
client_shell,
cluster,
s3gates=[cluster_node.s3_gate for cluster_node in cluster.cluster_nodes],
policy=s3_policy,
container_placement_policy=auth_container_placement_policy,
)
containers_list = list_containers(wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
s3_client_cls = request.param
client = s3_client_cls(
default_user.s3_credentials.access_key, default_user.s3_credentials.secret_key, cluster.default_s3_gate_endpoint
)
return client
client = s3_client_cls(access_key_id, secret_access_key, cluster.default_s3_gate_endpoint)
yield client
@pytest.fixture
@ -337,11 +322,11 @@ def two_buckets(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
@allure.title("[Autouse/Session] Check binary versions")
@reporter.step("[Autouse/Session] Check binary versions")
@pytest.fixture(scope="session", autouse=True)
def check_binary_versions(hosting: Hosting, client_shell: Shell, request: pytest.FixtureRequest):
local_versions = version_utils.get_local_binaries_versions(client_shell)
remote_versions, exсeptions_remote_binaries_versions = version_utils.get_remote_binaries_versions(hosting)
remote_versions = version_utils.get_remote_binaries_versions(hosting)
all_versions = {
**local_versions,
@ -423,22 +408,27 @@ def run_health_check(healthcheck: Healthcheck, cluster: Cluster, request: pytest
parallel(healthcheck.storage_healthcheck, cluster.cluster_nodes)
@reporter.step("Prepare default user with wallet")
@reporter.step("Prepare wallet and deposit")
@pytest.fixture(scope="session")
def default_user(credentials_provider: CredentialsProvider, cluster: Cluster) -> User:
# always unique username
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
node = cluster.cluster_nodes[0]
credentials_provider.GRPC.provide(user, node)
return user
def default_wallet(wallet_factory: WalletFactory) -> str:
wallet = wallet_factory.create_wallet(password=DEFAULT_WALLET_PASS)
reporter.attach(wallet.path, os.path.basename(wallet.path))
return wallet.path
@reporter.step("Get wallet for default user")
@pytest.fixture(scope="session")
def default_wallet(default_user: User) -> WalletInfo:
return default_user.wallet
@reporter.step("[Class]: Container placement policy for keys")
@pytest.fixture(scope="class")
def auth_container_placement_policy(cluster: Cluster, request: pytest.FixtureRequest):
placeholders = {
"$ALPHABET_NODE_COUNT$": 4 if len(cluster.cluster_nodes) < 8 else 8,
"$NODE_COUNT$": len(cluster.cluster_nodes),
}
placement_policy = None
if "param" in request.__dict__:
placement_policy = request.param
for key, value in placeholders.items():
placement_policy = placement_policy.replace(key, str(value))
return placement_policy
@pytest.fixture()

View file

@ -1,3 +1,5 @@
import json
import allure
import pytest
from frostfs_testlib import reporter
@ -10,7 +12,6 @@ from frostfs_testlib.steps.cli.container import (
wait_for_container_creation,
wait_for_container_deletion,
)
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.utility import placement_policy_from_container
@ -21,11 +22,13 @@ from pytest_tests.helpers.utility import placement_policy_from_container
class TestContainer(ClusterTestBase):
@pytest.mark.parametrize("name", ["", "test-container"], ids=["No name", "Set particular name"])
@pytest.mark.smoke
def test_container_creation(self, default_wallet: WalletInfo, name: str):
def test_container_creation(self, default_wallet: str, name: str):
scenario_title = "with name" if name else "without name"
allure.dynamic.title(f"Create container {scenario_title}")
wallet = default_wallet
with open(wallet) as file:
json_wallet = json.load(file)
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
cid = create_container(
@ -50,8 +53,8 @@ class TestContainer(ClusterTestBase):
info_to_check = {
f"basic ACL: {PRIVATE_ACL_F} (private)",
f"owner ID: {wallet.get_address_from_json(0)}",
f"CID: {cid}",
f"owner ID: {json_wallet.get('accounts')[0].get('address')}",
f"container ID: {cid}",
}
if name:
info_to_check.add(f"Name={name}")
@ -79,7 +82,7 @@ class TestContainer(ClusterTestBase):
wait_for_container_deletion(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
@allure.title("Parallel container creation and deletion")
def test_container_creation_deletion_parallel(self, default_wallet: WalletInfo):
def test_container_creation_deletion_parallel(self, default_wallet: str):
containers_count = 3
wallet = default_wallet
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
@ -112,10 +115,6 @@ class TestContainer(ClusterTestBase):
with reporter.step("Delete containers and check they were deleted"):
for cid in cids:
delete_container(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, await_mode=True
)
containers_list = list_containers(
wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid not in containers_list, "Container not deleted"
delete_container(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, await_mode=True)
containers_list = list_containers(wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid not in containers_list, "Container not deleted"

View file

@ -385,8 +385,8 @@ class TestPolicy(ClusterTestBase):
placement_params["country"] == netmap[node_address]["country"]
), f"The node is selected from the wrong country. Got {netmap[node_address]['country']}"
with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
with reporter.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
with reporter.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)

View file

@ -21,7 +21,6 @@ from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.utils.failover_utils import wait_object_replication
@ -75,7 +74,7 @@ class TestFailoverNetwork(ClusterTestBase):
def storage_objects(
self,
simple_object_size: ObjectSize,
default_wallet: WalletInfo,
default_wallet: str,
) -> list[StorageObjectInfo]:
file_path = generate_file(simple_object_size.value)
@ -106,7 +105,7 @@ class TestFailoverNetwork(ClusterTestBase):
storage_object = StorageObjectInfo(cid=cid, oid=oid)
storage_object.size = simple_object_size.value
storage_object.wallet = default_wallet
storage_object.wallet_file_path = default_wallet
storage_object.file_path = file_path
storage_object.file_hash = file_hash
storage_object.attributes = attribute
@ -120,7 +119,7 @@ class TestFailoverNetwork(ClusterTestBase):
@allure.title("Block Storage node traffic")
def test_block_storage_node_traffic(
self,
default_wallet: WalletInfo,
default_wallet: str,
require_multiple_hosts,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
@ -202,7 +201,7 @@ class TestFailoverNetwork(ClusterTestBase):
def test_block_data_interface(
self,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
restore_down_interfaces: None,
delete_file_after_test: None,
storage_objects: list[StorageObjectInfo],
@ -259,7 +258,7 @@ class TestFailoverNetwork(ClusterTestBase):
def test_block_internal_interface(
self,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
restore_down_interfaces: None,
delete_file_after_test: None,
storage_objects: list[StorageObjectInfo],
@ -343,7 +342,7 @@ class TestFailoverNetwork(ClusterTestBase):
self,
require_multiple_interfaces,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
simple_object_size: ObjectSize,
delete_file_after_test: None,
restore_down_interfaces: None,
@ -399,7 +398,7 @@ class TestFailoverNetwork(ClusterTestBase):
self,
require_multiple_interfaces,
cluster_state_controller: ClusterStateController,
default_wallet: WalletInfo,
default_wallet: str,
simple_object_size: ObjectSize,
delete_file_after_test: None,
restore_down_interfaces: None,

View file

@ -5,6 +5,7 @@ import random
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object
@ -15,8 +16,8 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils import datetime_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import get_file_hash
from pytest import FixtureRequest
@ -40,7 +41,7 @@ class TestFailoverServer(ClusterTestBase):
def containers(
self,
request: FixtureRequest,
default_wallet: WalletInfo,
default_wallet: str,
) -> list[StorageContainer]:
placement_rule = "REP 2 CBF 2 SELECT 2 FROM *"
@ -55,14 +56,17 @@ class TestFailoverServer(ClusterTestBase):
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
storage_cont_info = StorageContainerInfo(cont_id, default_wallet)
containers.append(StorageContainer(storage_cont_info, self.shell, self.cluster))
wallet = WalletInfo(path=default_wallet)
storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
containers.append(
StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
)
return containers
@reporter.step("Creation container")
@pytest.fixture()
def container(self, default_wallet: WalletInfo) -> StorageContainer:
def container(self, default_wallet: str) -> StorageContainer:
select = len(self.cluster.cluster_nodes)
placement_rule = f"REP {select - 1} CBF 1 SELECT {select} FROM *"
cont_id = create_container(
@ -72,8 +76,9 @@ class TestFailoverServer(ClusterTestBase):
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
storage_cont_info = StorageContainerInfo(cont_id, default_wallet)
return StorageContainer(storage_cont_info, self.shell, self.cluster)
wallet = WalletInfo(path=default_wallet)
storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
return StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
@reporter.step("Create object and delete after test")
@pytest.fixture(scope="class")
@ -115,7 +120,7 @@ class TestFailoverServer(ClusterTestBase):
for storage_object in storage_objects:
try:
got_file_path = get_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
endpoint=node.get_rpc_endpoint(),
@ -147,7 +152,7 @@ class TestFailoverServer(ClusterTestBase):
@pytest.fixture()
def object_and_nodes(
self, simple_object_size: ObjectSize, container: StorageContainer
self, simple_object_size: ObjectSize, container: StorageContainer, default_wallet: str
) -> tuple[StorageObjectInfo, list[ClusterNode]]:
object_info = container.generate_object(simple_object_size.value)
object_nodes = get_object_nodes(
@ -168,6 +173,7 @@ class TestFailoverServer(ClusterTestBase):
node_to_stop: ClusterNode,
cluster_state_controller: ClusterStateController,
):
with reporter.step(f"Remove {node_to_stop} from the list of nodes"):
alive_nodes = list(set(self.cluster.cluster_nodes) - {node_to_stop})
@ -181,6 +187,7 @@ class TestFailoverServer(ClusterTestBase):
with reporter.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
with reporter.step(f"check {node_to_stop.storage_node} in map"):
@ -213,7 +220,7 @@ class TestFailoverServer(ClusterTestBase):
self.tick_epochs(1, storage_nodes[0], wait_block=2)
with reporter.step(f"Stop node"):
cluster_state_controller.stop_node_host(node_to_stop, "hard")
cluster_state_controller.stop_node_host(node=node_to_stop, mode="hard")
with reporter.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
@ -231,76 +238,102 @@ class TestFailoverServer(ClusterTestBase):
@allure.title("Not enough nodes in the container with policy - 'REP 3 CBF 1 SELECT 4 FROM *'")
def test_not_enough_nodes_in_container_rep_3(
self,
container: list[StorageContainer],
object_and_nodes: tuple[StorageObjectInfo, list[ClusterNode]],
default_wallet: WalletInfo,
default_wallet: str,
cluster_state_controller: ClusterStateController,
simple_file: str,
up_stop_nodes: None,
):
object_info, object_nodes = object_and_nodes
endpoint_without_object = list(set(self.cluster.cluster_nodes) - set(object_nodes))[
0
].storage_node.get_rpc_endpoint()
endpoint_with_object = object_nodes[0].storage_node.get_rpc_endpoint()
node_not_object = list(set(self.cluster.cluster_nodes) - set(object_nodes))[0]
with reporter.step("Stop all nodes with object except first one"):
parallel(cluster_state_controller.stop_node_host, object_nodes[1:], mode="hard")
with reporter.step("Stop all nodes with object, except 1"):
for cluster_node in object_nodes[1:]:
cluster_state_controller.stop_node_host(node=cluster_node, mode="hard")
with reporter.step(f"Get object from node without object"):
get_object(default_wallet, object_info.cid, object_info.oid, self.shell, endpoint_without_object)
with reporter.step(f"Get object from node with object"):
get_object(default_wallet, object_info.cid, object_info.oid, self.shell, endpoint_with_object)
with reporter.step("Get object"):
with reporter.step(f"Get operation to {node_not_object} where it does not exist, expect success"):
get_object(
wallet=default_wallet,
cid=object_info.cid,
oid=object_info.oid,
shell=self.shell,
endpoint=node_not_object.storage_node.get_rpc_endpoint(),
)
with reporter.step(f"Get operation to {object_nodes[0]} with object, expect success"):
get_object(
wallet=default_wallet,
cid=object_info.cid,
oid=object_info.oid,
shell=self.shell,
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
)
with reporter.step(f"Put operation to node with object, expect error"):
with pytest.raises(RuntimeError):
put_object(default_wallet, simple_file, object_info.cid, self.shell, endpoint_with_object)
put_object(
wallet=default_wallet,
path=simple_file,
cid=object_info.cid,
shell=self.shell,
endpoint=node_not_object.storage_node.get_rpc_endpoint(),
)
@allure.title("Not enough nodes in the container with policy - 'REP 2 CBF 2 SELECT 4 FROM *'")
def test_not_enough_nodes_in_container_rep_2(
self,
default_wallet: WalletInfo,
default_wallet: str,
cluster_state_controller: ClusterStateController,
simple_file: str,
up_stop_nodes: None,
):
with reporter.step("Create container with full network map"):
node_count = len(self.cluster.cluster_nodes)
placement_rule = f"REP {node_count - 2} IN X CBF 2 SELECT {node_count} FROM * AS X"
cid = create_container(
with reporter.step("Creating a container with a full network map"):
select = len(self.cluster.cluster_nodes)
placement_rule = f"REP {select - 2} IN X CBF 2 SELECT {select} FROM * AS X"
cid_1 = create_container(
default_wallet,
self.shell,
self.cluster.default_rpc_endpoint,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
with reporter.step("Put object"):
oid = put_object(default_wallet, simple_file, cid, self.shell, self.cluster.default_rpc_endpoint)
oid = put_object(
wallet=default_wallet,
path=simple_file,
cid=cid_1,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Search nodes with object"):
object_nodes = get_object_nodes(self.cluster, cid, oid, self.cluster.cluster_nodes[0])
with reporter.step("Choose node to stop"):
node_to_stop = random.choice(object_nodes)
alive_node_with_object = random.choice(list(set(object_nodes) - {node_to_stop}))
alive_endpoint_with_object = alive_node_with_object.storage_node.get_rpc_endpoint()
with reporter.step("Stop random node with object"):
cluster_state_controller.stop_node_host(node_to_stop, "hard")
with reporter.step("Put object to alive node with object"):
oid_2 = put_object(default_wallet, simple_file, cid, self.shell, alive_endpoint_with_object)
with reporter.step("Get object from alive node with object"):
get_file = get_object(default_wallet, cid, oid_2, self.shell, alive_endpoint_with_object)
object_nodes = get_object_nodes(
cluster=self.cluster, cid=cid_1, oid=oid, alive_node=self.cluster.cluster_nodes[0]
)
with reporter.step("Turn off random node with object"):
cluster_state_controller.stop_node_host(node=random.choice(object_nodes[1:]), mode="hard")
with reporter.step("Checking PUT operation"):
oid_2 = put_object(
wallet=default_wallet,
path=simple_file,
cid=cid_1,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Checking GET operation"):
get_file = get_object(
wallet=default_wallet,
cid=cid_1,
oid=oid_2,
shell=self.shell,
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
)
os.remove(get_file)
with reporter.step("Create container on alive node"):
create_container(
with reporter.step("Checking creating container"):
_ = create_container(
default_wallet,
self.shell,
alive_endpoint_with_object,
shell=self.shell,
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)

View file

@ -20,7 +20,6 @@ from frostfs_testlib.steps.node_management import (
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.s3.s3_helper import search_nodes_with_bucket
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -127,7 +126,6 @@ class TestFailoverStorage(ClusterTestBase):
def test_unhealthy_tree(
self,
s3_client: S3ClientWrapper,
default_wallet: WalletInfo,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
after_run_return_all_stopped_services,
@ -157,22 +155,11 @@ class TestFailoverStorage(ClusterTestBase):
put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
node_bucket = search_nodes_with_bucket(
cluster=self.cluster,
bucket_name=bucket,
wallet=default_wallet,
shell=self.shell,
endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(),
)[0]
with reporter.step("Turn off all storage nodes except bucket node"):
for node in [node_to_stop for node_to_stop in self.cluster.cluster_nodes if node_to_stop != node_bucket]:
with reporter.step("Turn off all storage nodes except default"):
for node in self.cluster.cluster_nodes[1:]:
with reporter.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_service_of_type(node, StorageNode)
with reporter.step(f"Change s3 endpoint to bucket node"):
s3_client.set_endpoint(node_bucket.s3_gate.get_endpoint())
with reporter.step("Check that object is available"):
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
@ -536,7 +523,7 @@ class TestStorageDataLoss(ClusterTestBase):
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
shards_watcher: ShardsWatcher,
default_wallet: WalletInfo,
default_wallet: str,
test_start_time: datetime,
after_run_return_all_stopped_services: str,
):
@ -556,7 +543,7 @@ class TestStorageDataLoss(ClusterTestBase):
rule=placement_rule,
)
container = StorageContainer(
StorageContainerInfo(cid, default_wallet),
StorageContainerInfo(cid, WalletInfo(default_wallet)),
self.shell,
cluster_state_controller.cluster,
)
@ -585,7 +572,7 @@ class TestStorageDataLoss(ClusterTestBase):
with reporter.step("Objects should be available"):
for storage_object in storage_objects:
get_object(
storage_object.wallet,
storage_object.wallet_file_path,
container.get_id(),
storage_object.oid,
self.shell,

View file

@ -10,7 +10,7 @@ from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.cli.netmap_parser import NetmapParser
from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, MORPH_BLOCK_TIME
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container
@ -36,11 +36,10 @@ from frostfs_testlib.steps.node_management import (
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeStatus
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.storage_object_info import ModeNode
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import datetime_utils, string_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication
@ -59,7 +58,7 @@ class TestNodeManagement(ClusterTestBase):
@pytest.fixture
@allure.title("Create container and pick the node with data")
def create_container_and_pick_node(
self, default_wallet: WalletInfo, simple_object_size: ObjectSize
self, default_wallet: str, simple_object_size: ObjectSize
) -> Tuple[str, StorageNode]:
file_path = generate_file(simple_object_size.value)
placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
@ -134,7 +133,7 @@ class TestNodeManagement(ClusterTestBase):
@pytest.mark.add_nodes
def test_add_nodes(
self,
default_wallet: WalletInfo,
default_wallet: str,
simple_object_size: ObjectSize,
return_nodes_after_test_run,
):
@ -333,6 +332,8 @@ class TestNodeManagement(ClusterTestBase):
@pytest.mark.maintenance
class TestMaintenanceMode(ClusterTestBase):
change_node: ClusterNode = None
@pytest.fixture()
@allure.title("Init Frostfs CLI remote")
def frostfs_cli_remote(self, node_under_test: ClusterNode) -> FrostfsCli:
@ -350,50 +351,42 @@ class TestMaintenanceMode(ClusterTestBase):
@pytest.fixture()
@allure.title("Init Frostfs CLI remote")
def frostfs_cli(self, default_wallet: WalletInfo) -> FrostfsCli:
cli = FrostfsCli(
shell=self.shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=default_wallet.config_path
)
def frostfs_cli(self) -> FrostfsCli:
cli = FrostfsCli(shell=self.shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG)
return cli
@pytest.fixture()
def restore_node_status(self, cluster_state_controller: ClusterStateController, default_wallet: WalletInfo):
nodes_to_restore = []
yield nodes_to_restore
for node_to_restore in nodes_to_restore:
cluster_state_controller.set_node_status(node_to_restore, default_wallet, NodeStatus.ONLINE)
def restore_online_mode_node(self, cluster_state_controller: ClusterStateController, default_wallet: str):
yield
cluster_state_controller.set_mode_node(cluster_node=self.change_node, wallet=default_wallet, status="online")
self.tick_epoch(wait_block=2)
def check_node_status(
self, expected_status: NodeStatus, node_under_test: ClusterNode, frostfs_cli: FrostfsCli, rpc_endpoint: str
):
netmap = frostfs_cli.netmap.snapshot(rpc_endpoint).stdout
all_snapshots = NetmapParser.snapshot_all_nodes(netmap)
node_snapshot = [snapshot for snapshot in all_snapshots if node_under_test.host_ip == snapshot.node]
if expected_status == NodeStatus.OFFLINE and not node_snapshot:
assert (
node_under_test.host_ip not in netmap
), f"{node_under_test} status should be {expected_status}. See netmap:\n{netmap}"
return
assert (
node_snapshot
), f"{node_under_test} status should be {expected_status}, but was not in netmap. See netmap:\n{netmap}"
node_snapshot = node_snapshot[0]
assert (
expected_status == node_snapshot.node_status
), f"{node_under_test} status should be {expected_status}, but was {node_snapshot.node_status}. See netmap:\n{netmap}"
def basic_operations(self, wallet, cid, oid, shell, endpoint, matchs, object_size):
file_path = generate_file(object_size)
default_kw = {"wallet": wallet, "cid": cid, "shell": shell, "endpoint": endpoint}
operations = {
get_object: {"oid": oid},
search_object: {},
delete_object: {"oid": oid},
put_object: {"path": file_path},
}
for index, operation, kw in enumerate(operations.items()):
with reporter.step(f"Run {operation.__name__} object, waiting response - {matchs[index]}"):
default_kw.update(kw)
if operation == search_object and "Found" in matchs[index]:
operation(**default_kw)
continue
with pytest.raises(RuntimeError, match=matchs[index]):
operation(**default_kw)
os.remove(file_path)
@allure.title("Test of basic node operations in maintenance mode")
def test_maintenance_mode(
self,
default_wallet: WalletInfo,
default_wallet: str,
simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController,
restore_node_status: list[ClusterNode],
restore_online_mode_node: None,
):
with reporter.step("Create container and create\put object"):
cid = create_container(
@ -402,59 +395,48 @@ class TestMaintenanceMode(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
rule="REP 1 CBF 1",
)
nodes_with_container = search_nodes_with_container(
node = search_nodes_with_container(
wallet=default_wallet,
cid=cid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
cluster=self.cluster,
)
node_under_test = nodes_with_container[0]
endpoint = node_under_test.storage_node.get_rpc_endpoint()
restore_node_status.append(node_under_test)
self.change_node = node[0]
file_path = generate_file(simple_object_size.value)
oid = put_object(
wallet=default_wallet,
path=file_path,
cid=cid,
shell=self.shell,
endpoint=endpoint,
endpoint=self.change_node.storage_node.get_rpc_endpoint(),
)
with reporter.step("Enable MaintenanceModeAllowed:"):
cluster_state_controller.set_mode_node(
cluster_node=self.change_node, wallet=default_wallet, status="maintenance"
)
with reporter.step("Set node status to 'maintenance'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)
node_under_maintenance_error = "node is under maintenance"
with reporter.step("Run basic operations with node in maintenance"):
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
get_object(default_wallet, cid, oid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
search_object(default_wallet, cid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
delete_object(default_wallet, cid, oid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
put_object(default_wallet, file_path, cid, self.shell, endpoint)
with reporter.step("Run basic operations with node not in maintenance"):
other_nodes = list(set(self.cluster.cluster_nodes) - set(nodes_with_container))
endpoint = other_nodes[0].storage_node.get_rpc_endpoint()
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
get_object(default_wallet, cid, oid, self.shell, endpoint)
search_object(default_wallet, cid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
delete_object(default_wallet, cid, oid, self.shell, endpoint)
with pytest.raises(RuntimeError, match=node_under_maintenance_error):
put_object(default_wallet, file_path, cid, self.shell, endpoint)
os.remove(file_path)
other_nodes = list(set(self.cluster.cluster_nodes) - set(node))
node_and_match = {
self.change_node: ["node is under maintenance"] * 4,
other_nodes[0]: [
"object not found",
"Found 0 objects",
"object not found",
"node is under maintenance",
],
}
with reporter.step("Run basic operations"):
for cluster_node, matchs in node_and_match.items():
self.basic_operations(
wallet=default_wallet,
cid=cid,
oid=oid,
shell=self.shell,
endpoint=cluster_node.storage_node.get_rpc_endpoint(),
matchs=matchs,
object_size=simple_object_size.value,
)
@pytest.mark.sanity
@allure.title("MAINTENANCE and OFFLINE mode transitions")
@ -462,101 +444,149 @@ class TestMaintenanceMode(ClusterTestBase):
self,
cluster_state_controller: ClusterStateController,
node_under_test: ClusterNode,
default_wallet: WalletInfo,
default_wallet: str,
frostfs_cli: FrostfsCli,
restore_node_status: list[ClusterNode],
restore_online_mode_node: None,
):
restore_node_status.append(node_under_test)
alive_nodes = list(set(self.cluster.cluster_nodes) - {node_under_test})
alive_storage_node = alive_nodes[0].storage_node
alive_rpc_endpoint = alive_storage_node.get_rpc_endpoint()
self.change_node = node_under_test
cluster_nodes = list(set(self.cluster.cluster_nodes) - {node_under_test})
with reporter.step("Tick epoch"):
self.tick_epochs(2, alive_storage_node, 2)
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Set node status to 'offline'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.OFFLINE)
with reporter.step("Set node mode to offline"):
cluster_state_controller.set_mode_node(
cluster_node=node_under_test,
wallet=default_wallet,
status=ModeNode.OFFLINE.value,
)
with reporter.step("Tick epoch to update the network map"):
self.tick_epochs(2, alive_storage_node, 2)
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Check node status is 'offline' after update the network map"):
self.check_node_status(NodeStatus.OFFLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Check node mode = offline, after update the network map"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
netmap = NetmapParser.snapshot_all_nodes(netmap)
assert node_under_test.host_ip not in [
netmap_node.node for netmap_node in netmap
], f"Node {node_under_test.host_ip} not in state offline. netmap - {netmap}"
with reporter.step("Restart storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch after restart storage service and set mode to offline"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Check node status is 'online' after storage service restart"):
self.check_node_status(NodeStatus.ONLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with (reporter.step("Check node mode = online, after restart storage service")):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert (
node.node_status == ModeNode.ONLINE.value.upper()
), f"{node_under_test} actual state in netmap - {netmap}"
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Set node status to 'maintenance'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Set node mode to maintenance"):
cluster_state_controller.set_mode_node(
cluster_node=node_under_test, wallet=default_wallet, status=ModeNode.MAINTENANCE.value
)
with reporter.step("Restart storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch after restart storage service"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Check node staus is 'maintenance' after storage service restart"):
self.check_node_status(NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Check node mode = maintenance, after restart storage service and tick epoch"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert node == ModeNode.MAINTENANCE.value.upper(), f"{node_under_test} actual state in netmap - {netmap}"
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Set node status to 'offline'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.OFFLINE)
with reporter.step("Set node mode to offline"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert (
node.node_status == ModeNode.OFFLINE.value.upper()
), f"{node_under_test} actual state in netmap - {netmap}"
with reporter.step("Stop storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Start storage service"):
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Check node status is 'offline' after storage service start"):
self.check_node_status(NodeStatus.OFFLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Check node mode = offline, after tick epoch and start storage service"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert node == ModeNode.OFFLINE.value.upper(), f"{node_under_test} actual state in netmap - {netmap}"
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Check node status is 'online' after storage service start"):
self.check_node_status(NodeStatus.ONLINE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Check node mode = online, after start storage service and tick epoch"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert (
node.node_status == ModeNode.ONLINE.value.upper()
), f"{node_under_test} actual state in netmap - {netmap}"
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Set node status to 'maintenance'"):
cluster_state_controller.set_node_status(node_under_test, default_wallet, NodeStatus.MAINTENANCE)
with reporter.step("Set node mode to maintenance"):
cluster_state_controller.set_mode_node(
cluster_node=node_under_test, wallet=default_wallet, status=ModeNode.MAINTENANCE.value
)
with reporter.step("Stop storage service"):
cluster_state_controller.stop_storage_service(node_under_test)
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Start storage service"):
cluster_state_controller.start_storage_service(node_under_test)
with reporter.step("Check node status is 'maintenance'"):
self.check_node_status(NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Check node mode = maintenance"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert (
node.node_status == ModeNode.MAINTENANCE.value.upper()
), f"{node_under_test} actual state in netmap - {netmap}"
with reporter.step("Tick 2 epochs"):
self.tick_epochs(2, alive_storage_node, 2)
with reporter.step("Tick epoch"):
self.tick_epochs(epochs_to_tick=2, alive_node=cluster_nodes[0].storage_node, wait_block=2)
with reporter.step("Check node status is 'maintenance'"):
self.check_node_status(NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, alive_rpc_endpoint)
with reporter.step("Check node mode = maintenance"):
netmap = frostfs_cli.netmap.snapshot(
rpc_endpoint=cluster_nodes[0].storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout
node = NetmapParser.snapshot_one_node(netmap, node_under_test)
assert (
node.node_status == ModeNode.MAINTENANCE.value.upper()
), f"{node_under_test} actual state in netmap - {netmap}"
@allure.title("A node cannot go into maintenance if maintenance is prohibited globally in the network")
def test_maintenance_globally_forbidden(
@ -565,40 +595,53 @@ class TestMaintenanceMode(ClusterTestBase):
node_under_test: ClusterNode,
frostfs_cli_remote: FrostfsCli,
frostfs_cli: FrostfsCli,
restore_node_status: list[ClusterNode],
default_wallet: str,
restore_online_mode_node: None,
):
restore_node_status.append(node_under_test)
self.change_node = node_under_test
control_endpoint = node_under_test.service(StorageNode).get_control_endpoint()
with reporter.step("Set MaintenanceModeAllowed = false"):
cluster_state_controller.set_maintenance_mode_allowed("false", node_under_test)
with reporter.step("Set node status to 'maintenance'"):
with reporter.step("Set status node - maintenance"):
with pytest.raises(RuntimeError, match="maintenance mode is not allowed by the network"):
frostfs_cli_remote.control.set_status(endpoint=control_endpoint, status="maintenance")
with reporter.step("Set MaintenanceModeAllowed = true"):
cluster_state_controller.set_maintenance_mode_allowed("true", node_under_test)
with reporter.step("Set node status to 'maintenance'"):
with reporter.step("Set status node - maintenance"):
output = frostfs_cli_remote.control.set_status(endpoint=control_endpoint, status="maintenance")
assert "update request successfully sent" in output.stdout, f"Response = {output}"
with reporter.step("Tick epoch"):
self.tick_epoch(wait_block=2)
with reporter.step("Check node status is 'maintenance'"):
self.check_node_status(
NodeStatus.MAINTENANCE, node_under_test, frostfs_cli, node_under_test.storage_node.get_rpc_endpoint()
with reporter.step("Check state node = maintenance "):
netmap_node = NetmapParser.snapshot_one_node(
frostfs_cli.netmap.snapshot(
rpc_endpoint=node_under_test.storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout,
node_under_test,
)
assert (
netmap_node.node_status == ModeNode.MAINTENANCE.value.upper()
), f"Node actual state - {netmap_node.node_status}, expect - {ModeNode.MAINTENANCE.value}"
with reporter.step("Set node status to 'online'"):
with reporter.step("Set status node - online "):
frostfs_cli_remote.control.set_status(endpoint=control_endpoint, status="online")
with reporter.step("Tick epoch"):
self.tick_epoch()
with reporter.step("Check node status is 'online'"):
self.check_node_status(
NodeStatus.ONLINE, node_under_test, frostfs_cli, node_under_test.storage_node.get_rpc_endpoint()
with reporter.step("Check state node: online"):
netmap_node = NetmapParser.snapshot_one_node(
frostfs_cli.netmap.snapshot(
rpc_endpoint=node_under_test.storage_node.get_rpc_endpoint(), wallet=default_wallet
).stdout,
node_under_test,
)
assert (
netmap_node.node_status == ModeNode.ONLINE.value.upper()
), f"Node actual state - {netmap_node.node_status}, expect - {ModeNode.ONLINE.value}"

View file

@ -27,9 +27,7 @@ from frostfs_testlib.steps.storage_object import delete_objects
from frostfs_testlib.steps.storage_policy import get_complex_object_copies, get_simple_object_copies
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_content, get_file_hash
@ -94,11 +92,11 @@ def generate_ranges(
scope="module"
)
def storage_objects(
default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, object_size: ObjectSize, placement_policy: PlacementPolicy
default_wallet: str, client_shell: Shell, cluster: Cluster, object_size: ObjectSize
) -> list[StorageObjectInfo]:
wallet = default_wallet
# Separate containers for complex/simple objects to avoid side-effects
cid = create_container(wallet, shell=client_shell, rule=placement_policy.value, endpoint=cluster.default_rpc_endpoint)
cid = create_container(wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
file_path = generate_file(object_size.value)
file_hash = get_file_hash(file_path)
@ -119,7 +117,7 @@ def storage_objects(
storage_object = StorageObjectInfo(cid, storage_object_id)
storage_object.size = object_size.value
storage_object.wallet = wallet
storage_object.wallet_file_path = wallet
storage_object.file_path = file_path
storage_object.file_hash = file_hash
storage_object.attributes = attributes
@ -135,7 +133,7 @@ def storage_objects(
@pytest.mark.sanity
@pytest.mark.grpc_api
class TestObjectApi(ClusterTestBase):
@allure.title("Storage policy by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Storage policy by native API (obj_size={object_size})")
def test_object_storage_policies(
self,
storage_objects: list[StorageObjectInfo],
@ -149,7 +147,7 @@ class TestObjectApi(ClusterTestBase):
for storage_object in storage_objects:
if storage_object.size == simple_object_size.value:
copies = get_simple_object_copies(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
shell=self.shell,
@ -157,7 +155,7 @@ class TestObjectApi(ClusterTestBase):
)
else:
copies = get_complex_object_copies(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
shell=self.shell,
@ -165,7 +163,7 @@ class TestObjectApi(ClusterTestBase):
)
assert copies == 2, "Expected 2 copies"
@allure.title("Get object by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Get object by native API (obj_size={object_size})")
def test_get_object_api(self, storage_objects: list[StorageObjectInfo]):
"""
Validate get object native API
@ -174,7 +172,7 @@ class TestObjectApi(ClusterTestBase):
with reporter.step("Get objects and compare hashes"):
for storage_object in storage_objects:
file_path = get_object_from_random_node(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -183,7 +181,7 @@ class TestObjectApi(ClusterTestBase):
file_hash = get_file_hash(file_path)
assert storage_object.file_hash == file_hash
@allure.title("Head object by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Head object by native API (obj_size={object_size})")
def test_head_object_api(self, storage_objects: list[StorageObjectInfo]):
"""
Validate head object native API
@ -194,14 +192,14 @@ class TestObjectApi(ClusterTestBase):
with reporter.step("Head object and validate"):
head_object(
storage_object_1.wallet,
storage_object_1.wallet_file_path,
storage_object_1.cid,
storage_object_1.oid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
head_info = head_object(
storage_object_2.wallet,
storage_object_2.wallet_file_path,
storage_object_2.cid,
storage_object_2.oid,
shell=self.shell,
@ -209,14 +207,14 @@ class TestObjectApi(ClusterTestBase):
)
self.check_header_is_presented(head_info, storage_object_2.attributes)
@allure.title("Search objects by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Search objects by native API (obj_size={object_size})")
def test_search_object_api(self, storage_objects: list[StorageObjectInfo]):
"""
Validate object search by native API
"""
oids = [storage_object.oid for storage_object in storage_objects]
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
test_table = [
@ -251,7 +249,7 @@ class TestObjectApi(ClusterTestBase):
assert sorted(expected_oids) == sorted(result)
@allure.title("Search objects with removed items (obj_size={object_size})")
def test_object_search_should_return_tombstone_items(self, default_wallet: WalletInfo, object_size: ObjectSize):
def test_object_search_should_return_tombstone_items(self, default_wallet: str, object_size: ObjectSize):
"""
Validate object search with removed items
"""
@ -267,7 +265,7 @@ class TestObjectApi(ClusterTestBase):
cid=cid,
oid=put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster),
size=object_size.value,
wallet=wallet,
wallet_file_path=wallet,
file_path=file_path,
file_hash=file_hash,
)
@ -303,14 +301,14 @@ class TestObjectApi(ClusterTestBase):
object_type == "TOMBSTONE"
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
@allure.title("Get range hash by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Get range hash by native API (obj_size={object_size})")
@pytest.mark.grpc_api
def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size):
"""
Validate get_range_hash for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
@ -334,14 +332,14 @@ class TestObjectApi(ClusterTestBase):
get_file_hash(file_path, range_len, range_start) == range_hash
), f"Expected range hash to match {range_cut} slice of file payload"
@allure.title("Get range by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("Get range by native API (obj_size={object_size})")
@pytest.mark.grpc_api
def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size):
"""
Validate get_range for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
@ -366,7 +364,7 @@ class TestObjectApi(ClusterTestBase):
== range_content
), f"Expected range content to match {range_cut} slice of file payload"
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size})")
@pytest.mark.grpc_api
def test_object_get_range_negatives(
self,
@ -376,7 +374,7 @@ class TestObjectApi(ClusterTestBase):
Validate get_range negative for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_size = storage_objects[0].size
@ -414,7 +412,7 @@ class TestObjectApi(ClusterTestBase):
range_cut=range_cut,
)
@allure.title("[NEGATIVE] Get invalid range hash by native API (obj_size={object_size}, policy={placement_policy})")
@allure.title("[NEGATIVE] Get invalid range hash by native API (obj_size={object_size})")
def test_object_get_range_hash_negatives(
self,
storage_objects: list[StorageObjectInfo],
@ -423,7 +421,7 @@ class TestObjectApi(ClusterTestBase):
Validate get_range_hash negative for object by native gRPC API
"""
wallet = storage_objects[0].wallet
wallet = storage_objects[0].wallet_file_path
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_size = storage_objects[0].size

View file

@ -25,7 +25,7 @@ from pytest import FixtureRequest
@pytest.fixture(scope="module")
@allure.title("Create bearer token for OTHERS with all operations allowed for all containers")
def bearer_token_file_all_allow(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> str:
def bearer_token_file_all_allow(default_wallet: str, client_shell: Shell, cluster: Cluster) -> str:
bearer = form_bearertoken_file(
default_wallet,
"",
@ -40,7 +40,7 @@ def bearer_token_file_all_allow(default_wallet: WalletInfo, client_shell: Shell,
@pytest.fixture(scope="module")
@allure.title("Create user container for bearer token usage")
def user_container(
default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, request: FixtureRequest
default_wallet: str, client_shell: Shell, cluster: Cluster, request: FixtureRequest
) -> StorageContainer:
container_id = create_container(
default_wallet,
@ -94,17 +94,18 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
storage_objects: list[StorageObjectInfo],
bearer_token_file_all_allow: str,
):
s3_gate_wallet = WalletInfo.from_node(self.cluster.s3_gates[0])
s3_gate_wallet = self.cluster.s3_gates[0]
with reporter.step("Try to delete each object from first storage node"):
for storage_object in storage_objects:
with expect_not_raises():
delete_object(
s3_gate_wallet,
s3_gate_wallet.get_wallet_path(),
storage_object.cid,
storage_object.oid,
self.shell,
endpoint=self.cluster.default_rpc_endpoint,
bearer=bearer_token_file_all_allow,
wallet_config=s3_gate_wallet.get_wallet_config_path(),
)
@allure.title("Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})")
@ -119,7 +120,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
object_size: ObjectSize,
bearer_token_file_all_allow: str,
):
s3_gate_wallet = WalletInfo.from_node(self.cluster.s3_gates[0])
s3_gate_wallet = self.cluster.s3_gates[0]
with reporter.step("Put one object to container"):
epoch = self.get_epoch()
storage_object = user_container.generate_object(
@ -130,10 +131,11 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
for node in self.cluster.storage_nodes:
with expect_not_raises():
get_object(
s3_gate_wallet,
s3_gate_wallet.get_wallet_path(),
storage_object.cid,
storage_object.oid,
self.shell,
endpoint=node.get_rpc_endpoint(),
bearer=bearer_token_file_all_allow,
wallet_config=s3_gate_wallet.get_wallet_config_path(),
)

View file

@ -8,7 +8,6 @@ from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node
from frostfs_testlib.steps.epoch import get_epoch
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
@ -21,7 +20,7 @@ logger = logging.getLogger("NeoLogger")
@pytest.mark.grpc_api
class TestObjectApiLifetime(ClusterTestBase):
@allure.title("Object is removed when lifetime expired (obj_size={object_size})")
def test_object_api_lifetime(self, default_wallet: WalletInfo, object_size: ObjectSize):
def test_object_api_lifetime(self, default_wallet: str, object_size: ObjectSize):
"""
Test object deleted after expiration epoch.
"""

View file

@ -1,11 +1,9 @@
import logging
import re
from datetime import datetime
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.resources.common import STORAGE_GC_TIME
from frostfs_testlib.resources.error_patterns import (
LIFETIME_REQUIRED,
@ -27,7 +25,7 @@ from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import LockObjectInfo, StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
from frostfs_testlib.utils import datetime_utils
@ -40,20 +38,26 @@ FIXTURE_LOCK_LIFETIME = 5
FIXTURE_OBJECT_LIFETIME = 10
@pytest.fixture(scope="module")
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
@pytest.fixture(
scope="module",
)
def user_wallet(wallet_factory: WalletFactory):
with reporter.step("Create user wallet with container"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
wallet_file = wallet_factory.create_wallet()
return wallet_file
@pytest.fixture(scope="module")
@pytest.fixture(
scope="module",
)
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
container_id = create_container(user_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
container_id = create_container(user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
@pytest.fixture(scope="module")
@pytest.fixture(
scope="module",
)
def locked_storage_object(
user_container: StorageContainer,
client_shell: Shell,
@ -71,7 +75,7 @@ def locked_storage_object(
object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME
)
lock_object_id = lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
client_shell,
@ -94,7 +98,7 @@ def locked_storage_object(
tick_epoch(client_shell, cluster)
try:
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
client_shell,
@ -109,10 +113,10 @@ def locked_storage_object(
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
def check_object_not_found(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(
wallet,
wallet_file_path,
cid,
oid,
shell,
@ -120,10 +124,10 @@ def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell,
)
def verify_object_available(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
def verify_object_available(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with expect_not_raises():
head_object(
wallet,
wallet_file_path,
cid,
oid,
shell,
@ -146,7 +150,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME
)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -166,7 +170,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
locked_storage_object.oid,
self.shell,
@ -185,7 +189,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
lock_object = locked_storage_object.locks[0]
wallet_path = locked_storage_object.wallet
wallet_path = locked_storage_object.wallet_file_path
with pytest.raises(Exception, match=LOCK_OBJECT_REMOVAL):
delete_object(
@ -208,7 +212,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
lock_object_info = locked_storage_object.locks[0]
wallet_path = locked_storage_object.wallet
wallet_path = locked_storage_object.wallet_file_path
with pytest.raises(Exception, match=LOCK_NON_REGULAR_OBJECT):
lock_object(
@ -248,7 +252,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
lock_object_info = locked_storage_object.locks[0]
wallet_path = locked_storage_object.wallet
wallet_path = locked_storage_object.wallet_file_path
with pytest.raises(Exception, match=expected_error):
lock_object(
@ -277,7 +281,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Lock object for couple epochs"):
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -285,7 +289,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
lifetime=2,
)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -299,7 +303,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
wait_for_gc_pass_on_storage_nodes()
with expect_not_raises():
head_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -309,7 +313,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Wait for object to be deleted after third epoch"):
self.tick_epoch()
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -334,7 +338,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
storage_objects.append(user_container.generate_object(object_size.value, expire_at=current_epoch + 5))
lock_object(
storage_objects[0].wallet,
storage_objects[0].wallet_file_path,
storage_objects[0].cid,
",".join([storage_object.oid for storage_object in storage_objects]),
self.shell,
@ -346,7 +350,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step(f"Try to delete object {storage_object.oid}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -380,7 +384,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
match=LOCK_OBJECT_EXPIRATION.format(expiration_epoch=expiration_epoch, current_epoch=current_epoch),
):
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -404,7 +408,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -415,7 +419,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.tick_epochs(2)
with expect_not_raises():
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -438,7 +442,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -450,7 +454,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with expect_not_raises():
delete_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -477,7 +481,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step(f"Try to delete chunk object {chunk_object_id}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
chunk_object_id,
self.shell,
@ -494,7 +498,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
)
def test_link_object_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
link_object_id = get_link_object(
new_locked_storage_object.wallet,
new_locked_storage_object.wallet_file_path,
new_locked_storage_object.cid,
new_locked_storage_object.oid,
self.shell,
@ -565,7 +569,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
link_object_id = get_link_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
locked_storage_object.oid,
self.shell,
@ -575,7 +579,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step(f"Try to delete link object {link_object_id}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
delete_object(
locked_storage_object.wallet,
locked_storage_object.wallet_file_path,
locked_storage_object.cid,
link_object_id,
self.shell,
@ -593,7 +597,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Apply first lock to object for 3 epochs"):
lock_object_id_0 = lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -605,7 +609,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Check first lock is still available"):
verify_object_available(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
lock_object_id_0,
self.shell,
@ -614,7 +618,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Apply second lock to object for 3 more epochs"):
lock_object_id_1 = lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -626,7 +630,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Verify first lock is expired and removed"):
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
lock_object_id_0,
self.shell,
@ -635,7 +639,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Verify second lock is still available"):
verify_object_available(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
lock_object_id_1,
self.shell,
@ -644,7 +648,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Apply third lock to object for 3 more epochs"):
lock_object(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -655,7 +659,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Verify object is deleted after all locks are expired"):
self.tick_epochs(4)
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -683,7 +687,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Lock objects for 4 epochs"):
lock_object(
storage_objects[0].wallet,
storage_objects[0].wallet_file_path,
storage_objects[0].cid,
",".join([storage_object.oid for storage_object in storage_objects]),
self.shell,
@ -697,7 +701,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step(f"Check objects at epoch {current_epoch + epoch_i + 2}"):
for storage_object in storage_objects:
verify_object_available(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -708,7 +712,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.tick_epoch()
for storage_object in storage_objects:
check_object_not_found(
storage_object.wallet,
storage_object.wallet_file_path,
storage_object.cid,
storage_object.oid,
self.shell,

View file

@ -1,5 +1,6 @@
import logging
import random
from time import sleep
import allure
import pytest
@ -10,7 +11,6 @@ from frostfs_testlib.steps.cli.object import delete_object, head_object, put_obj
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file
@ -33,7 +33,7 @@ class TestReplication(ClusterTestBase):
@allure.title("Replication (obj_size={object_size})")
def test_replication(
self,
default_wallet: WalletInfo,
default_wallet: str,
client_shell: Shell,
cluster: Cluster,
object_size: ObjectSize,
@ -98,19 +98,18 @@ class TestReplication(ClusterTestBase):
f"expected attribute value: {attribute_value}"
)
# TODO: Research why this fails
# with reporter.step("Cleanup"):
# delete_object(
# wallet=default_wallet,
# cid=cid,
# oid=oid,
# shell=client_shell,
# endpoint=cluster.default_rpc_endpoint,
# )
with reporter.step("Cleanup"):
delete_object(
wallet=default_wallet,
cid=cid,
oid=oid,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
)
# delete_container(
# wallet=default_wallet,
# cid=cid,
# shell=client_shell,
# endpoint=cluster.default_rpc_endpoint,
# )
delete_container(
wallet=default_wallet,
cid=cid,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
)

View file

@ -22,6 +22,7 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger")
@pytest.mark.skip("Skipped temporarly")
@pytest.mark.http_gate
@pytest.mark.http_put
class Test_http_bearer(ClusterTestBase):
@ -69,7 +70,7 @@ class Test_http_bearer(ClusterTestBase):
bearer_signed = f"{bearer}_signed"
sign_bearer(
shell=self.shell,
wallet=self.wallet,
wallet_path=self.wallet,
eacl_rules_file_from=bearer,
eacl_rules_file_to=bearer_signed,
json=False,
@ -113,5 +114,5 @@ class Test_http_bearer(ClusterTestBase):
cid=user_container,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
)

View file

@ -92,7 +92,8 @@ class TestHttpGate(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@ -102,6 +103,7 @@ class TestHttpGate(ClusterTestBase):
)
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@pytest.mark.skip("Skipped temporarly")
@pytest.mark.http_gate
@pytest.mark.http_put
class TestHttpPut(ClusterTestBase):
@ -148,7 +150,8 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.link(
@ -201,7 +204,8 @@ class TestHttpPut(ClusterTestBase):
file_name=file_path,
cid=cid,
attrs=attributes,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Expiration-Epoch in HTTP header (epoch_gap={epoch_gap})")
@ -232,13 +236,19 @@ class TestHttpPut(ClusterTestBase):
path=file_path,
headers=headers,
endpoint=http_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
if get_epoch(self.shell, self.cluster) + 1 <= valid_until:
oids_to_be_valid.append(oid)
else:
oids_to_be_expired.append(oid)
with reporter.step("This object can be got"):
get_via_http_gate(cid=cid, oid=oid, node=self.cluster.cluster_nodes[0])
get_via_http_gate(
cid=cid,
oid=oid,
endpoint=http_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
self.tick_epoch()
@ -250,12 +260,18 @@ class TestHttpPut(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=cid,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern=OBJECT_NOT_FOUND_ERROR,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
for oid in oids_to_be_valid:
with reporter.step(f"{oid} shall be valid and can be got"):
get_via_http_gate(cid=cid, oid=oid, node=self.cluster.cluster_nodes[0])
get_via_http_gate(
cid=cid,
oid=oid,
endpoint=http_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Zip in HTTP header")
def test_zip_in_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
@ -286,7 +302,12 @@ class TestHttpPut(ClusterTestBase):
endpoint=self.cluster.default_http_gate_endpoint,
)
dir_path = get_via_zip_http_gate(cid=cid, prefix=common_prefix, node=self.cluster.cluster_nodes[0])
dir_path = get_via_zip_http_gate(
cid=cid,
prefix=common_prefix,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("Verify hashes"):
assert get_file_hash(f"{dir_path}/file1") == get_file_hash(file_path_simple)
@ -324,7 +345,8 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
verify_object_hash(
oid=oid_curl,
@ -333,7 +355,8 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
object_getter=get_via_http_curl,
)
@ -370,6 +393,7 @@ class TestHttpPut(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
object_getter=get_via_http_curl,
)

View file

@ -28,6 +28,7 @@ OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
logger = logging.getLogger("NeoLogger")
@pytest.mark.skip("Skipped temporarly")
@pytest.mark.http_gate
@pytest.mark.http_put
class Test_http_headers(ClusterTestBase):
@ -69,7 +70,7 @@ class Test_http_headers(ClusterTestBase):
)
storage_object = StorageObjectInfo(cid, storage_object_id)
storage_object.size = os.path.getsize(file_path)
storage_object.wallet = wallet
storage_object.wallet_file_path = wallet
storage_object.file_path = file_path
storage_object.attributes = attributes
@ -96,7 +97,8 @@ class Test_http_headers(ClusterTestBase):
file_name=storage_object_1.file_path,
cid=storage_object_1.cid,
attrs={"Chapter2": storage_object_1.attributes["Chapter2"]},
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Get object2 with different attributes, then delete object2 and get object1")
@ -127,7 +129,8 @@ class Test_http_headers(ClusterTestBase):
file_name=storage_object_2.file_path,
cid=storage_object_2.cid,
attrs=attributes,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("Delete object#2 and verify is the container deleted"):
delete_object(
@ -140,8 +143,9 @@ class Test_http_headers(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=storage_object_2.cid,
oid=storage_object_2.oid,
node=self.cluster.cluster_nodes[0],
error_pattern=OBJECT_ALREADY_REMOVED_ERROR,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
storage_objects_with_attributes.remove(storage_object_2)
@ -154,7 +158,8 @@ class Test_http_headers(ClusterTestBase):
file_name=storage_object_1.file_path,
cid=storage_object_1.cid,
attrs=key_value_pair,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("[NEGATIVE] Put object and get right after container is deleted")
@ -210,8 +215,9 @@ class Test_http_headers(ClusterTestBase):
try_to_get_object_via_passed_request_and_expect_error(
cid=storage_object_1.cid,
oid="",
node=self.cluster.cluster_nodes[0],
error_pattern=error_pattern,
attrs=attrs_obj3,
http_request_path=request,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)

View file

@ -91,7 +91,8 @@ class Test_http_object(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("[Negative] try to get object: [get/$CID/chapter1/peace]"):
@ -101,10 +102,11 @@ class Test_http_object(ClusterTestBase):
try_to_get_object_via_passed_request_and_expect_error(
cid=cid,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern=expected_err_msg,
http_request_path=request,
attrs=attrs,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("Download the object with attribute [get_by_attribute/$CID/chapter1/peace]"):
@ -113,16 +115,18 @@ class Test_http_object(ClusterTestBase):
file_name=file_path,
cid=cid,
attrs=attrs,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
with reporter.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
request = f"/get_by_attribute/{cid}/{oid}"
try_to_get_object_via_passed_request_and_expect_error(
cid=cid,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern=expected_err_msg,
http_request_path=request,
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
@allure.title("Put over s3, Get over HTTP with bucket name and key")
@ -152,7 +156,8 @@ class Test_http_object(ClusterTestBase):
obj_http = get_via_http_gate(
cid=None,
oid=None,
node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
request_path=request,
)
with reporter.step("Verify hashes"):

View file

@ -13,6 +13,7 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger")
@pytest.mark.skip("Skipped temporarly")
@pytest.mark.http_gate
@pytest.mark.http_put
class Test_http_streaming(ClusterTestBase):
@ -60,5 +61,6 @@ class Test_http_streaming(ClusterTestBase):
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)

View file

@ -32,6 +32,7 @@ SYSTEM_EXPIRATION_TIMESTAMP = "System-Expiration-Timestamp"
SYSTEM_EXPIRATION_RFC3339 = "System-Expiration-RFC3339"
@pytest.mark.skip("Skipped temporarly")
@pytest.mark.http_gate
@pytest.mark.http_put
class Test_http_system_header(ClusterTestBase):
@ -123,7 +124,8 @@ class Test_http_system_header(ClusterTestBase):
cid=user_container,
shell=self.shell,
nodes=self.cluster.storage_nodes,
request_node=self.cluster.cluster_nodes[0],
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
head = head_object(
wallet=self.wallet,
@ -220,8 +222,9 @@ class Test_http_system_header(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@ -259,8 +262,9 @@ class Test_http_system_header(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@ -300,8 +304,9 @@ class Test_http_system_header(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@ -351,8 +356,9 @@ class Test_http_system_header(ClusterTestBase):
try_to_get_object_and_expect_error(
cid=user_container,
oid=oid,
node=self.cluster.cluster_nodes[0],
error_pattern="404 Not Found",
endpoint=self.cluster.default_http_gate_endpoint,
http_hostname=self.cluster.default_http_hostname[0],
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):

View file

@ -5,7 +5,6 @@ from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash, split_file
@ -55,7 +54,7 @@ class TestS3GateMultipart(ClusterTestBase):
def test_s3_abort_multipart(
self,
s3_client: S3ClientWrapper,
default_wallet: WalletInfo,
default_wallet: str,
bucket: str,
simple_object_size: ObjectSize,
complex_object_size: ObjectSize,

View file

@ -8,7 +8,6 @@ from frostfs_testlib.steps.cli.container import search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.storage_policy import get_simple_object_copies
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file
@ -18,9 +17,7 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True)
class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
def test_s3_bucket_location(
self, default_wallet: WalletInfo, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
):
def test_s3_bucket_location(self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path_1 = generate_file(simple_object_size.value)
file_name_1 = s3_helper.object_key_from_file_path(file_path_1)
file_path_2 = generate_file(simple_object_size.value)
@ -67,7 +64,7 @@ class TestS3GatePolicy(ClusterTestBase):
)
assert copies_1 == 1
for cluster_node in self.cluster.cluster_nodes:
cid_2 = search_container_by_name(name=bucket_2, node=cluster_node)
cid_2 = search_container_by_name(name=bucket_1, node=cluster_node)
if cid_2:
break
copies_2 = get_simple_object_copies(

View file

@ -23,21 +23,21 @@ def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
Compare binaries versions from external source (url) and deployed on servers.
"""
with reporter.step("Get binaries versions from servers"):
got_versions, exсeptions_remote_binaries_versions = get_remote_binaries_versions(hosting)
got_versions = get_remote_binaries_versions(hosting)
environment_dir = request.config.getoption("--alluredir") or ASSETS_DIR
env_file = os.path.join(environment_dir, "environment.properties")
env_properties = read_env_properties(env_file)
# compare versions from servers and file
exсeptions = []
exeptions = []
additional_env_properties = {}
for binary_name, binary in got_versions.items():
version = binary["version"]
requires_check = binary["check"]
if requires_check and not fullmatch(r"^\d+\.\d+\.\d+(-.*)?(?<!dirty)", version):
exсeptions.append(f"{binary_name}: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}")
exeptions.append(f"{binary_name}: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}")
# If some binary was not listed in the env properties file, let's add it
# so that we have full information about versions in allure report
@ -47,11 +47,9 @@ def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
if env_properties and additional_env_properties:
save_env_properties(env_file, additional_env_properties)
exсeptions.extend(exсeptions_remote_binaries_versions)
# create clear beautiful error with aggregation info
if exсeptions:
msg = "\n".join(exсeptions)
if exeptions:
msg = "\n".join(exeptions)
raise AssertionError(f"Found binaries with unexpected versions:\n{msg}")

View file

@ -1,28 +1,26 @@
from datetime import datetime
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
@pytest.fixture(scope="module")
def owner_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create user wallet which owns containers and objects"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
def owner_wallet(wallet_factory: WalletFactory) -> WalletInfo:
"""
Returns wallet which owns containers and objects
"""
return wallet_factory.create_wallet()
@pytest.fixture(scope="module")
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create user wallet which will use objects from owner via static session"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
def user_wallet(wallet_factory: WalletFactory) -> WalletInfo:
"""
Returns wallet which will use objects from owner via static session
"""
return wallet_factory.create_wallet()
@pytest.fixture(scope="module")
def stranger_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create stranger user wallet which should fail to obtain data"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}")
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
def stranger_wallet(wallet_factory: WalletFactory) -> WalletInfo:
"""
Returns stranger wallet which should fail to obtain data
"""
return wallet_factory.create_wallet()

View file

@ -3,12 +3,12 @@ import random
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import DEFAULT_WALLET_PASS
from frostfs_testlib.resources.error_patterns import SESSION_NOT_FOUND
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import delete_object, put_object, put_object_to_random_node
from frostfs_testlib.steps.session_token import create_session_token
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import generate_file
@ -18,7 +18,7 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.session_token
class TestDynamicObjectSession(ClusterTestBase):
@allure.title("Object Operations with Session Token (obj_size={object_size})")
def test_object_session_token(self, default_wallet: WalletInfo, object_size: ObjectSize):
def test_object_session_token(self, default_wallet: str, object_size: ObjectSize):
"""
Test how operations over objects are executed with a session token
@ -34,6 +34,7 @@ class TestDynamicObjectSession(ClusterTestBase):
with reporter.step("Init wallet"):
wallet = default_wallet
address = wallet_utils.get_last_address_from_wallet(wallet, "")
with reporter.step("Nodes Settlements"):
session_token_node, container_node, non_container_node = random.sample(self.cluster.storage_nodes, 3)
@ -41,8 +42,9 @@ class TestDynamicObjectSession(ClusterTestBase):
with reporter.step("Create Session Token"):
session_token = create_session_token(
shell=self.shell,
owner=default_wallet.get_address(),
wallet=default_wallet,
owner=address,
wallet_path=wallet,
wallet_password=DEFAULT_WALLET_PASS,
rpc_endpoint=session_token_node.get_rpc_endpoint(),
)

View file

@ -50,8 +50,8 @@ RANGE_OFFSET_FOR_COMPLEX_OBJECT = 200
@pytest.fixture(scope="module")
def storage_containers(owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> list[str]:
cid = create_container(owner_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
other_cid = create_container(owner_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
other_cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
yield [cid, other_cid]
@ -74,7 +74,7 @@ def storage_objects(
# upload couple objects
for _ in range(3):
storage_object_id = put_object_to_random_node(
wallet=owner_wallet,
wallet=owner_wallet.path,
path=file_path,
cid=storage_containers[0],
shell=client_shell,
@ -83,7 +83,7 @@ def storage_objects(
storage_object = StorageObjectInfo(storage_containers[0], storage_object_id)
storage_object.size = object_size.value
storage_object.wallet = owner_wallet
storage_object.wallet_file_path = owner_wallet.path
storage_object.file_path = file_path
storage_objects.append(storage_object)
@ -163,7 +163,7 @@ class TestObjectStaticSession(ClusterTestBase):
for node in self.cluster.storage_nodes:
for storage_object in storage_objects[0:2]:
method_under_test(
wallet=user_wallet,
wallet=user_wallet.path,
cid=storage_object.cid,
oid=storage_object.oid,
shell=self.shell,
@ -196,7 +196,7 @@ class TestObjectStaticSession(ClusterTestBase):
with reporter.step(f"Check range {range_to_test}"):
with expect_not_raises():
method_under_test(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
shell=self.shell,
@ -219,7 +219,7 @@ class TestObjectStaticSession(ClusterTestBase):
cid = storage_objects[0].cid
expected_object_ids = [storage_object.oid for storage_object in storage_objects[0:2]]
actual_object_ids = search_object(
user_wallet,
user_wallet.path,
cid,
self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -240,7 +240,7 @@ class TestObjectStaticSession(ClusterTestBase):
"""
with pytest.raises(Exception, match=UNRELATED_OBJECT):
head_object(
user_wallet,
user_wallet.path,
storage_objects[2].cid,
storage_objects[2].oid,
self.shell,
@ -262,7 +262,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=UNRELATED_KEY):
head_object(
stranger_wallet,
stranger_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -284,7 +284,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=WRONG_VERB):
get_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -307,7 +307,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=UNRELATED_CONTAINER):
get_object_from_random_node(
user_wallet,
user_wallet.path,
storage_containers[1],
storage_object.oid,
self.shell,
@ -341,7 +341,7 @@ class TestObjectStaticSession(ClusterTestBase):
signed_token_file = sign_session_token(self.shell, session_token_file, stranger_wallet)
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
head_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -375,7 +375,7 @@ class TestObjectStaticSession(ClusterTestBase):
signed_token_file = sign_session_token(self.shell, session_token_file, owner_wallet)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(
user_wallet,
user_wallet.path,
container,
storage_object.oid,
self.shell,
@ -407,7 +407,7 @@ class TestObjectStaticSession(ClusterTestBase):
)
with pytest.raises(Exception, match=INVALID_SIGNATURE):
head_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -448,7 +448,7 @@ class TestObjectStaticSession(ClusterTestBase):
with reporter.step("Object should be available with session token after token creation"):
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -460,7 +460,7 @@ class TestObjectStaticSession(ClusterTestBase):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -472,7 +472,7 @@ class TestObjectStaticSession(ClusterTestBase):
self.tick_epoch()
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -514,7 +514,7 @@ class TestObjectStaticSession(ClusterTestBase):
with reporter.step("Object should NOT be available with session token after token creation"):
with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -526,7 +526,7 @@ class TestObjectStaticSession(ClusterTestBase):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -538,7 +538,7 @@ class TestObjectStaticSession(ClusterTestBase):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -550,7 +550,7 @@ class TestObjectStaticSession(ClusterTestBase):
self.tick_epoch()
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -589,7 +589,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,
@ -610,7 +610,7 @@ class TestObjectStaticSession(ClusterTestBase):
storage_object = storage_objects[0]
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
delete_object(
user_wallet,
user_wallet.path,
storage_object.cid,
storage_object.oid,
self.shell,
@ -632,7 +632,7 @@ class TestObjectStaticSession(ClusterTestBase):
storage_object = storage_objects[0]
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
put_object_to_random_node(
user_wallet,
user_wallet.path,
storage_object.file_path,
storage_object.cid,
self.shell,
@ -671,7 +671,7 @@ class TestObjectStaticSession(ClusterTestBase):
with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object(
user_wallet,
user_wallet.path,
container,
object_id,
self.shell,

View file

@ -43,7 +43,7 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
with reporter.step("Create container with static session token"):
cid = create_container(
user_wallet,
user_wallet.path,
session_token=static_sessions[ContainerVerb.CREATE],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -51,12 +51,14 @@ class TestSessionTokenContainer(ClusterTestBase):
)
container_info: dict[str, str] = get_container(
owner_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
owner_wallet.path, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert container_info["ownerID"] == owner_wallet.get_address()
assert cid not in list_containers(user_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid in list_containers(owner_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid not in list_containers(
user_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid in list_containers(owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
def test_static_session_token_container_create_with_other_verb(
self,
@ -70,7 +72,7 @@ class TestSessionTokenContainer(ClusterTestBase):
for verb in [verb for verb in ContainerVerb if verb != ContainerVerb.CREATE]:
with pytest.raises(Exception):
create_container(
user_wallet,
user_wallet.path,
session_token=static_sessions[verb],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -88,7 +90,7 @@ class TestSessionTokenContainer(ClusterTestBase):
with reporter.step("Try create container with static session token without PUT rule"):
with pytest.raises(Exception):
create_container(
stranger_wallet,
stranger_wallet.path,
session_token=static_sessions[ContainerVerb.CREATE],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
@ -106,14 +108,14 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
with reporter.step("Create container"):
cid = create_container(
owner_wallet,
owner_wallet.path,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
wait_for_creation=False,
)
with reporter.step("Delete container with static session token"):
delete_container(
wallet=user_wallet,
wallet=user_wallet.path,
cid=cid,
session_token=static_sessions[ContainerVerb.DELETE],
shell=self.shell,
@ -121,7 +123,9 @@ class TestSessionTokenContainer(ClusterTestBase):
await_mode=True,
)
assert cid not in list_containers(owner_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
assert cid not in list_containers(
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
@pytest.mark.sanity
def test_static_session_token_container_set_eacl(
@ -137,18 +141,18 @@ class TestSessionTokenContainer(ClusterTestBase):
"""
with reporter.step("Create container"):
cid = create_container(
owner_wallet,
owner_wallet.path,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
file_path = generate_file(simple_object_size.value)
assert can_put_object(stranger_wallet, cid, file_path, self.shell, self.cluster)
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
with reporter.step("Deny all operations for other via eACL"):
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet,
user_wallet.path,
cid,
create_eacl(cid, eacl_deny, shell=self.shell),
shell=self.shell,
@ -157,4 +161,4 @@ class TestSessionTokenContainer(ClusterTestBase):
)
wait_for_cache_expired()
assert not can_put_object(stranger_wallet, cid, file_path, self.shell, self.cluster)
assert not can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)

View file

@ -1,10 +1,12 @@
import json
import pathlib
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.resources.wellknown_acl import EACL_PUBLIC_READ_WRITE
from frostfs_testlib.steps.cli.container import create_container, delete_container
from frostfs_testlib.steps.cli.object import delete_object, get_object, get_object_nodes, put_object
@ -12,8 +14,7 @@ from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.shard import Shard
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel, wait_for_success
from frostfs_testlib.testing import parallel
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
@ -21,25 +22,18 @@ from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.shard
class TestControlShard(ClusterTestBase):
@staticmethod
@wait_for_success(180, 30)
def get_object_path_and_name_file(oid: str, cid: str, node: ClusterNode) -> tuple[str, str]:
oid_path = f"{oid[0]}/{oid[1]}/{oid[2]}/{oid[3]}"
object_path = None
with reporter.step("Search object file"):
node_shell = node.storage_node.host.get_shell()
data_path = node.storage_node.get_data_directory()
all_datas = node_shell.exec(f"ls -la {data_path}/data | awk '{{ print $9 }}'").stdout.strip()
for data_dir in all_datas.replace(".", "").strip().split("\n"):
check_dir = node_shell.exec(
f" [ -d {data_path}/data/{data_dir}/data/{oid_path} ] && echo 1 || echo 0"
).stdout
all_datas = node_shell.exec(f"ls -la {data_path} | awk '{{ print $9 }}'").stdout.strip()
for data in all_datas.replace(".", "").strip().split("\n"):
check_dir = node_shell.exec(f" [ -d {data_path}/{data}/{oid_path} ] && echo 1 || echo 0").stdout
if "1" in check_dir:
object_path = f"{data_path}/data/{data_dir}/data/{oid_path}"
object_path = f"{data_path}/{data}/{oid_path}"
object_name = f"{oid[4:]}.{cid}"
break
assert object_path is not None, f"{oid} object not found in directory - {data_path}/data"
return object_path, object_name
def set_shard_rw_mode(self, node: ClusterNode):
@ -56,7 +50,7 @@ class TestControlShard(ClusterTestBase):
parallel(self.set_shard_rw_mode, self.cluster.cluster_nodes)
@pytest.fixture()
def oid_cid_node(self, default_wallet: WalletInfo, max_object_size: int) -> tuple[str, str, ClusterNode]:
def oid_cid_node(self, default_wallet: str) -> tuple[str, str, ClusterNode]:
with reporter.step("Create container, and put object"):
cid = create_container(
wallet=default_wallet,
@ -65,7 +59,7 @@ class TestControlShard(ClusterTestBase):
rule="REP 1 CBF 1",
basic_acl=EACL_PUBLIC_READ_WRITE,
)
file = generate_file(round(max_object_size * 0.8))
file = generate_file(5242880)
oid = put_object(
wallet=default_wallet, path=file, cid=cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
@ -81,6 +75,20 @@ class TestControlShard(ClusterTestBase):
)
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
@staticmethod
def get_shards_from_config(node: StorageNode) -> list[Shard]:
config_file = node.get_shard_config_path()
file_type = pathlib.Path(config_file).suffix
parser_method = {
".env": node.get_shards_from_env,
".yaml": node.get_shards,
".yml": node.get_shards,
}
shards = parser_method[file_type]()
return shards
@staticmethod
def get_shards_from_cli(node: StorageNode) -> list[Shard]:
wallet_path = node.get_remote_wallet_path()
@ -89,7 +97,7 @@ class TestControlShard(ClusterTestBase):
cli_config = node.host.get_cli_config("frostfs-cli")
cli = FrostfsCli(node.host.get_shell(), cli_config.exec_path)
cli = FrostfsCli(node.host.get_shell(), cli_config.exec_path, DEFAULT_WALLET_CONFIG)
result = cli.shards.list(
endpoint=control_endpoint,
wallet=wallet_path,
@ -112,7 +120,7 @@ class TestControlShard(ClusterTestBase):
@allure.title("All shards are available")
def test_control_shard(self, cluster: Cluster):
for storage_node in cluster.storage_nodes:
shards_from_config = storage_node.get_shards()
shards_from_config = self.get_shards_from_config(storage_node)
shards_from_cli = self.get_shards_from_cli(storage_node)
assert set(shards_from_config) == set(shards_from_cli)
@ -120,14 +128,13 @@ class TestControlShard(ClusterTestBase):
@pytest.mark.failover
def test_shard_errors(
self,
default_wallet: WalletInfo,
default_wallet: str,
oid_cid_node: tuple[str, str, ClusterNode],
change_config_storage: None,
revert_all_shards_mode: None,
):
oid, cid, node = oid_cid_node
with reporter.step("Search object in system."):
object_path, object_name = self.get_object_path_and_name_file(*oid_cid_node)
object_path, object_name = self.get_object_path_and_name_file(*oid_cid_node)
with reporter.step("Block read file"):
node.host.get_shell().exec(f"chmod a-r {object_path}/{object_name}")
with reporter.step("Get object, expect 6 errors"):
@ -144,5 +151,5 @@ class TestControlShard(ClusterTestBase):
for shard in ShardsWatcher(node).get_shards():
if shard["blobstor"][1]["path"] in object_path:
with reporter.step(f"Shard - {shard['shard_id']} to {node.host_ip}, mode - {shard['mode']}"):
assert shard["mode"] == "degraded-read-only"
assert shard["mode"] == "read-only"
break

View file

@ -36,7 +36,7 @@ class TestLogs:
logs_dir = os.path.join(temp_directory, "logs")
os.makedirs(logs_dir)
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|wallet passwd|secret \bkey\b|access \bkey\b|cannot assign requested address"
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|wallet passwd|secret key|access key|cannot assign requested address"
exclude_filter = r"too many requests"
time.sleep(2)