Compare commits

...

15 commits

Author SHA1 Message Date
56fcbe8b77 fix password for node
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
2022-11-23 15:28:02 +03:00
bef01eec48 add ir pass to storage_group
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
2022-11-22 19:41:24 +03:00
14316c1fd7 get password from hosting config to s3_gate
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
2022-11-18 14:12:16 +03:00
a1ab09a074 Fix too long logs dir for analyze logs
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2022-11-18 13:49:31 +03:00
4a9e3facff Fix too long logs dir for analyze logs
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2022-11-18 13:42:16 +03:00
2140ce7f89 Add log analyze for each test
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2022-11-17 18:18:15 +03:00
1ec6540063 Refactor balance tests
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2022-11-16 19:00:35 +03:00
72d30c7e06 Find critical pattern in system logs
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2022-11-15 19:36:56 +03:00
431902c118 Skip storage goup lifetime only test
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2022-11-14 10:51:19 +03:00
b66d04faf8 Add timeout between commands
According to developers opinion (@fyrchik)

Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2022-11-10 17:11:40 +03:00
e3b27a7643 Disable storagegroup tests for support
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2022-11-10 15:20:50 +03:00
26962617c9 Bump neofs-testlib version 0.3.0 -> 0.4.0
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2022-11-07 12:50:07 +03:00
24fdcd88df Move requirements.txt to root repository folder
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2022-11-07 12:49:52 +03:00
4eb4963f17 Make node management tests to be last
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2022-11-03 16:57:41 +03:00
af9c726624 Fix storage group tests
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2022-11-03 11:44:04 +03:00
28 changed files with 403 additions and 255 deletions

View file

@ -164,3 +164,15 @@ def get_file_content(
content = file.read()
return content
def sanitize_for_file_name(string: str) -> str:
"""
Returns string with only alpha num string with all other characters replaced by '_'
which is valid string to use as file name
Args:
string: string to sanitize
"""
return "".join(character if character.isalnum() else "_" for character in string).strip("_")

View file

@ -5,7 +5,6 @@ from time import sleep
import allure
from neofs_testlib.shell import Shell
from remote_process import RemoteProcess
EXIT_RESULT_CODE = 0

View file

@ -41,9 +41,7 @@ class RemoteProcess:
Returns:
RemoteProcess instance for further examination
"""
remote_process = cls(
cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", shell=shell
)
remote_process = cls(cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", shell=shell)
remote_process._create_process_dir()
remote_process._generate_command_script(command)
remote_process._start_process()

View file

@ -1,7 +1,11 @@
import os
import time
from typing import Any, Optional
import allure
import yaml
from common import STORAGE_GC_TIME
from neofs_testlib.hosting import Hosting
def parse_time(value: str) -> int:
@ -60,3 +64,16 @@ def wait_for_gc_pass_on_storage_nodes() -> None:
wait_time = parse_time(STORAGE_GC_TIME)
with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"):
time.sleep(wait_time)
def get_wallet_password(hosting: Hosting, service_name: str) -> Optional[str]:
service_config = hosting.get_service_config(service_name)
return service_config.attributes.get("wallet_password")
def create_wallet_config(hosting: Hosting, service_name: str) -> Optional[str]:
password = get_wallet_password(hosting=hosting, service_name=service_name)
wallet_config_path = os.path.join(os.getcwd(), f"{service_name}_wallet_config.yml")
with open(wallet_config_path, "w") as file:
yaml.dump({"password": password}, file)
return wallet_config_path

View file

@ -6,6 +6,8 @@ log_format = %(asctime)s [%(levelname)4s] %(message)s
log_cli_date_format = %Y-%m-%d %H:%M:%S
log_date_format = %H:%M:%S
markers =
# controller markers
no_log_analyze: skip critical errors analyzer at the end of test
# special markers
sanity: small tests subset
staging: test to be excluded from run in verifier/pr-validation/sanity jobs and run test in staging job

View file

@ -1,68 +1 @@
aiodns==3.0.0
aiohttp==3.7.4.post0
aioresponses==0.7.2
allure-pytest==2.9.45
allure-python-commons==2.9.45
async-timeout==3.0.1
asynctest==0.13.0
attrs==21.4.0
base58==2.1.0
bitarray==2.3.4
black==22.8.0
boto3==1.16.33
botocore==1.19.33
certifi==2022.5.18
cffi==1.15.0
chardet==4.0.0
charset-normalizer==2.0.12
coverage==6.3.3
docker==4.4.0
docutils==0.17.1
Events==0.4
flake8==4.0.1
idna==3.3
iniconfig==1.1.1
isort==5.10.1
jmespath==0.10.0
jsonschema==4.5.1
lz4==3.1.3
mccabe==0.6.1
mmh3==3.0.0
multidict==6.0.2
mypy==0.950
mypy-extensions==0.4.3
neo-mamba==0.10.0
neo3crypto==0.2.1
neo3vm==0.9.0
neo3vm-stubs==0.9.0
neofs-testlib==0.3.0
netaddr==0.8.0
orjson==3.6.8
packaging==21.3
paramiko==2.10.3
pexpect==4.8.0
pluggy==1.0.0
pre-commit==2.20.0
ptyprocess==0.7.0
py==1.11.0
pybiginteger==1.2.6
pybiginteger-stubs==1.2.6
pycares==4.1.2
pycodestyle==2.8.0
pycparser==2.21
pycryptodome==3.11.0
pyflakes==2.4.0
pyparsing==3.0.9
pyrsistent==0.18.1
pytest==7.1.2
python-dateutil==2.8.2
requests==2.28.0
robotframework==4.1.2
s3transfer==0.3.7
six==1.16.0
tenacity==8.0.1
tomli==2.0.1
typing-extensions==4.2.0
urllib3==1.26.9
websocket-client==1.3.2
yarl==1.7.2
-r ../requirements.txt

View file

@ -3,11 +3,10 @@ from dataclasses import asdict
import allure
from common import STORAGE_NODE_SERVICE_NAME_REGEX
from k6 import K6, LoadParams, LoadResults
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import SSHShell
from k6 import K6, LoadParams, LoadResults
@allure.title("Get services endpoints")
def get_services_endpoints(

View file

@ -12,14 +12,9 @@ import urllib3
from botocore.config import Config
from botocore.exceptions import ClientError
from cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
from common import (
NEOFS_AUTHMATE_EXEC,
NEOFS_ENDPOINT,
S3_GATE,
S3_GATE_WALLET_PASS,
S3_GATE_WALLET_PATH,
)
from common import NEOFS_AUTHMATE_EXEC, NEOFS_ENDPOINT, S3_GATE, S3_GATE_WALLET_PATH
from data_formatters import get_wallet_public_key
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from python_keywords.container import list_containers
@ -43,7 +38,7 @@ class TestS3GateBase:
@pytest.fixture(scope="class", autouse=True)
@allure.title("[Class/Autouse]: Create S3 client")
def s3_client(self, prepare_wallet_and_deposit, client_shell: Shell, request):
def s3_client(self, prepare_wallet_and_deposit, client_shell: Shell, request, hosting: Hosting):
wallet = prepare_wallet_and_deposit
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
@ -53,7 +48,7 @@ class TestS3GateBase:
access_key_id,
secret_access_key,
owner_private_key,
) = init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file)
) = init_s3_credentials(wallet, hosting, s3_bearer_rules_file=s3_bearer_rules_file)
containers_list = list_containers(wallet, shell=client_shell)
assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
@ -65,11 +60,22 @@ class TestS3GateBase:
TestS3GateBase.wallet = wallet
def get_wallet_password(hosting: Hosting, s3_service_name: str) -> str:
service_config = hosting.get_service_config(s3_service_name)
return service_config.attributes.get("wallet_password")
@allure.step("Init S3 Credentials")
def init_s3_credentials(wallet_path: str, s3_bearer_rules_file: Optional[str] = None):
def init_s3_credentials(
wallet_path: str,
hosting: Hosting,
s3_bearer_rules_file: Optional[str] = None,
s3_service_name: str = "s3-gate01",
):
bucket = str(uuid.uuid4())
s3_bearer_rules = s3_bearer_rules_file or "robot/resources/files/s3_bearer_rules.json"
gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS)
s3_password = get_wallet_password(hosting, s3_service_name)
gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, s3_password)
cmd = (
f"{NEOFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} "

View file

@ -5,20 +5,14 @@ from typing import Dict, List, Optional
import allure
import pytest
from common import (
ASSETS_DIR,
IR_WALLET_CONFIG,
IR_WALLET_PATH,
STORAGE_WALLET_CONFIG,
STORAGE_WALLET_PATH,
WALLET_CONFIG,
WALLET_PASS,
)
import yaml
from common import ASSETS_DIR, IR_WALLET_PATH, STORAGE_WALLET_PATH, WALLET_CONFIG, WALLET_PASS
from file_helper import generate_file
from neofs_testlib.utils.wallet import init_wallet
from python_keywords.acl import EACLRole
from python_keywords.container import create_container
from python_keywords.neofs_verbs import put_object
from utility import create_wallet_config
from wellknown_acl import PUBLIC_ACL
OBJECT_COUNT = 5
@ -42,13 +36,16 @@ class Wallets:
@pytest.fixture(scope="module")
def wallets(prepare_wallet_and_deposit):
def wallets(prepare_wallet_and_deposit, hosting):
other_wallets_paths = [
os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") for _ in range(2)
]
for other_wallet_path in other_wallets_paths:
init_wallet(other_wallet_path, WALLET_PASS)
ir_wallet_config = create_wallet_config(hosting, "ir01")
storage_wallet_config = create_wallet_config(hosting, "s01")
yield Wallets(
wallets={
EACLRole.USER: [
@ -59,8 +56,8 @@ def wallets(prepare_wallet_and_deposit):
for other_wallet_path in other_wallets_paths
],
EACLRole.SYSTEM: [
Wallet(wallet_path=IR_WALLET_PATH, config_path=IR_WALLET_CONFIG),
Wallet(wallet_path=STORAGE_WALLET_PATH, config_path=STORAGE_WALLET_CONFIG),
Wallet(wallet_path=IR_WALLET_PATH, config_path=ir_wallet_config),
Wallet(wallet_path=STORAGE_WALLET_PATH, config_path=storage_wallet_config),
],
}
)

View file

@ -1,16 +1,15 @@
import logging
import os
import uuid
import logging
from typing import Optional
import allure
import pytest
import yaml
from common import (
ASSETS_DIR,
COMPLEX_OBJ_SIZE,
FREE_STORAGE,
IR_WALLET_CONFIG,
IR_WALLET_PASS,
IR_WALLET_PATH,
SIMPLE_OBJ_SIZE,
WALLET_PASS,
@ -18,6 +17,7 @@ from common import (
from epoch import tick_epoch
from file_helper import generate_file
from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from neofs_testlib.utils.wallet import init_wallet
from python_keywords.acl import (
@ -40,6 +40,7 @@ from python_keywords.storage_group import (
verify_get_storage_group,
verify_list_storage_group,
)
from utility import get_wallet_password
logger = logging.getLogger("NeoLogger")
deposit = 30
@ -73,16 +74,15 @@ class TestStorageGroup:
)
@allure.title("Test Storage Group in Private Container")
def test_storagegroup_basic_private_container(self, client_shell, object_size):
def test_storagegroup_basic_private_container(self, client_shell, object_size, hosting):
cid = create_container(self.main_wallet, shell=client_shell)
file_path = generate_file(object_size)
password = get_wallet_password(hosting=hosting, service_name="ir01")
oid = put_object(self.main_wallet, file_path, cid, shell=client_shell)
objects = [oid]
storage_group = put_storagegroup(
shell=client_shell,
wallet=self.main_wallet,
cid=cid,
objects=objects)
shell=client_shell, wallet=self.main_wallet, cid=cid, objects=objects
)
self.expect_success_for_storagegroup_operations(
shell=client_shell,
@ -104,10 +104,11 @@ class TestStorageGroup:
cid=cid,
obj_list=objects,
object_size=object_size,
hosting=hosting,
)
@allure.title("Test Storage Group in Public Container")
def test_storagegroup_basic_public_container(self, client_shell, object_size):
def test_storagegroup_basic_public_container(self, client_shell, object_size, hosting):
cid = create_container(self.main_wallet, basic_acl="public-read-write", shell=client_shell)
file_path = generate_file(object_size)
oid = put_object(self.main_wallet, file_path, cid, shell=client_shell)
@ -132,10 +133,11 @@ class TestStorageGroup:
cid=cid,
obj_list=objects,
object_size=object_size,
hosting=hosting,
)
@allure.title("Test Storage Group in Read-Only Container")
def test_storagegroup_basic_ro_container(self, client_shell, object_size):
def test_storagegroup_basic_ro_container(self, client_shell, object_size, hosting):
cid = create_container(self.main_wallet, basic_acl="public-read", shell=client_shell)
file_path = generate_file(object_size)
oid = put_object(self.main_wallet, file_path, cid, shell=client_shell)
@ -161,6 +163,7 @@ class TestStorageGroup:
cid=cid,
obj_list=objects,
object_size=object_size,
hosting=hosting,
)
@allure.title("Test Storage Group with Bearer Allow")
@ -211,8 +214,9 @@ class TestStorageGroup:
bearer=bearer_file,
)
@pytest.mark.skip
@allure.title("Test to check Storage Group lifetime")
def test_storagegroup_lifetime(self, client_shell, object_size):
def test_storagegroup_lifetime(self, client_shell, object_size, hosting):
cid = create_container(self.main_wallet, shell=client_shell)
file_path = generate_file(object_size)
oid = put_object(self.main_wallet, file_path, cid, shell=client_shell)
@ -220,7 +224,7 @@ class TestStorageGroup:
storage_group = put_storagegroup(client_shell, self.main_wallet, cid, objects, lifetime=1)
with allure.step("Tick two epochs"):
for _ in range(2):
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_storagegroup(
shell=client_shell, wallet=self.main_wallet, cid=cid, gid=storage_group
@ -303,33 +307,48 @@ class TestStorageGroup:
@staticmethod
@allure.step("Run Storage Group Operations On Systems's Behalf In RO Container")
def storagegroup_operations_by_system_ro_container(
shell: Shell, wallet: str, cid: str, obj_list: list, object_size: int
shell: Shell, wallet: str, cid: str, obj_list: list, object_size: int, hosting: Hosting
):
"""
In this func we create a Storage Group on Inner Ring's key behalf
and include an Object created on behalf of some user. We expect
that System key is granted to make all operations except PUT and DELETE.
"""
password = get_wallet_password(hosting=hosting, service_name="ir01")
ir_wallet_config = os.path.join(os.getcwd(), "ir_wallet_config.yml")
with open(ir_wallet_config, "w") as file:
yaml.dump({"password": password}, file)
if not FREE_STORAGE:
deposit = 30
transfer_gas(
shell=shell,
amount=deposit + 1,
wallet_to_path=IR_WALLET_PATH,
wallet_to_password=IR_WALLET_PASS,
wallet_to_password=password,
)
deposit_gas(
shell=shell,
amount=deposit,
wallet_from_path=IR_WALLET_PATH,
wallet_from_password=IR_WALLET_PASS,
wallet_from_password=password,
)
storage_group = put_storagegroup(shell, wallet, cid, obj_list)
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
put_storagegroup(shell, IR_WALLET_PATH, cid, obj_list, wallet_config=IR_WALLET_CONFIG)
verify_list_storage_group(
IR_WALLET_PATH, cid, storage_group, wallet_config=IR_WALLET_CONFIG
put_storagegroup(
shell=shell,
wallet=IR_WALLET_PATH,
cid=cid,
objects=obj_list,
wallet_config=ir_wallet_config,
)
verify_list_storage_group(
shell=shell,
wallet=IR_WALLET_PATH,
cid=cid,
gid=storage_group,
wallet_config=ir_wallet_config,
)
verify_get_storage_group(
shell=shell,
wallet=IR_WALLET_PATH,
@ -337,7 +356,7 @@ class TestStorageGroup:
gid=storage_group,
obj_list=obj_list,
object_size=object_size,
wallet_config=IR_WALLET_CONFIG,
wallet_config=ir_wallet_config,
)
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
delete_storagegroup(
@ -345,5 +364,5 @@ class TestStorageGroup:
wallet=IR_WALLET_PATH,
cid=cid,
gid=storage_group,
wallet_config=IR_WALLET_CONFIG,
wallet_config=ir_wallet_config,
)

View file

@ -29,6 +29,7 @@ from python_keywords.object_access import (
can_put_object,
can_search_object,
)
from utility import create_wallet_config
from wellknown_acl import PUBLIC_ACL
@ -230,13 +231,21 @@ class TestEACLContainer:
drop_object(hosting, node_name=[*NEOFS_NETMAP_DICT][0], cid=cid, oid=oid)
storage_wallet_path = NEOFS_NETMAP_DICT[[*NEOFS_NETMAP_DICT][0]]["wallet_path"]
storage_wallet_conf = create_wallet_config(hosting, [*NEOFS_NETMAP_DICT][0])
with allure.step("Wait for dropped object replicated"):
wait_object_replication_on_nodes(
storage_wallet_path, cid, oid, self.NODE_COUNT, shell=client_shell
storage_wallet_path,
cid,
oid,
self.NODE_COUNT,
shell=client_shell,
wallet_config=storage_wallet_conf,
)
@allure.title("Testcase to validate NeoFS system operations with extended ACL")
def test_extended_actions_system(self, wallets, client_shell, eacl_container_with_objects):
def test_extended_actions_system(
self, wallets, client_shell, eacl_container_with_objects, hosting
):
user_wallet = wallets.get_wallet()
ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2]

View file

@ -1,5 +1,6 @@
import logging
import os
import re
import shutil
import uuid
from datetime import datetime
@ -10,16 +11,27 @@ import yaml
from binary_version_helper import get_local_binaries_versions, get_remote_binaries_versions
from common import ASSETS_DIR, FREE_STORAGE, HOSTING_CONFIG_FILE, NEOFS_NETMAP_DICT, WALLET_PASS
from env_properties import save_env_properties
from file_helper import sanitize_for_file_name
from neofs_testlib.hosting import Hosting
from neofs_testlib.reporter import AllureHandler, get_reporter
from neofs_testlib.shell import LocalShell, Shell
from neofs_testlib.utils.wallet import init_wallet
from payment_neogo import deposit_gas, transfer_gas
from pytest import FixtureRequest
from python_keywords.node_management import node_healthcheck
logger = logging.getLogger("NeoLogger")
def pytest_collection_modifyitems(items):
# Make network tests last based on @pytest.mark.node_mgmt
def priority(item: pytest.Item) -> int:
is_node_mgmt_test = item.get_closest_marker("node_mgmt")
return 0 if not is_node_mgmt_test else 1
items.sort(key=lambda item: priority(item))
@pytest.fixture(scope="session")
def configure_testlib():
get_reporter().register_handler(AllureHandler())
@ -72,6 +84,24 @@ def prepare_tmp_dir():
shutil.rmtree(full_path)
@pytest.fixture(scope="function", autouse=True)
@allure.title("Analyze logs")
def analyze_logs(prepare_tmp_dir: str, hosting: Hosting, request: FixtureRequest):
start_time = datetime.utcnow()
yield
end_time = datetime.utcnow()
# Skip tests where we expect failures in logs
if request.node.get_closest_marker("no_log_analyze"):
with allure.step("Skip analyze logs due to no_log_analyze mark"):
return
# Test name may exceed os NAME_MAX (255 bytes), so we use timestamp instead
logs_dir = os.path.join(prepare_tmp_dir, f"logs_{sanitize_for_file_name(str(start_time))}")
dump_logs(hosting, logs_dir, start_time, end_time)
check_logs(logs_dir)
@pytest.fixture(scope="session", autouse=True)
@allure.title("Collect logs")
def collect_logs(prepare_tmp_dir, hosting: Hosting):
@ -81,15 +111,8 @@ def collect_logs(prepare_tmp_dir, hosting: Hosting):
# Dump logs to temp directory (because they might be too large to keep in RAM)
logs_dir = os.path.join(prepare_tmp_dir, "logs")
os.makedirs(logs_dir)
for host in hosting.hosts:
host.dump_logs(logs_dir, since=start_time, until=end_time)
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")
dump_logs(hosting, logs_dir, start_time, end_time)
attach_logs(logs_dir)
@pytest.fixture(scope="session", autouse=True)
@ -108,6 +131,10 @@ def run_health_check(collect_logs, hosting: Hosting):
@pytest.fixture(scope="session")
@allure.title("Prepare wallet and deposit")
def prepare_wallet_and_deposit(client_shell, prepare_tmp_dir):
return create_wallet_with_gas(client_shell, prepare_tmp_dir)
def create_wallet_with_gas(client_shell, prepare_tmp_dir):
wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(wallet_path, WALLET_PASS)
allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON)
@ -128,3 +155,39 @@ def prepare_wallet_and_deposit(client_shell, prepare_tmp_dir):
)
return wallet_path
def check_logs(logs_dir: str):
problem_pattern = r"\Wpanic\W|\Woom\W"
log_file_paths = []
for directory_path, _, file_names in os.walk(logs_dir):
log_file_paths += [
os.path.join(directory_path, file_name)
for file_name in file_names
if re.match(r"\.(txt|log)", os.path.splitext(file_name)[-1], flags=re.IGNORECASE)
]
logs_with_problem = []
for file_path in log_file_paths:
with open(file_path, "r") as log_file:
if re.search(problem_pattern, log_file.read(), flags=re.IGNORECASE):
attach_logs(logs_dir)
logs_with_problem.append(file_path)
if logs_with_problem:
raise pytest.fail(f"System logs {', '.join(logs_with_problem)} contain critical errors")
def dump_logs(hosting: Hosting, logs_dir: str, since: datetime, until: datetime) -> None:
# Dump logs to temp directory (because they might be too large to keep in RAM)
os.makedirs(logs_dir)
for host in hosting.hosts:
host.dump_logs(logs_dir, since=since, until=until)
def attach_logs(logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")

View file

@ -3,6 +3,7 @@ import json
import allure
import pytest
from epoch import tick_epoch
from neofs_testlib.hosting import Hosting
from python_keywords.container import (
create_container,
delete_container,
@ -18,7 +19,7 @@ from wellknown_acl import PRIVATE_ACL_F
@pytest.mark.parametrize("name", ["", "test-container"], ids=["No name", "Set particular name"])
@pytest.mark.sanity
@pytest.mark.container
def test_container_creation(client_shell, prepare_wallet_and_deposit, name):
def test_container_creation(client_shell, prepare_wallet_and_deposit, name, hosting):
scenario_title = f"with name {name}" if name else "without name"
allure.dynamic.title(f"User can create container {scenario_title}")
@ -58,14 +59,14 @@ def test_container_creation(client_shell, prepare_wallet_and_deposit, name):
with allure.step("Delete container and check it was deleted"):
delete_container(wallet, cid, shell=client_shell)
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
wait_for_container_deletion(wallet, cid, shell=client_shell)
@allure.title("Parallel container creation and deletion")
@pytest.mark.sanity
@pytest.mark.container
def test_container_creation_deletion_parallel(client_shell, prepare_wallet_and_deposit):
def test_container_creation_deletion_parallel(client_shell, prepare_wallet_and_deposit, hosting):
containers_count = 3
wallet = prepare_wallet_and_deposit
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
@ -92,5 +93,5 @@ def test_container_creation_deletion_parallel(client_shell, prepare_wallet_and_d
with allure.step("Delete containers and check they were deleted"):
for cid in cids:
delete_container(wallet, cid, shell=client_shell)
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
wait_for_container_deletion(wallet, cid, shell=client_shell)

View file

@ -3,14 +3,12 @@ from enum import Enum
import allure
import pytest
from common import (
HTTP_GATE_SERVICE_NAME_REGEX,
LOAD_NODE_SSH_PRIVATE_KEY_PATH,
LOAD_NODE_SSH_USER,
LOAD_NODES,
STORAGE_NODE_SERVICE_NAME_REGEX,
HTTP_GATE_SERVICE_NAME_REGEX,
)
from neofs_testlib.hosting import Hosting
from k6 import LoadParams
from load import (
clear_cache_and_data,
@ -18,6 +16,7 @@ from load import (
multi_node_k6_run,
prepare_k6_instances,
)
from neofs_testlib.hosting import Hosting
class LoadTime(Enum):

View file

@ -11,7 +11,6 @@ from common import (
NEOFS_CONTRACT_CACHE_TIMEOUT,
NEOFS_NETMAP_DICT,
STORAGE_RPC_ENDPOINT_1,
STORAGE_WALLET_PASS,
)
from data_formatters import get_wallet_public_key
from epoch import tick_epoch
@ -38,7 +37,12 @@ from python_keywords.node_management import (
stop_nodes,
)
from storage_policy import get_nodes_with_object, get_simple_object_copies
from utility import parse_time, placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
from utility import (
get_wallet_password,
parse_time,
placement_policy_from_container,
wait_for_gc_pass_on_storage_nodes,
)
from wellknown_acl import PUBLIC_ACL
logger = logging.getLogger("NeoLogger")
@ -109,12 +113,12 @@ def return_nodes(shell: Shell, hosting: Hosting, alive_node: Optional[str] = Non
sleep(parse_time(MORPH_BLOCK_TIME))
for __attempt in range(3):
try:
tick_epoch(shell=shell)
tick_epoch(shell=shell, hosting=hosting)
break
except RuntimeError:
sleep(3)
check_node_in_map(node, shell=shell, alive_node=alive_node)
check_node_in_map(node, shell=shell, hosting=hosting, alive_node=alive_node)
@allure.title("Add one node to cluster")
@ -141,7 +145,7 @@ def test_add_nodes(
)
alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != additional_node])
check_node_in_map(additional_node, shell=client_shell, alive_node=alive_node)
check_node_in_map(additional_node, shell=client_shell, hosting=hosting, alive_node=alive_node)
# Add node to recovery list before messing with it
check_nodes.append(additional_node)
@ -197,10 +201,11 @@ def test_nodes_management(prepare_tmp_dir, client_shell, hosting: Hosting):
# Calculate public key that identifies node in netmap
random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]["wallet_path"]
random_node_netmap_key = get_wallet_public_key(random_node_wallet_path, STORAGE_WALLET_PASS)
node_wallet_password = get_wallet_password(hosting=hosting, service_name=random_node)
random_node_netmap_key = get_wallet_public_key(random_node_wallet_path, node_wallet_password)
with allure.step("Check node {random_node} is in netmap"):
snapshot = get_netmap_snapshot(node_name=alive_node, shell=client_shell)
snapshot = get_netmap_snapshot(node_name=alive_node, shell=client_shell, hosting=hosting)
assert random_node_netmap_key in snapshot, f"Expected node {random_node} in netmap"
with allure.step("Run health check for all storage nodes"):
@ -212,24 +217,24 @@ def test_nodes_management(prepare_tmp_dir, client_shell, hosting: Hosting):
node_set_status(hosting, random_node, status="offline")
sleep(parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
with allure.step(f"Check node {random_node} went to offline"):
health_check = node_healthcheck(hosting, random_node)
assert health_check.health_status == "READY" and health_check.network_status == "OFFLINE"
snapshot = get_netmap_snapshot(node_name=alive_node, shell=client_shell)
snapshot = get_netmap_snapshot(node_name=alive_node, shell=client_shell, hosting=hosting)
assert random_node_netmap_key not in snapshot, f"Expected node {random_node} not in netmap"
with allure.step(f"Check node {random_node} went to online"):
node_set_status(hosting, random_node, status="online")
sleep(parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
with allure.step(f"Check node {random_node} went to online"):
health_check = node_healthcheck(hosting, random_node)
assert health_check.health_status == "READY" and health_check.network_status == "ONLINE"
snapshot = get_netmap_snapshot(node_name=alive_node, shell=client_shell)
snapshot = get_netmap_snapshot(node_name=alive_node, shell=client_shell, hosting=hosting)
assert random_node_netmap_key in snapshot, f"Expected node {random_node} in netmap"
@ -372,7 +377,7 @@ def test_replication(
wait_for_expected_object_copies(client_shell, wallet, cid, oid)
start_nodes(hosting, stopped_nodes)
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
for node_name in node_names:
wait_for_node_go_online(hosting, node_name)
@ -508,13 +513,13 @@ def wait_for_node_to_be_ready(hosting: Hosting, node_name: str) -> None:
@allure.step("Wait for {expected_copies} object copies in the wallet")
def wait_for_expected_object_copies(
shell: Shell, wallet: str, cid: str, oid: str, expected_copies: int = 2
shell: Shell, wallet: str, cid: str, oid: str, hosting: Hosting, expected_copies: int = 2
) -> None:
for i in range(2):
copies = get_simple_object_copies(wallet, cid, oid)
if copies == expected_copies:
break
tick_epoch(shell=shell)
tick_epoch(shell=shell, hosting=hosting)
sleep(parse_time(NEOFS_CONTRACT_CACHE_TIMEOUT))
else:
raise AssertionError(f"There are no {expected_copies} copies during time")

View file

@ -11,6 +11,7 @@ from container import create_container
from epoch import tick_epoch
from file_helper import generate_file, get_file_content, get_file_hash
from grpc_responses import OBJECT_ALREADY_REMOVED, OUT_OF_RANGE, error_matches_status
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from pytest import FixtureRequest
from python_keywords.neofs_verbs import (
@ -94,7 +95,7 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
return file_ranges_to_test
def delete_objects(storage_objects: list, client_shell: Shell) -> None:
def delete_objects(storage_objects: list, client_shell: Shell, hosting: Hosting) -> None:
with allure.step("Delete objects"):
for storage_object in storage_objects:
storage_object.tombstone = delete_object(
@ -108,7 +109,7 @@ def delete_objects(storage_objects: list, client_shell: Shell) -> None:
shell=client_shell,
)
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
sleep(CLEANUP_TIMEOUT)
with allure.step("Get objects and check errors"):
@ -129,7 +130,7 @@ def delete_objects(storage_objects: list, client_shell: Shell) -> None:
scope="session",
)
def storage_objects(
prepare_wallet_and_deposit: str, client_shell: Shell, request: FixtureRequest
prepare_wallet_and_deposit: str, client_shell: Shell, request: FixtureRequest, hosting: Hosting
) -> list[StorageObjectInfo]:
wallet = prepare_wallet_and_deposit
# Separate containers for complex/simple objects to avoid side-effects
@ -163,7 +164,7 @@ def storage_objects(
yield storage_objects
# Teardown after all tests done with current param
delete_objects(storage_objects, client_shell)
delete_objects(storage_objects, client_shell, hosting)
@allure.title("Validate object storage policy by native API")
@ -290,7 +291,11 @@ def test_search_object_api(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"]
)
def test_object_search_should_return_tombstone_items(
prepare_wallet_and_deposit: str, client_shell: Shell, request: FixtureRequest, object_size: int
prepare_wallet_and_deposit: str,
client_shell: Shell,
request: FixtureRequest,
object_size: int,
hosting: Hosting,
):
"""
Validate object search with removed items
@ -321,7 +326,7 @@ def test_object_search_should_return_tombstone_items(
assert result == [storage_object.oid]
with allure.step("Delete file"):
delete_objects([storage_object], client_shell)
delete_objects([storage_object], client_shell, hosting)
with allure.step("Search deleted object with --root"):
# Root Search object should return nothing

View file

@ -7,6 +7,7 @@ from container import create_container
from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from grpc_responses import OBJECT_NOT_FOUND
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from pytest import FixtureRequest
from python_keywords.neofs_verbs import get_object, put_object
@ -22,7 +23,11 @@ logger = logging.getLogger("NeoLogger")
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"]
)
def test_object_api_lifetime(
prepare_wallet_and_deposit: str, client_shell: Shell, request: FixtureRequest, object_size: int
prepare_wallet_and_deposit: str,
client_shell: Shell,
request: FixtureRequest,
object_size: int,
hosting: Hosting,
):
"""
Test object deleted after expiration epoch.
@ -42,7 +47,7 @@ def test_object_api_lifetime(
with allure.step("Tick two epochs"):
for _ in range(2):
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
# Wait for GC, because object with expiration is counted as alive until GC removes it
wait_for_gc_pass_on_storage_nodes()

View file

@ -1,13 +1,15 @@
import logging
import os
import uuid
import allure
import pytest
import yaml
from common import ASSETS_DIR, FREE_STORAGE, NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG, WALLET_PASS
from common import FREE_STORAGE, NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG, WALLET_PASS
from neofs_testlib.cli import NeofsCli
from neofs_testlib.utils.wallet import get_last_address_from_wallet, init_wallet
from neofs_testlib.shell import CommandResult, Shell
from neofs_testlib.utils.wallet import get_last_address_from_wallet
from testsuites.conftest import create_wallet_with_gas
logger = logging.getLogger("NeoLogger")
DEPOSIT_AMOUNT = 30
@ -16,51 +18,30 @@ DEPOSIT_AMOUNT = 30
@pytest.mark.payments
@pytest.mark.skipif(FREE_STORAGE, reason="Test only works on public network with paid storage")
class TestBalanceAccounting:
@pytest.fixture(autouse=True)
def prepare_two_wallets(self, prepare_wallet_and_deposit):
self.user_wallet = prepare_wallet_and_deposit
self.address = get_last_address_from_wallet(self.user_wallet, WALLET_PASS)
another_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(another_wallet, WALLET_PASS)
self.another_address = get_last_address_from_wallet(another_wallet, WALLET_PASS)
@pytest.fixture(scope="class")
def main_wallet(self, client_shell, prepare_tmp_dir) -> str:
return create_wallet_with_gas(client_shell=client_shell, prepare_tmp_dir=prepare_tmp_dir)
@allure.title("Test balance request with wallet and address")
def test_balance_wallet_address(self, client_shell):
cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
result = cli.accounting.balance(
wallet=self.user_wallet,
rpc_endpoint=NEOFS_ENDPOINT,
address=self.address,
)
assert int(result.stdout.rstrip()) == DEPOSIT_AMOUNT
@pytest.fixture(scope="class")
def other_wallet(self, client_shell, prepare_tmp_dir) -> str:
return create_wallet_with_gas(client_shell=client_shell, prepare_tmp_dir=prepare_tmp_dir)
@allure.title("Test balance request with wallet only")
def test_balance_wallet(self, client_shell):
cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
result = cli.accounting.balance(wallet=self.user_wallet, rpc_endpoint=NEOFS_ENDPOINT)
assert int(result.stdout.rstrip()) == DEPOSIT_AMOUNT
@pytest.fixture(scope="class")
def cli(self, client_shell: Shell) -> NeofsCli:
return NeofsCli(client_shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
@allure.title("Test balance request with wallet and wrong address")
def test_balance_wrong_address(self, client_shell):
with pytest.raises(Exception, match="address option must be specified and valid"):
cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
cli.accounting.balance(
wallet=self.user_wallet,
rpc_endpoint=NEOFS_ENDPOINT,
address=self.another_address,
@allure.step("Check deposit amount")
def check_amount(self, result: CommandResult) -> None:
amount_str = result.stdout.rstrip()
try:
amount = int(amount_str)
except Exception as ex:
pytest.fail(
f"Amount parse error, should be parsable as int({DEPOSIT_AMOUNT}), but given {amount_str}: {ex}"
)
@allure.title("Test balance request with config file")
def test_balance_api(self, prepare_tmp_dir, client_shell):
config_file = self.write_api_config(
config_dir=prepare_tmp_dir, endpoint=NEOFS_ENDPOINT, wallet=self.user_wallet
)
logger.info(f"Config with API endpoint: {config_file}")
cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, config_file=config_file)
result = cli.accounting.balance()
assert int(result.stdout.rstrip()) == DEPOSIT_AMOUNT
assert amount == DEPOSIT_AMOUNT
@staticmethod
@allure.step("Write config with API endpoint")
@ -76,3 +57,39 @@ class TestBalanceAccounting:
with open(api_config_file, "w") as file:
yaml.dump(api_config, file)
return api_config_file
@allure.title("Test balance request with wallet and address")
def test_balance_wallet_address(self, main_wallet: str, cli: NeofsCli):
result = cli.accounting.balance(
wallet=main_wallet,
rpc_endpoint=NEOFS_ENDPOINT,
address=get_last_address_from_wallet(main_wallet, WALLET_PASS),
)
self.check_amount(result)
@allure.title("Test balance request with wallet only")
def test_balance_wallet(self, main_wallet: str, cli: NeofsCli):
result = cli.accounting.balance(wallet=main_wallet, rpc_endpoint=NEOFS_ENDPOINT)
self.check_amount(result)
@allure.title("Test balance request with wallet and wrong address")
def test_balance_wrong_address(self, main_wallet: str, other_wallet: str, cli: NeofsCli):
with pytest.raises(Exception, match="address option must be specified and valid"):
cli.accounting.balance(
wallet=main_wallet,
rpc_endpoint=NEOFS_ENDPOINT,
address=get_last_address_from_wallet(other_wallet, WALLET_PASS),
)
@allure.title("Test balance request with config file")
def test_balance_api(self, prepare_tmp_dir: str, main_wallet: str, client_shell: Shell):
config_file = self.write_api_config(
config_dir=prepare_tmp_dir, endpoint=NEOFS_ENDPOINT, wallet=main_wallet
)
logger.info(f"Config with API endpoint: {config_file}")
cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, config_file=config_file)
result = cli.accounting.balance()
self.check_amount(result)

View file

@ -66,7 +66,7 @@ class TestS3Gate(TestS3GateBase):
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket)
@allure.title("Test S3 Bucket API")
def test_s3_buckets(self, client_shell):
def test_s3_buckets(self, client_shell, hosting):
"""
Test base S3 Bucket API (Create/List/Head/Delete).
"""
@ -110,7 +110,7 @@ class TestS3Gate(TestS3GateBase):
with allure.step(f"Delete empty bucket {bucket_2}"):
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket_2)
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
with allure.step(f"Check bucket {bucket_2} deleted"):
with pytest.raises(Exception, match=r".*Not Found.*"):
@ -126,7 +126,7 @@ class TestS3Gate(TestS3GateBase):
with allure.step(f"Delete bucket {bucket_1}"):
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket_1)
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
with allure.step(f"Check bucket {bucket_1} deleted"):
with pytest.raises(Exception, match=r".*Not Found.*"):

View file

@ -8,9 +8,9 @@ import pytest
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS
from data_formatters import get_wallet_public_key
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
from neofs_testlib.utils.wallet import init_wallet
from python_keywords.payment_neogo import deposit_gas, transfer_gas
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, set_bucket_versioning
from neofs_testlib.utils.wallet import init_wallet
from steps import s3_gate_bucket, s3_gate_object
from steps.aws_cli_client import AwsCliClient

View file

@ -9,6 +9,7 @@ from common import COMPLEX_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from python_keywords.http_gate import (
get_via_http_curl,
@ -147,7 +148,7 @@ class TestHttpGate:
self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes)
@allure.title("Test Expiration-Epoch in HTTP header")
def test_expiration_epoch_in_http(self, client_shell):
def test_expiration_epoch_in_http(self, client_shell, hosting):
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL
)
@ -170,7 +171,7 @@ class TestHttpGate:
get_via_http_gate(cid=cid, oid=oid)
for expired_objects, not_expired_objects in [(oids[:1], oids[1:]), (oids[:2], oids[2:])]:
tick_epoch(shell=client_shell)
tick_epoch(shell=client_shell, hosting=hosting)
# Wait for GC, because object with expiration is counted as alive until GC removes it
wait_for_gc_pass_on_storage_nodes()

View file

@ -1,8 +1,68 @@
robotframework==4.1.2
requests==2.25.1
pexpect==4.8.0
aiodns==3.0.0
aiohttp==3.7.4.post0
aioresponses==0.7.2
allure-pytest==2.9.45
allure-python-commons==2.9.45
async-timeout==3.0.1
asynctest==0.13.0
attrs==21.4.0
base58==2.1.0
bitarray==2.3.4
black==22.8.0
boto3==1.16.33
docker==4.4.0
botocore==1.19.33
urllib3==1.26.3
base58==1.0.3
certifi==2022.5.18
cffi==1.15.0
chardet==4.0.0
charset-normalizer==2.0.12
coverage==6.3.3
docker==4.4.0
docutils==0.17.1
Events==0.4
flake8==4.0.1
idna==3.3
iniconfig==1.1.1
isort==5.10.1
jmespath==0.10.0
jsonschema==4.5.1
lz4==3.1.3
mccabe==0.6.1
mmh3==3.0.0
multidict==6.0.2
mypy==0.950
mypy-extensions==0.4.3
neo-mamba==0.10.0
neo3crypto==0.2.1
neo3vm==0.9.0
neo3vm-stubs==0.9.0
neofs-testlib==0.4.0
netaddr==0.8.0
orjson==3.6.8
packaging==21.3
paramiko==2.10.3
pexpect==4.8.0
pluggy==1.0.0
pre-commit==2.20.0
ptyprocess==0.7.0
py==1.11.0
pybiginteger==1.2.6
pybiginteger-stubs==1.2.6
pycares==4.1.2
pycodestyle==2.8.0
pycparser==2.21
pycryptodome==3.11.0
pyflakes==2.4.0
pyparsing==3.0.9
pyrsistent==0.18.1
pytest==7.1.2
python-dateutil==2.8.2
requests==2.28.0
robotframework==4.1.2
s3transfer==0.3.7
six==1.16.0
tenacity==8.0.1
tomli==2.0.1
typing-extensions==4.2.0
urllib3==1.26.9
websocket-client==1.3.2
yarl==1.7.2

View file

@ -4,7 +4,6 @@ from time import sleep
import allure
from common import (
IR_WALLET_PASS,
IR_WALLET_PATH,
MAINNET_BLOCK_TIME,
MORPH_ENDPOINT,
@ -13,10 +12,11 @@ from common import (
NEOGO_EXECUTABLE,
)
from neofs_testlib.cli import NeofsAdm, NeoGo
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from neofs_testlib.utils.wallet import get_last_address_from_wallet
from payment_neogo import get_contract_hash
from utility import parse_time
from utility import get_wallet_password, parse_time
logger = logging.getLogger("NeoLogger")
@ -33,7 +33,7 @@ def get_epoch(shell: Shell):
@allure.step("Tick Epoch")
def tick_epoch(shell: Shell):
def tick_epoch(shell: Shell, hosting: Hosting):
if NEOFS_ADM_EXEC and NEOFS_ADM_CONFIG_PATH:
# If neofs-adm is available, then we tick epoch with it (to be consistent with UAT tests)
neofsadm = NeofsAdm(
@ -44,13 +44,13 @@ def tick_epoch(shell: Shell):
# Otherwise we tick epoch using transaction
cur_epoch = get_epoch(shell)
ir_address = get_last_address_from_wallet(IR_WALLET_PATH, IR_WALLET_PASS)
ir_wallet_password = get_wallet_password(hosting, "ir01")
ir_address = get_last_address_from_wallet(IR_WALLET_PATH, ir_wallet_password)
neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
neogo.contract.invokefunction(
wallet=IR_WALLET_PATH,
wallet_password=IR_WALLET_PASS,
wallet_password=ir_wallet_password,
scripthash=get_contract_hash("netmap.neofs", shell=shell),
method="newEpoch",
arguments=f"int:{cur_epoch + 1}",

View file

@ -20,12 +20,15 @@ def wait_object_replication_on_nodes(
expected_copies: int,
shell: Shell,
excluded_nodes: Optional[list[str]] = None,
wallet_config: Optional[str] = None,
) -> list[str]:
excluded_nodes = excluded_nodes or []
sleep_interval, attempts = 15, 20
nodes = []
for __attempt in range(attempts):
nodes = get_nodes_with_object(wallet, cid, oid, shell=shell, skip_nodes=excluded_nodes)
nodes = get_nodes_with_object(
wallet, cid, oid, shell=shell, skip_nodes=excluded_nodes, wallet_config=wallet_config
)
if len(nodes) >= expected_copies:
return nodes
sleep(sleep_interval)

View file

@ -6,19 +6,13 @@ from dataclasses import dataclass
from typing import Optional
import allure
from common import (
MORPH_BLOCK_TIME,
NEOFS_CLI_EXEC,
NEOFS_NETMAP_DICT,
STORAGE_WALLET_CONFIG,
STORAGE_WALLET_PASS,
)
from common import MORPH_BLOCK_TIME, NEOFS_CLI_EXEC, NEOFS_NETMAP_DICT
from data_formatters import get_wallet_public_key
from epoch import tick_epoch
from neofs_testlib.cli import NeofsCli
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from utility import parse_time
from utility import create_wallet_config, get_wallet_password, parse_time
logger = logging.getLogger("NeoLogger")
@ -104,7 +98,7 @@ def node_set_status(hosting: Hosting, node_name: str, status: str, retries: int
@allure.step("Get netmap snapshot")
def get_netmap_snapshot(node_name: str, shell: Shell) -> str:
def get_netmap_snapshot(node_name: str, shell: Shell, hosting: Hosting) -> str:
"""
The function returns string representation of netmap snapshot.
Args:
@ -113,7 +107,8 @@ def get_netmap_snapshot(node_name: str, shell: Shell) -> str:
string representation of netmap
"""
node_info = NEOFS_NETMAP_DICT[node_name]
cli = NeofsCli(shell, NEOFS_CLI_EXEC, config_file=STORAGE_WALLET_CONFIG)
storage_wallet_config = create_wallet_config(hosting, node_name)
cli = NeofsCli(shell, NEOFS_CLI_EXEC, config_file=storage_wallet_config)
return cli.netmap.snapshot(
rpc_endpoint=node_info["rpc"],
wallet=node_info["wallet_path"],
@ -169,14 +164,15 @@ def exclude_node_from_network_map(
hosting: Hosting, node_to_exclude: str, alive_node: str, shell: Shell
) -> None:
node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"]
node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS)
node_wallet_password = get_wallet_password(hosting=hosting, service_name=node_to_exclude)
node_netmap_key = get_wallet_public_key(node_wallet_path, node_wallet_password)
node_set_status(hosting, node_to_exclude, status="offline")
time.sleep(parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell=shell)
tick_epoch(shell=shell, hosting=hosting)
snapshot = get_netmap_snapshot(node_name=alive_node, shell=shell)
snapshot = get_netmap_snapshot(node_name=alive_node, shell=shell, hosting=hosting)
assert (
node_netmap_key not in snapshot
), f"Expected node with key {node_netmap_key} not in network map"
@ -188,21 +184,25 @@ def include_node_to_network_map(
) -> None:
node_set_status(hosting, node_to_include, status="online")
time.sleep(parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell=shell)
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
tick_epoch(shell=shell, hosting=hosting)
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
check_node_in_map(node_to_include, shell, alive_node)
check_node_in_map(node_to_include, shell, hosting, alive_node)
@allure.step("Check node {node_name} in network map")
def check_node_in_map(node_name: str, shell: Shell, alive_node: Optional[str] = None) -> None:
def check_node_in_map(
node_name: str, shell: Shell, hosting: Hosting, alive_node: Optional[str] = None
) -> None:
alive_node = alive_node or node_name
node_wallet_path = NEOFS_NETMAP_DICT[node_name]["wallet_path"]
node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS)
node_wallet_password = get_wallet_password(hosting=hosting, service_name=node_name)
node_netmap_key = get_wallet_public_key(node_wallet_path, node_wallet_password)
logger.info(f"Node {node_name} netmap key: {node_netmap_key}")
snapshot = get_netmap_snapshot(node_name=alive_node, shell=shell)
snapshot = get_netmap_snapshot(node_name=alive_node, shell=shell, hosting=hosting)
assert node_netmap_key in snapshot, f"Expected node with key {node_netmap_key} in network map"

View file

@ -7,9 +7,9 @@ import uuid
import allure
import json_transformers
from common import ASSETS_DIR, NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG
from neo3 import wallet
from neofs_testlib.cli import NeofsCli
from neofs_testlib.shell import Shell
from neo3 import wallet
logger = logging.getLogger("NeoLogger")

View file

@ -97,7 +97,12 @@ def get_complex_object_copies(wallet: str, cid: str, oid: str, shell: Shell) ->
@allure.step("Get Nodes With Object")
def get_nodes_with_object(
wallet: str, cid: str, oid: str, shell: Shell, skip_nodes: Optional[list[str]] = None
wallet: str,
cid: str,
oid: str,
shell: Shell,
skip_nodes: Optional[list[str]] = None,
wallet_config: Optional[str] = None,
) -> list[str]:
"""
The function returns list of nodes which store
@ -120,7 +125,13 @@ def get_nodes_with_object(
for node in nodes_to_search:
try:
res = neofs_verbs.head_object(
wallet, cid, oid, shell=shell, endpoint=node, is_direct=True
wallet,
cid,
oid,
shell=shell,
endpoint=node,
is_direct=True,
wallet_config=wallet_config,
)
if res is not None:
logger.info(f"Found object {oid} on node {node}")

View file

@ -64,7 +64,7 @@ STORAGE_WALLET_PATH_4 = os.getenv(
"STORAGE_WALLET_PATH_4", os.path.join(DEVENV_PATH, "services", "storage", "wallet04.json")
)
STORAGE_WALLET_PATH = STORAGE_WALLET_PATH_1
STORAGE_WALLET_PASS = os.getenv("STORAGE_WALLET_PASS", "")
NEOFS_NETMAP_DICT = {
"s01": {
@ -107,7 +107,6 @@ MAINNET_SINGLE_ADDR = os.getenv("MAINNET_SINGLE_ADDR", "NfgHwwTi3wHAS8aFAN243C5v
MAINNET_WALLET_PASS = os.getenv("MAINNET_WALLET_PASS", "one")
IR_WALLET_PATH = os.getenv("IR_WALLET_PATH", os.path.join(DEVENV_PATH, "services", "ir", "az.json"))
IR_WALLET_PASS = os.getenv("IR_WALLET_PASS", "one")
S3_GATE_WALLET_PATH = os.getenv(
"S3_GATE_WALLET_PATH", os.path.join(DEVENV_PATH, "services", "s3_gate", "wallet.json")
@ -130,15 +129,3 @@ S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d"
WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")
with open(WALLET_CONFIG, "w") as file:
yaml.dump({"password": WALLET_PASS}, file)
STORAGE_WALLET_CONFIG = os.path.join(os.getcwd(), "storage_wallet_config.yml")
with open(STORAGE_WALLET_CONFIG, "w") as file:
yaml.dump({"password": STORAGE_WALLET_PASS}, file)
MAINNET_WALLET_CONFIG = os.path.join(os.getcwd(), "mainnet_wallet_config.yml")
with open(MAINNET_WALLET_CONFIG, "w") as file:
yaml.dump({"password": MAINNET_WALLET_PASS}, file)
IR_WALLET_CONFIG = os.path.join(os.getcwd(), "ir_wallet_config.yml")
with open(IR_WALLET_CONFIG, "w") as file:
yaml.dump({"password": IR_WALLET_PASS}, file)