From 3f6ba19a8be1f5c268e50f571b41644c2a53d87e Mon Sep 17 00:00:00 2001 From: Elizaveta Chichindaeva Date: Wed, 17 Aug 2022 16:10:02 +0300 Subject: [PATCH] Tests: Storagegroup tests into pytest Tests for Storagegroups rewritten in pytest Signed-off-by: Elizaveta Chichindaeva --- pytest_tests/pytest.ini | 1 + .../acl/storage_group/test_storagegroup.py | 230 ++++++++++++++++++ .../lib/python_keywords/storage_group.py | 221 +++++++++++------ 3 files changed, 378 insertions(+), 74 deletions(-) create mode 100644 pytest_tests/testsuites/acl/storage_group/test_storagegroup.py diff --git a/pytest_tests/pytest.ini b/pytest_tests/pytest.ini index 7b4180e4..aab93e7e 100644 --- a/pytest_tests/pytest.ini +++ b/pytest_tests/pytest.ini @@ -18,6 +18,7 @@ markers = long: long tests (with long execution time) node_mgmt: neofs control commands acl: tests for basic and extended ACL + storage_group: tests for storage groups failover: tests for system recovery after a failure failover_panic: tests for system recovery after panic reboot of a node failover_net: tests for network failure diff --git a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py new file mode 100644 index 00000000..2f8ff22e --- /dev/null +++ b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py @@ -0,0 +1,230 @@ +import logging + +import pytest +from common import ( + COMPLEX_OBJ_SIZE, + IR_WALLET_CONFIG, + IR_WALLET_PASS, + IR_WALLET_PATH, + SIMPLE_OBJ_SIZE, +) +from epoch import tick_epoch +from typing import Optional +from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND +from python_keywords.acl import ( + EACLAccess, + EACLOperation, + EACLRole, + EACLRule, + create_eacl, + form_bearertoken_file, + set_eacl, +) +from python_keywords.container import create_container +from python_keywords.neofs_verbs import put_object +from python_keywords.payment_neogo import neofs_deposit, transfer_mainnet_gas +from python_keywords.storage_group import ( + delete_storagegroup, + get_storagegroup, + list_storagegroup, + put_storagegroup, + verify_get_storage_group, + verify_list_storage_group, +) +from python_keywords.utility_keywords import generate_file +from wallet import init_wallet + +import allure + +logger = logging.getLogger("NeoLogger") +deposit = 30 + + +@pytest.mark.parametrize( + "object_size", + [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + ids=["simple object", "complex object"], +) +@pytest.mark.storage_group +class TestStorageGroup: + @pytest.fixture(autouse=True) + def prepare_two_wallets(self, prepare_wallet_and_deposit, tmp_path): + self.main_wallet = prepare_wallet_and_deposit + self.other_wallet, _, _ = init_wallet(tmp_path) + transfer_mainnet_gas(self.other_wallet, 31) + neofs_deposit(self.other_wallet, 30) + + @allure.title("Test Storage Group in Private Container") + def test_storagegroup_basic_private_container(self, object_size): + cid = create_container(self.main_wallet) + file_path = generate_file(object_size) + oid = put_object(self.main_wallet, file_path, cid) + objects = [oid] + storage_group = put_storagegroup(self.main_wallet, cid, objects) + + self.expect_success_for_storagegroup_operations( + self.main_wallet, cid, objects, object_size + ) + self.expect_failure_for_storagegroup_operations( + self.other_wallet, cid, objects, storage_group + ) + self.storagegroup_operations_by_system_ro_container( + self.main_wallet, cid, objects, object_size + ) + + @allure.title("Test Storage Group in Public Container") + def test_storagegroup_basic_public_container(self, object_size): + cid = create_container(self.main_wallet, basic_acl="public-read-write") + file_path = generate_file(object_size) + oid = put_object(self.main_wallet, file_path, cid) + objects = [oid] + self.expect_success_for_storagegroup_operations( + self.main_wallet, cid, objects, object_size + ) + self.expect_success_for_storagegroup_operations( + self.other_wallet, cid, objects, object_size + ) + self.storagegroup_operations_by_system_ro_container( + self.main_wallet, cid, objects, object_size + ) + + @allure.title("Test Storage Group in Read-Only Container") + def test_storagegroup_basic_ro_container(self, object_size): + cid = create_container(self.main_wallet, basic_acl="public-read") + file_path = generate_file(object_size) + oid = put_object(self.main_wallet, file_path, cid) + objects = [oid] + self.expect_success_for_storagegroup_operations( + self.main_wallet, cid, objects, object_size + ) + self.storagegroup_operations_by_other_ro_container( + self.main_wallet, self.other_wallet, cid, objects, object_size + ) + self.storagegroup_operations_by_system_ro_container( + self.main_wallet, cid, objects, object_size + ) + + @allure.title("Test Storage Group with Bearer Allow") + def test_storagegroup_bearer_allow(self, object_size): + cid = create_container(self.main_wallet, basic_acl="eacl-public-read-write") + file_path = generate_file(object_size) + oid = put_object(self.main_wallet, file_path, cid) + objects = [oid] + self.expect_success_for_storagegroup_operations( + self.main_wallet, cid, objects, object_size + ) + storage_group = put_storagegroup(self.main_wallet, cid, objects) + eacl_deny = [ + EACLRule(access=EACLAccess.DENY, role=role, operation=op) + for op in EACLOperation + for role in EACLRole + ] + set_eacl(self.main_wallet, cid, create_eacl(cid, eacl_deny)) + self.expect_failure_for_storagegroup_operations( + self.main_wallet, cid, objects, storage_group + ) + bearer_file = form_bearertoken_file( + self.main_wallet, + cid, + [ + EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.USER) + for op in EACLOperation + ], + ) + self.expect_success_for_storagegroup_operations( + self.main_wallet, cid, objects, object_size, bearer_file + ) + + @allure.title("Test to check Storage Group lifetime") + def test_storagegroup_lifetime(self, object_size): + cid = create_container(self.main_wallet) + file_path = generate_file(object_size) + oid = put_object(self.main_wallet, file_path, cid) + objects = [oid] + storage_group = put_storagegroup(self.main_wallet, cid, objects, lifetime="1") + tick_epoch() + tick_epoch() + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_storagegroup(self.main_wallet, cid, storage_group) + + @staticmethod + @allure.step("Run Storage Group Operations And Expect Success") + def expect_success_for_storagegroup_operations( + wallet: str, cid: str, obj_list: list, object_size: int, bearer_token: Optional[str] = None + ): + """ + This func verifies if the Object's owner is allowed to + Put, List, Get and Delete the Storage Group which contains + the Object. + """ + storage_group = put_storagegroup(wallet, cid, obj_list, bearer_token) + verify_list_storage_group(wallet, cid, storage_group, bearer_token) + verify_get_storage_group(wallet, cid, storage_group, obj_list, object_size, bearer_token) + delete_storagegroup(wallet, cid, storage_group, bearer_token) + + @staticmethod + @allure.step("Run Storage Group Operations And Expect Failure") + def expect_failure_for_storagegroup_operations( + wallet: str, cid: str, obj_list: list, storagegroup: str + ): + """ + This func verifies if the Object's owner isn't allowed to + Put, List, Get and Delete the Storage Group which contains + the Object. + """ + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + put_storagegroup(wallet, cid, obj_list) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + list_storagegroup(wallet, cid) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + get_storagegroup(wallet, cid, storagegroup) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + delete_storagegroup(wallet, cid, storagegroup) + + @staticmethod + @allure.step("Run Storage Group Operations On Other's Behalf In RO Container") + def storagegroup_operations_by_other_ro_container( + owner_wallet: str, other_wallet: str, cid: str, obj_list: list, object_size: int + ): + storage_group = put_storagegroup(owner_wallet, cid, obj_list) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + put_storagegroup(other_wallet, cid, obj_list) + verify_list_storage_group(other_wallet, cid, storage_group) + verify_get_storage_group(other_wallet, cid, storage_group, obj_list, object_size) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + delete_storagegroup(other_wallet, cid, storage_group) + + @staticmethod + @allure.step("Run Storage Group Operations On Systems's Behalf In RO Container") + def storagegroup_operations_by_system_ro_container( + wallet: str, cid: str, obj_list: list, object_size: int + ): + """ + In this func we create a Storage Group on Inner Ring's key behalf + and include an Object created on behalf of some user. We expect + that System key is granted to make all operations except PUT and DELETE. + """ + transfer_mainnet_gas( + IR_WALLET_PATH, deposit + 1, wallet_password=IR_WALLET_PASS + ) + neofs_deposit(IR_WALLET_PATH, deposit, wallet_password=IR_WALLET_PASS) + storage_group = put_storagegroup(wallet, cid, obj_list) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + put_storagegroup( + IR_WALLET_PATH, cid, obj_list, wallet_config=IR_WALLET_CONFIG + ) + verify_list_storage_group( + IR_WALLET_PATH, cid, storage_group, wallet_config=IR_WALLET_CONFIG + ) + verify_get_storage_group( + IR_WALLET_PATH, + cid, + storage_group, + obj_list, + object_size, + wallet_config=IR_WALLET_CONFIG, + ) + with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): + delete_storagegroup( + IR_WALLET_PATH, cid, storage_group, wallet_config=IR_WALLET_CONFIG + ) diff --git a/robot/resources/lib/python_keywords/storage_group.py b/robot/resources/lib/python_keywords/storage_group.py index 0706ac70..f310623f 100644 --- a/robot/resources/lib/python_keywords/storage_group.py +++ b/robot/resources/lib/python_keywords/storage_group.py @@ -4,88 +4,102 @@ This module contains keywords for work with Storage Groups. It contains wrappers for `neofs-cli storagegroup` verbs. """ +import logging from cli_helpers import _cmd_run -from common import NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG - -from robot.api.deco import keyword - -ROBOT_AUTO_KEYWORDS = False +from common import ( + COMPLEX_OBJ_SIZE, + NEOFS_CLI_EXEC, + NEOFS_ENDPOINT, + SIMPLE_OBJ_SIZE, + WALLET_CONFIG, +) +from complex_object_actions import get_link_object +from neofs_verbs import head_object -@keyword('Put Storagegroup') -def put_storagegroup(wallet: str, cid: str, objects: list, bearer_token: str = "", - wallet_config: str = WALLET_CONFIG): +def put_storagegroup( + wallet: str, + cid: str, + objects: list, + bearer_token: str = "", + wallet_config: str = WALLET_CONFIG, + lifetime: str = "10", +): """ - Wrapper for `neofs-cli storagegroup put`. Before the SG is created, - neofs-cli performs HEAD on `objects`, so this verb must be allowed - for `wallet` in `cid`. - Args: - wallet (str): path to wallet on whose behalf the SG is created - cid (str): ID of Container to put SG to - objects (list): list of Object IDs to include into the SG - bearer_token (optional, str): path to Bearer token file - wallet_config (optional, str): path to neofs-cli config file - Returns: - (str): Object ID of created Storage Group + Wrapper for `neofs-cli storagegroup put`. Before the SG is created, + neofs-cli performs HEAD on `objects`, so this verb must be allowed + for `wallet` in `cid`. + Args: + wallet (str): path to wallet on whose behalf the SG is created + cid (str): ID of Container to put SG to + objects (list): list of Object IDs to include into the SG + bearer_token (optional, str): path to Bearer token file + wallet_config (optional, str): path to neofs-cli config file + Returns: + (str): Object ID of created Storage Group """ cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} ' - f'--wallet {wallet} --config {wallet_config} ' - f'storagegroup put --cid {cid} ' + f"{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} " + f"--wallet {wallet} --config {wallet_config} " + f"storagegroup put --cid {cid} --lifetime {lifetime} " f'--members {",".join(objects)} ' f'{"--bearer " + bearer_token if bearer_token else ""}' ) output = _cmd_run(cmd) - oid = output.split('\n')[1].split(': ')[1] + oid = output.split("\n")[1].split(": ")[1] return oid -@keyword('List Storagegroup') -def list_storagegroup(wallet: str, cid: str, bearer_token: str = "", - wallet_config: str = WALLET_CONFIG): +def list_storagegroup( + wallet: str, cid: str, bearer_token: str = "", wallet_config: str = WALLET_CONFIG +): """ - Wrapper for `neofs-cli storagegroup list`. This operation - requires SEARCH allowed for `wallet` in `cid`. - Args: - wallet (str): path to wallet on whose behalf the SGs are - listed in the container - cid (str): ID of Container to list - bearer_token (optional, str): path to Bearer token file - wallet_config (optional, str): path to neofs-cli config file - Returns: - (list): Object IDs of found Storage Groups + Wrapper for `neofs-cli storagegroup list`. This operation + requires SEARCH allowed for `wallet` in `cid`. + Args: + wallet (str): path to wallet on whose behalf the SGs are + listed in the container + cid (str): ID of Container to list + bearer_token (optional, str): path to Bearer token file + wallet_config (optional, str): path to neofs-cli config file + Returns: + (list): Object IDs of found Storage Groups """ cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} ' - f'--wallet {wallet} --config {wallet_config} storagegroup list ' + f"{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} " + f"--wallet {wallet} --config {wallet_config} storagegroup list " f'--cid {cid} {"--bearer " + bearer_token if bearer_token else ""}' ) output = _cmd_run(cmd) # throwing off the first string of output - found_objects = output.split('\n')[1:] + found_objects = output.split("\n")[1:] return found_objects -@keyword('Get Storagegroup') -def get_storagegroup(wallet: str, cid: str, oid: str, bearer_token: str = '', - wallet_config: str = WALLET_CONFIG): +def get_storagegroup( + wallet: str, + cid: str, + oid: str, + bearer_token: str = "", + wallet_config: str = WALLET_CONFIG, +): """ - Wrapper for `neofs-cli storagegroup get`. - Args: - wallet (str): path to wallet on whose behalf the SG is got - cid (str): ID of Container where SG is stored - oid (str): ID of the Storage Group - bearer_token (optional, str): path to Bearer token file - wallet_config (optional, str): path to neofs-cli config file - Returns: - (dict): detailed information on the Storage Group + Wrapper for `neofs-cli storagegroup get`. + Args: + wallet (str): path to wallet on whose behalf the SG is got + cid (str): ID of Container where SG is stored + oid (str): ID of the Storage Group + bearer_token (optional, str): path to Bearer token file + wallet_config (optional, str): path to neofs-cli config file + Returns: + (dict): detailed information on the Storage Group """ cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} ' - f'--wallet {wallet} --config {wallet_config} ' - f'storagegroup get --cid {cid} --id {oid} ' + f"{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} " + f"--wallet {wallet} --config {wallet_config} " + f"storagegroup get --cid {cid} --id {oid} " f'{"--bearer " + bearer_token if bearer_token else ""}' ) output = _cmd_run(cmd) @@ -93,42 +107,101 @@ def get_storagegroup(wallet: str, cid: str, oid: str, bearer_token: str = '', # TODO: temporary solution for parsing output. Needs to be replaced with # JSON parsing when https://github.com/nspcc-dev/neofs-node/issues/1355 # is done. - strings = output.strip().split('\n') + strings = output.strip().split("\n") # first three strings go to `data`; # skip the 'Members:' string; # the rest of strings go to `members` data, members = strings[:3], strings[3:] sg_dict = {} for i in data: - key, val = i.split(': ') + key, val = i.split(": ") sg_dict[key] = val - sg_dict['Members'] = [] + sg_dict["Members"] = [] for member in members[1:]: - sg_dict['Members'].append(member.strip()) + sg_dict["Members"].append(member.strip()) return sg_dict -@keyword('Delete Storagegroup') -def delete_storagegroup(wallet: str, cid: str, oid: str, bearer_token: str = "", - wallet_config: str = WALLET_CONFIG): +def delete_storagegroup( + wallet: str, + cid: str, + oid: str, + bearer_token: str = "", + wallet_config: str = WALLET_CONFIG, +): """ - Wrapper for `neofs-cli storagegroup delete`. - Args: - wallet (str): path to wallet on whose behalf the SG is deleted - cid (str): ID of Container where SG is stored - oid (str): ID of the Storage Group - bearer_token (optional, str): path to Bearer token file - wallet_config (optional, str): path to neofs-cli config file - Returns: - (str): Tombstone ID of the deleted Storage Group + Wrapper for `neofs-cli storagegroup delete`. + Args: + wallet (str): path to wallet on whose behalf the SG is deleted + cid (str): ID of Container where SG is stored + oid (str): ID of the Storage Group + bearer_token (optional, str): path to Bearer token file + wallet_config (optional, str): path to neofs-cli config file + Returns: + (str): Tombstone ID of the deleted Storage Group """ cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} ' - f'--wallet {wallet} --config {wallet_config} ' - f'storagegroup delete --cid {cid} --id {oid} ' + f"{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} " + f"--wallet {wallet} --config {wallet_config} " + f"storagegroup delete --cid {cid} --id {oid} " f'{"--bearer " + bearer_token if bearer_token else ""}' ) output = _cmd_run(cmd) - tombstone_id = output.strip().split('\n')[1].split(': ')[1] + tombstone_id = output.strip().split("\n")[1].split(": ")[1] return tombstone_id + + +def verify_list_storage_group( + wallet: str, + cid: str, + storagegroup: str, + bearer: str = None, + wallet_config: str = WALLET_CONFIG, +): + storage_groups = list_storagegroup( + wallet, cid, bearer_token=bearer, wallet_config=wallet_config + ) + try: + storagegroup in storage_groups + except: + logging.error("Storage Group hasn't been persisted") + + +def verify_get_storage_group( + wallet: str, + cid: str, + storagegroup: str, + obj_list: list, + object_size: int, + bearer: str = None, + wallet_config: str = WALLET_CONFIG, +): + obj_parts = [] + if object_size == COMPLEX_OBJ_SIZE: + for obj in obj_list: + link_oid = get_link_object( + wallet, cid, obj, bearer_token=bearer, wallet_config=wallet_config + ) + obj_head = head_object( + wallet, + cid, + link_oid, + is_raw=True, + bearer_token=bearer, + wallet_config=wallet_config, + ) + obj_parts = obj_head["header"]["split"]["children"] + + obj_num = len(obj_list) + storagegroup_data = get_storagegroup( + wallet, cid, storagegroup, bearer_token=bearer, wallet_config=wallet_config + ) + if object_size == SIMPLE_OBJ_SIZE: + exp_size = SIMPLE_OBJ_SIZE * obj_num + assert int(storagegroup_data["Group size"]) == exp_size + assert storagegroup_data["Members"] == obj_list + else: + exp_size = COMPLEX_OBJ_SIZE * obj_num + assert int(storagegroup_data["Group size"]) == exp_size + assert storagegroup_data["Members"] == obj_parts