From 892b8f227a137f3e671c4ed9b1e1e647c04db4c7 Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Wed, 27 Jul 2022 13:42:29 +0300 Subject: [PATCH] Add helper function to wait for GC pass on storage nodes Signed-off-by: Vladimir Domnich --- .flake8 | 2 +- pytest_tests/helpers/utility.py | 12 +++++++++++- .../testsuites/network/test_node_management.py | 6 +++--- pytest_tests/testsuites/object/test_object_api.py | 7 +++---- pytest_tests/testsuites/services/test_http_gate.py | 8 ++++---- pytest_tests/testsuites/services/test_s3_gate.py | 1 + .../resources/lib/python_keywords/s3_gate_bucket.py | 1 + 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.flake8 b/.flake8 index eaa71fb..c65a31b 100644 --- a/.flake8 +++ b/.flake8 @@ -8,5 +8,5 @@ exclude = per-file-ignores = # imported but unused __init__.py: F401 -max-line-length = 120 +max-line-length = 100 disable-noqa \ No newline at end of file diff --git a/pytest_tests/helpers/utility.py b/pytest_tests/helpers/utility.py index 7e4f013..0d6b1f8 100644 --- a/pytest_tests/helpers/utility.py +++ b/pytest_tests/helpers/utility.py @@ -1,7 +1,10 @@ import os +import time import uuid -from common import ASSETS_DIR, SIMPLE_OBJ_SIZE +import allure + +from common import ASSETS_DIR, SIMPLE_OBJ_SIZE, SHARD_0_GC_SLEEP def create_file_with_content(file_path: str = None, content: str = None) -> str: @@ -83,3 +86,10 @@ def placement_policy_from_container(container_info: str) -> str: """ assert ':' in container_info, f'Could not find placement rule in the output {container_info}' return container_info.split(':')[-1].replace('\n', ' ').strip() + + +def wait_for_gc_pass_on_storage_nodes() -> None: + # We add 15 seconds to allow some time for GC process itself + wait_time = robot_time_to_int(SHARD_0_GC_SLEEP) + 15 + with allure.step(f'Wait {wait_time}s until GC completes on storage nodes'): + time.sleep(wait_time) diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index bb43364..bebc2a0 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -7,7 +7,7 @@ import base58 import pytest from cli_helpers import _cmd_run from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT, - NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC, SHARD_0_GC_SLEEP) + NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC) from epoch import tick_epoch from python_keywords.container import create_container, get_container from python_keywords.neofs_verbs import (delete_object, get_object, @@ -20,7 +20,7 @@ from python_keywords.node_management import (drop_object, get_netmap_snapshot, start_nodes_remote, stop_nodes_remote) from storage_policy import get_nodes_with_object, get_simple_object_copies -from utility import placement_policy_from_container, robot_time_to_int +from utility import placement_policy_from_container, robot_time_to_int, wait_for_gc_pass_on_storage_nodes from utility_keywords import generate_file from wellknown_acl import PUBLIC_ACL @@ -329,7 +329,7 @@ def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker): for _ in range(3): try: checker(wallet, cid, oid) - sleep(robot_time_to_int(SHARD_0_GC_SLEEP)) + wait_for_gc_pass_on_storage_nodes() except Exception as err: if 'object not found' in str(err): break diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py index 3564bb4..1dd7976 100644 --- a/pytest_tests/testsuites/object/test_object_api.py +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -3,7 +3,7 @@ from time import sleep import allure import pytest -from common import SHARD_0_GC_SLEEP, SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE +from common import SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE from container import create_container from epoch import get_epoch, tick_epoch from python_keywords.neofs_verbs import (delete_object, get_object, get_range, @@ -12,7 +12,7 @@ from python_keywords.neofs_verbs import (delete_object, get_object, get_range, from python_keywords.storage_policy import get_simple_object_copies from python_keywords.utility_keywords import generate_file, get_file_hash from tombstone import verify_head_tombstone -from utility import get_file_content, robot_time_to_int +from utility import get_file_content, wait_for_gc_pass_on_storage_nodes logger = logging.getLogger('NeoLogger') @@ -123,8 +123,7 @@ def test_object_api_lifetime(prepare_wallet_and_deposit, request, object_size): tick_epoch() # Wait for GC, because object with expiration is counted as alive until GC removes it - with allure.step('Wait until GC completes on storage nodes'): - sleep(1.5 * robot_time_to_int(SHARD_0_GC_SLEEP)) + wait_for_gc_pass_on_storage_nodes() with allure.step('Check object deleted because it expires-on epoch'): with pytest.raises(Exception, match='.*object not found.*'): diff --git a/pytest_tests/testsuites/services/test_http_gate.py b/pytest_tests/testsuites/services/test_http_gate.py index 97ebf62..9a0cf53 100644 --- a/pytest_tests/testsuites/services/test_http_gate.py +++ b/pytest_tests/testsuites/services/test_http_gate.py @@ -5,7 +5,7 @@ from time import sleep import allure import pytest -from common import COMPLEX_OBJ_SIZE, SHARD_0_GC_SLEEP +from common import COMPLEX_OBJ_SIZE from container import create_container from epoch import get_epoch, tick_epoch from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate, @@ -14,13 +14,14 @@ from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate, from python_keywords.neofs_verbs import get_object, put_object from python_keywords.storage_policy import get_nodes_without_object from python_keywords.utility_keywords import generate_file, get_file_hash -from utility import robot_time_to_int +from utility import wait_for_gc_pass_on_storage_nodes from wellknown_acl import PUBLIC_ACL logger = logging.getLogger('NeoLogger') # For some reason object uploaded via http gateway is not immediately available for downloading # Until this issue is resolved we are waiting for some time before attempting to read an object +# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed OBJECT_UPLOAD_DELAY = 10 @allure.link('https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway', name='neofs-http-gateway') @@ -149,8 +150,7 @@ class TestHttpGate: tick_epoch() # Wait for GC, because object with expiration is counted as alive until GC removes it - with allure.step('Wait until GC completes on storage nodes'): - sleep(robot_time_to_int(SHARD_0_GC_SLEEP)) + wait_for_gc_pass_on_storage_nodes() for oid in expired_objects: self.try_to_get_object_and_expect_error( diff --git a/pytest_tests/testsuites/services/test_s3_gate.py b/pytest_tests/testsuites/services/test_s3_gate.py index b08d064..54b3ba7 100644 --- a/pytest_tests/testsuites/services/test_s3_gate.py +++ b/pytest_tests/testsuites/services/test_s3_gate.py @@ -95,6 +95,7 @@ class TestS3Gate: with allure.step('Check buckets are presented in the system'): # We have an issue that sometimes bucket is not available in the list # immediately after creation, so we take several attempts with sleep + # TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/628 is fixed buckets = [] for attempt in range(8): with allure.step(f'Loading buckets list (attempt #{attempt})'): diff --git a/robot/resources/lib/python_keywords/s3_gate_bucket.py b/robot/resources/lib/python_keywords/s3_gate_bucket.py index 08a8951..6fca362 100644 --- a/robot/resources/lib/python_keywords/s3_gate_bucket.py +++ b/robot/resources/lib/python_keywords/s3_gate_bucket.py @@ -30,6 +30,7 @@ NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate') # Artificial delay that we add after object deletion and container creation # Delay is added because sometimes immediately after deletion object still appears # to be existing (probably because tombstone object takes some time to replicate) +# TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/610 is fixed S3_SYNC_WAIT_TIME = 5 class VersioningStatus(Enum):