Add helper function to wait for GC pass on storage nodes

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>

.flake8

@@ -8,5 +8,5 @@ exclude =
 per-file-ignores =
     # imported but unused
     __init__.py: F401
-max-line-length = 120
+max-line-length = 100
 disable-noqa

pytest_tests/helpers/utility.py

@@ -1,7 +1,10 @@
 import os
+import time
 import uuid
 
-from common import ASSETS_DIR, SIMPLE_OBJ_SIZE
+import allure
+
+from common import ASSETS_DIR, SIMPLE_OBJ_SIZE, SHARD_0_GC_SLEEP
 
 
 def create_file_with_content(file_path: str = None, content: str = None) -> str:
@@ -83,3 +86,10 @@ def placement_policy_from_container(container_info: str) -> str:
     """
     assert ':' in container_info, f'Could not find placement rule in the output {container_info}'
     return container_info.split(':')[-1].replace('\n', ' ').strip()
+
+
+def wait_for_gc_pass_on_storage_nodes() -> None:
+    # We add 15 seconds to allow some time for GC process itself
+    wait_time = robot_time_to_int(SHARD_0_GC_SLEEP) + 15
+    with allure.step(f'Wait {wait_time}s until GC completes on storage nodes'):
+        time.sleep(wait_time)

pytest_tests/testsuites/network/test_node_management.py

@@ -7,7 +7,7 @@ import base58
 import pytest
 from cli_helpers import _cmd_run
 from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT,
-                    NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC, SHARD_0_GC_SLEEP)
+                    NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC)
 from epoch import tick_epoch
 from python_keywords.container import create_container, get_container
 from python_keywords.neofs_verbs import (delete_object, get_object,
@@ -20,7 +20,7 @@ from python_keywords.node_management import (drop_object, get_netmap_snapshot,
                                              start_nodes_remote,
                                              stop_nodes_remote)
 from storage_policy import get_nodes_with_object, get_simple_object_copies
-from utility import placement_policy_from_container, robot_time_to_int
+from utility import placement_policy_from_container, robot_time_to_int, wait_for_gc_pass_on_storage_nodes
 from utility_keywords import generate_file
 from wellknown_acl import PUBLIC_ACL
 
@@ -329,7 +329,7 @@ def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker):
     for _ in range(3):
         try:
             checker(wallet, cid, oid)
-            sleep(robot_time_to_int(SHARD_0_GC_SLEEP))
+            wait_for_gc_pass_on_storage_nodes()
         except Exception as err:
             if 'object not found' in str(err):
                 break

pytest_tests/testsuites/object/test_object_api.py

@@ -3,7 +3,7 @@ from time import sleep
 
 import allure
 import pytest
-from common import SHARD_0_GC_SLEEP, SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE
+from common import SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE
 from container import create_container
 from epoch import get_epoch, tick_epoch
 from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
@@ -12,7 +12,7 @@ from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
 from python_keywords.storage_policy import get_simple_object_copies
 from python_keywords.utility_keywords import generate_file, get_file_hash
 from tombstone import verify_head_tombstone
-from utility import get_file_content, robot_time_to_int
+from utility import get_file_content, wait_for_gc_pass_on_storage_nodes
 
 logger = logging.getLogger('NeoLogger')
 
@@ -123,8 +123,7 @@ def test_object_api_lifetime(prepare_wallet_and_deposit, request, object_size):
             tick_epoch()
 
     # Wait for GC, because object with expiration is counted as alive until GC removes it
-    with allure.step('Wait until GC completes on storage nodes'):
-        sleep(1.5 * robot_time_to_int(SHARD_0_GC_SLEEP))
+    wait_for_gc_pass_on_storage_nodes()
 
     with allure.step('Check object deleted because it expires-on epoch'):
         with pytest.raises(Exception, match='.*object not found.*'):

pytest_tests/testsuites/services/test_http_gate.py

@@ -5,7 +5,7 @@ from time import sleep
 
 import allure
 import pytest
-from common import COMPLEX_OBJ_SIZE, SHARD_0_GC_SLEEP
+from common import COMPLEX_OBJ_SIZE
 from container import create_container
 from epoch import get_epoch, tick_epoch
 from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate,
@@ -14,13 +14,14 @@ from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate,
 from python_keywords.neofs_verbs import get_object, put_object
 from python_keywords.storage_policy import get_nodes_without_object
 from python_keywords.utility_keywords import generate_file, get_file_hash
-from utility import robot_time_to_int
+from utility import wait_for_gc_pass_on_storage_nodes
 from wellknown_acl import PUBLIC_ACL
 
 logger = logging.getLogger('NeoLogger')
 
 # For some reason object uploaded via http gateway is not immediately available for downloading
 # Until this issue is resolved we are waiting for some time before attempting to read an object
+# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed
 OBJECT_UPLOAD_DELAY = 10
 
 @allure.link('https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway', name='neofs-http-gateway')
@@ -149,8 +150,7 @@ class TestHttpGate:
             tick_epoch()
 
             # Wait for GC, because object with expiration is counted as alive until GC removes it
-            with allure.step('Wait until GC completes on storage nodes'):
-                sleep(robot_time_to_int(SHARD_0_GC_SLEEP))
+            wait_for_gc_pass_on_storage_nodes()
 
             for oid in expired_objects:
                 self.try_to_get_object_and_expect_error(

pytest_tests/testsuites/services/test_s3_gate.py

@@ -95,6 +95,7 @@ class TestS3Gate:
         with allure.step('Check buckets are presented in the system'):
             # We have an issue that sometimes bucket is not available in the list
             # immediately after creation, so we take several attempts with sleep
+            # TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/628 is fixed
             buckets = []
             for attempt in range(8):
                 with allure.step(f'Loading buckets list (attempt #{attempt})'):

robot/resources/lib/python_keywords/s3_gate_bucket.py

@@ -30,6 +30,7 @@ NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
 # Artificial delay that we add after object deletion and container creation
 # Delay is added because sometimes immediately after deletion object still appears
 # to be existing (probably because tombstone object takes some time to replicate)
+# TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/610 is fixed
 S3_SYNC_WAIT_TIME = 5
 
 class VersioningStatus(Enum):