Add helper function to wait for GC pass on storage nodes

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-07-27 13:42:29 +03:00
parent 05da181998
commit 892b8f227a
7 changed files with 24 additions and 13 deletions

View file

@ -8,5 +8,5 @@ exclude =
per-file-ignores = per-file-ignores =
# imported but unused # imported but unused
__init__.py: F401 __init__.py: F401
max-line-length = 120 max-line-length = 100
disable-noqa disable-noqa

View file

@ -1,7 +1,10 @@
import os import os
import time
import uuid import uuid
from common import ASSETS_DIR, SIMPLE_OBJ_SIZE import allure
from common import ASSETS_DIR, SIMPLE_OBJ_SIZE, SHARD_0_GC_SLEEP
def create_file_with_content(file_path: str = None, content: str = None) -> str: def create_file_with_content(file_path: str = None, content: str = None) -> str:
@ -83,3 +86,10 @@ def placement_policy_from_container(container_info: str) -> str:
""" """
assert ':' in container_info, f'Could not find placement rule in the output {container_info}' assert ':' in container_info, f'Could not find placement rule in the output {container_info}'
return container_info.split(':')[-1].replace('\n', ' ').strip() return container_info.split(':')[-1].replace('\n', ' ').strip()
def wait_for_gc_pass_on_storage_nodes() -> None:
# We add 15 seconds to allow some time for GC process itself
wait_time = robot_time_to_int(SHARD_0_GC_SLEEP) + 15
with allure.step(f'Wait {wait_time}s until GC completes on storage nodes'):
time.sleep(wait_time)

View file

@ -7,7 +7,7 @@ import base58
import pytest import pytest
from cli_helpers import _cmd_run from cli_helpers import _cmd_run
from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT, from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT,
NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC, SHARD_0_GC_SLEEP) NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC)
from epoch import tick_epoch from epoch import tick_epoch
from python_keywords.container import create_container, get_container from python_keywords.container import create_container, get_container
from python_keywords.neofs_verbs import (delete_object, get_object, from python_keywords.neofs_verbs import (delete_object, get_object,
@ -20,7 +20,7 @@ from python_keywords.node_management import (drop_object, get_netmap_snapshot,
start_nodes_remote, start_nodes_remote,
stop_nodes_remote) stop_nodes_remote)
from storage_policy import get_nodes_with_object, get_simple_object_copies from storage_policy import get_nodes_with_object, get_simple_object_copies
from utility import placement_policy_from_container, robot_time_to_int from utility import placement_policy_from_container, robot_time_to_int, wait_for_gc_pass_on_storage_nodes
from utility_keywords import generate_file from utility_keywords import generate_file
from wellknown_acl import PUBLIC_ACL from wellknown_acl import PUBLIC_ACL
@ -329,7 +329,7 @@ def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker):
for _ in range(3): for _ in range(3):
try: try:
checker(wallet, cid, oid) checker(wallet, cid, oid)
sleep(robot_time_to_int(SHARD_0_GC_SLEEP)) wait_for_gc_pass_on_storage_nodes()
except Exception as err: except Exception as err:
if 'object not found' in str(err): if 'object not found' in str(err):
break break

View file

@ -3,7 +3,7 @@ from time import sleep
import allure import allure
import pytest import pytest
from common import SHARD_0_GC_SLEEP, SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE from common import SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE
from container import create_container from container import create_container
from epoch import get_epoch, tick_epoch from epoch import get_epoch, tick_epoch
from python_keywords.neofs_verbs import (delete_object, get_object, get_range, from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
@ -12,7 +12,7 @@ from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
from python_keywords.storage_policy import get_simple_object_copies from python_keywords.storage_policy import get_simple_object_copies
from python_keywords.utility_keywords import generate_file, get_file_hash from python_keywords.utility_keywords import generate_file, get_file_hash
from tombstone import verify_head_tombstone from tombstone import verify_head_tombstone
from utility import get_file_content, robot_time_to_int from utility import get_file_content, wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger('NeoLogger') logger = logging.getLogger('NeoLogger')
@ -123,8 +123,7 @@ def test_object_api_lifetime(prepare_wallet_and_deposit, request, object_size):
tick_epoch() tick_epoch()
# Wait for GC, because object with expiration is counted as alive until GC removes it # Wait for GC, because object with expiration is counted as alive until GC removes it
with allure.step('Wait until GC completes on storage nodes'): wait_for_gc_pass_on_storage_nodes()
sleep(1.5 * robot_time_to_int(SHARD_0_GC_SLEEP))
with allure.step('Check object deleted because it expires-on epoch'): with allure.step('Check object deleted because it expires-on epoch'):
with pytest.raises(Exception, match='.*object not found.*'): with pytest.raises(Exception, match='.*object not found.*'):

View file

@ -5,7 +5,7 @@ from time import sleep
import allure import allure
import pytest import pytest
from common import COMPLEX_OBJ_SIZE, SHARD_0_GC_SLEEP from common import COMPLEX_OBJ_SIZE
from container import create_container from container import create_container
from epoch import get_epoch, tick_epoch from epoch import get_epoch, tick_epoch
from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate, from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate,
@ -14,13 +14,14 @@ from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate,
from python_keywords.neofs_verbs import get_object, put_object from python_keywords.neofs_verbs import get_object, put_object
from python_keywords.storage_policy import get_nodes_without_object from python_keywords.storage_policy import get_nodes_without_object
from python_keywords.utility_keywords import generate_file, get_file_hash from python_keywords.utility_keywords import generate_file, get_file_hash
from utility import robot_time_to_int from utility import wait_for_gc_pass_on_storage_nodes
from wellknown_acl import PUBLIC_ACL from wellknown_acl import PUBLIC_ACL
logger = logging.getLogger('NeoLogger') logger = logging.getLogger('NeoLogger')
# For some reason object uploaded via http gateway is not immediately available for downloading # For some reason object uploaded via http gateway is not immediately available for downloading
# Until this issue is resolved we are waiting for some time before attempting to read an object # Until this issue is resolved we are waiting for some time before attempting to read an object
# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed
OBJECT_UPLOAD_DELAY = 10 OBJECT_UPLOAD_DELAY = 10
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway', name='neofs-http-gateway') @allure.link('https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway', name='neofs-http-gateway')
@ -149,8 +150,7 @@ class TestHttpGate:
tick_epoch() tick_epoch()
# Wait for GC, because object with expiration is counted as alive until GC removes it # Wait for GC, because object with expiration is counted as alive until GC removes it
with allure.step('Wait until GC completes on storage nodes'): wait_for_gc_pass_on_storage_nodes()
sleep(robot_time_to_int(SHARD_0_GC_SLEEP))
for oid in expired_objects: for oid in expired_objects:
self.try_to_get_object_and_expect_error( self.try_to_get_object_and_expect_error(

View file

@ -95,6 +95,7 @@ class TestS3Gate:
with allure.step('Check buckets are presented in the system'): with allure.step('Check buckets are presented in the system'):
# We have an issue that sometimes bucket is not available in the list # We have an issue that sometimes bucket is not available in the list
# immediately after creation, so we take several attempts with sleep # immediately after creation, so we take several attempts with sleep
# TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/628 is fixed
buckets = [] buckets = []
for attempt in range(8): for attempt in range(8):
with allure.step(f'Loading buckets list (attempt #{attempt})'): with allure.step(f'Loading buckets list (attempt #{attempt})'):

View file

@ -30,6 +30,7 @@ NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
# Artificial delay that we add after object deletion and container creation # Artificial delay that we add after object deletion and container creation
# Delay is added because sometimes immediately after deletion object still appears # Delay is added because sometimes immediately after deletion object still appears
# to be existing (probably because tombstone object takes some time to replicate) # to be existing (probably because tombstone object takes some time to replicate)
# TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/610 is fixed
S3_SYNC_WAIT_TIME = 5 S3_SYNC_WAIT_TIME = 5
class VersioningStatus(Enum): class VersioningStatus(Enum):