From 469ab4db43303d2bbc8b552be8efc6c21dffbe37 Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Tue, 7 Feb 2023 11:40:31 +0300 Subject: [PATCH] 1. Adding timeout control things 2. Add logs filtering Signed-off-by: Andrey Berezin --- src/neofs_testlib/cli/neofs_cli/container.py | 14 ++++++++ src/neofs_testlib/cli/neofs_cli/object.py | 16 +++++++++ src/neofs_testlib/defaults.py | 10 ++++++ src/neofs_testlib/hosting/docker_host.py | 30 ++++++++++++++++ src/neofs_testlib/hosting/interfaces.py | 37 +++++++++++++++----- src/neofs_testlib/shell/interfaces.py | 8 ++++- 6 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 src/neofs_testlib/defaults.py diff --git a/src/neofs_testlib/cli/neofs_cli/container.py b/src/neofs_testlib/cli/neofs_cli/container.py index ca0b081..1952448 100644 --- a/src/neofs_testlib/cli/neofs_cli/container.py +++ b/src/neofs_testlib/cli/neofs_cli/container.py @@ -21,6 +21,7 @@ class NeofsCliContainer(CliCommand): subnet: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Create a new container and register it in the NeoFS. @@ -43,6 +44,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -63,6 +65,7 @@ class NeofsCliContainer(CliCommand): ttl: Optional[int] = None, xhdr: Optional[dict] = None, force: bool = False, + timeout: Optional[str] = None, ) -> CommandResult: """ Delete an existing container. @@ -78,6 +81,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -99,6 +103,7 @@ class NeofsCliContainer(CliCommand): json_mode: bool = False, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Get container field info. @@ -113,6 +118,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -133,6 +139,7 @@ class NeofsCliContainer(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Get extended ACL table of container. @@ -147,6 +154,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -165,6 +173,7 @@ class NeofsCliContainer(CliCommand): owner: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, **params, ) -> CommandResult: """ @@ -177,6 +186,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -194,6 +204,7 @@ class NeofsCliContainer(CliCommand): address: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ List existing objects in container. @@ -205,6 +216,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -225,6 +237,7 @@ class NeofsCliContainer(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Set a new extended ACL table for the container. @@ -240,6 +253,7 @@ class NeofsCliContainer(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. diff --git a/src/neofs_testlib/cli/neofs_cli/object.py b/src/neofs_testlib/cli/neofs_cli/object.py index ae847c9..164076c 100644 --- a/src/neofs_testlib/cli/neofs_cli/object.py +++ b/src/neofs_testlib/cli/neofs_cli/object.py @@ -16,6 +16,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Delete object from NeoFS. @@ -30,6 +31,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -54,6 +56,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Get object from NeoFS. @@ -72,6 +75,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -95,6 +99,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, hash_type: Optional[str] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Get object hash. @@ -112,6 +117,7 @@ class NeofsCliObject(CliCommand): hash_type: Hash type. Either 'sha256' or 'tz' (default "sha256"). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -139,6 +145,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Get object header. @@ -158,6 +165,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -180,6 +188,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Lock object in container. @@ -196,6 +205,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -222,6 +232,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Put object to NeoFS. @@ -243,6 +254,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -267,6 +279,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Get payload range data of an object. @@ -285,6 +298,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. @@ -308,6 +322,7 @@ class NeofsCliObject(CliCommand): session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = None, ) -> CommandResult: """ Search object. @@ -325,6 +340,7 @@ class NeofsCliObject(CliCommand): ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation (default 15s). Returns: Command's result. diff --git a/src/neofs_testlib/defaults.py b/src/neofs_testlib/defaults.py new file mode 100644 index 0000000..687fbd6 --- /dev/null +++ b/src/neofs_testlib/defaults.py @@ -0,0 +1,10 @@ +class Options: + DEFAULT_SHELL_TIMEOUT = 90 + + @staticmethod + def get_default_shell_timeout(): + return Options.DEFAULT_SHELL_TIMEOUT + + @staticmethod + def set_default_shell_timeout(value: int): + Options.DEFAULT_SHELL_TIMEOUT = value diff --git a/src/neofs_testlib/hosting/docker_host.py b/src/neofs_testlib/hosting/docker_host.py index cba70ef..257847b 100644 --- a/src/neofs_testlib/hosting/docker_host.py +++ b/src/neofs_testlib/hosting/docker_host.py @@ -1,6 +1,7 @@ import json import logging import os +import re import time from dataclasses import dataclass from datetime import datetime @@ -143,6 +144,7 @@ class DockerHost(Host): directory_path: str, since: Optional[datetime] = None, until: Optional[datetime] = None, + filter_regex: Optional[str] = None, ) -> None: client = self._get_docker_client() for service_config in self._config.services: @@ -153,6 +155,12 @@ class DockerHost(Host): logger.info(f"Got exception while dumping logs of '{container_name}': {exc}") continue + if filter_regex: + logs = ( + "\n".join(match[0] for match in re.findall(filter_regex, logs, re.IGNORECASE)) + or f"No matches found in logs based on given filter '{filter_regex}'" + ) + # Save logs to the directory file_path = os.path.join( directory_path, @@ -161,6 +169,28 @@ class DockerHost(Host): with open(file_path, "wb") as file: file.write(logs) + def is_message_in_logs( + self, + message_regex: str, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + ) -> bool: + client = self._get_docker_client() + for service_config in self._config.services: + container_name = self._get_service_attributes(service_config.name).container_name + try: + logs = client.logs(container_name, since=since, until=until) + except HTTPError as exc: + logger.info(f"Got exception while dumping logs of '{container_name}': {exc}") + continue + + if message_regex: + matches = re.findall(message_regex, logs, re.IGNORECASE) + if matches: + return True + + return False + def _get_service_attributes(self, service_name) -> ServiceAttributes: service_config = self.get_service_config(service_name) return ServiceAttributes.parse(service_config.attributes) diff --git a/src/neofs_testlib/hosting/interfaces.py b/src/neofs_testlib/hosting/interfaces.py index b90eb3d..ddb07b7 100644 --- a/src/neofs_testlib/hosting/interfaces.py +++ b/src/neofs_testlib/hosting/interfaces.py @@ -118,11 +118,11 @@ class Host(ABC): """Detaches disk device to simulate disk offline/failover scenario. Args: - device: Device name to detach + device: Device name to detach. Returns: internal service disk info related to host plugin (i.e. volume id for cloud devices), - which may be used to identify or re-attach existing volume back + which may be used to identify or re-attach existing volume back. """ @abstractmethod @@ -130,8 +130,8 @@ class Host(ABC): """Attaches disk device back. Args: - device: Device name to attach - service_info: any info required for host plugin to identify/attach disk + device: Device name to attach. + service_info: any info required for host plugin to identify/attach disk. """ @abstractmethod @@ -139,12 +139,12 @@ class Host(ABC): """Checks if disk device is attached. Args: - device: Device name to check - service_info: any info required for host plugin to identify disk + device: Device name to check. + service_info: any info required for host plugin to identify disk. Returns: - True if attached - False if detached + True if attached. + False if detached. """ @abstractmethod @@ -153,6 +153,7 @@ class Host(ABC): directory_path: str, since: Optional[datetime] = None, until: Optional[datetime] = None, + filter_regex: Optional[str] = None, ) -> None: """Dumps logs of all services on the host to specified directory. @@ -160,4 +161,24 @@ class Host(ABC): directory_path: Path to the directory where logs should be stored. since: If set, limits the time from which logs should be collected. Must be in UTC. until: If set, limits the time until which logs should be collected. Must be in UTC. + filter_regex: regex to filter output + """ + + @abstractmethod + def is_message_in_logs( + self, + message_regex: str, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + ) -> bool: + """Checks logs on host for specified message regex. + + Args: + message_regex: message to find. + since: If set, limits the time from which logs should be collected. Must be in UTC. + until: If set, limits the time until which logs should be collected. Must be in UTC. + + Returns: + True if message found in logs in the given time frame. + False otherwise. """ diff --git a/src/neofs_testlib/shell/interfaces.py b/src/neofs_testlib/shell/interfaces.py index ca02582..e4f7dea 100644 --- a/src/neofs_testlib/shell/interfaces.py +++ b/src/neofs_testlib/shell/interfaces.py @@ -2,6 +2,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Optional +from neofs_testlib.defaults import Options + @dataclass class InteractiveInput: @@ -49,10 +51,14 @@ class CommandOptions: interactive_inputs: Optional[list[InteractiveInput]] = None close_stdin: bool = False - timeout: int = 30 + timeout: Optional[int] = None check: bool = True no_log: bool = False + def __post_init__(self): + if self.timeout is None: + self.timeout = Options.get_default_shell_timeout() + @dataclass class CommandResult: