EliChin/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Issues Pull requests Projects Releases Packages Wiki Activity

1. Adding timeout control things 2. Add logs filtering

Browse source 
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-02-07 11:40:31 +03:00 committed by abereziny
parent 22f73e6cde
commit 469ab4db43
6 changed files with 106 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/neofs_testlib/cli/neofs_cli/container.py
View file

@ -21,6 +21,7 @@ class NeofsCliContainer(CliCommand):
subnet: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Create a new container and register it in the NeoFS.
@ -43,6 +44,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -63,6 +65,7 @@ class NeofsCliContainer(CliCommand):
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
force: bool = False,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Delete an existing container.
@ -78,6 +81,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -99,6 +103,7 @@ class NeofsCliContainer(CliCommand):
json_mode: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Get container field info.
@ -113,6 +118,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -133,6 +139,7 @@ class NeofsCliContainer(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Get extended ACL table of container.
@ -147,6 +154,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -165,6 +173,7 @@ class NeofsCliContainer(CliCommand):
owner: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
**params,
) -> CommandResult:
"""
@ -177,6 +186,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -194,6 +204,7 @@ class NeofsCliContainer(CliCommand):
address: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
List existing objects in container.
@ -205,6 +216,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -225,6 +237,7 @@ class NeofsCliContainer(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Set a new extended ACL table for the container.
@ -240,6 +253,7 @@ class NeofsCliContainer(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.

16
src/neofs_testlib/cli/neofs_cli/object.py
View file

@ -16,6 +16,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Delete object from NeoFS.
@ -30,6 +31,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -54,6 +56,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Get object from NeoFS.
@ -72,6 +75,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -95,6 +99,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
hash_type: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Get object hash.
@ -112,6 +117,7 @@ class NeofsCliObject(CliCommand):
hash_type: Hash type. Either 'sha256' or 'tz' (default "sha256").
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -139,6 +145,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Get object header.
@ -158,6 +165,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -180,6 +188,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Lock object in container.
@ -196,6 +205,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -222,6 +232,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Put object to NeoFS.
@ -243,6 +254,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -267,6 +279,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Get payload range data of an object.
@ -285,6 +298,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.
@ -308,6 +322,7 @@ class NeofsCliObject(CliCommand):
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""
Search object.
@ -325,6 +340,7 @@ class NeofsCliObject(CliCommand):
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation (default 15s).
Returns:
Command's result.

10
src/neofs_testlib/defaults.py Normal file
View file

@ -0,0 +1,10 @@
class Options:
DEFAULT_SHELL_TIMEOUT = 90
@staticmethod
def get_default_shell_timeout():
return Options.DEFAULT_SHELL_TIMEOUT
@staticmethod
def set_default_shell_timeout(value: int):
Options.DEFAULT_SHELL_TIMEOUT = value

30
src/neofs_testlib/hosting/docker_host.py
View file

@ -1,6 +1,7 @@
import json
import logging
import os
import re
import time
from dataclasses import dataclass
from datetime import datetime
@ -143,6 +144,7 @@ class DockerHost(Host):
directory_path: str,
since: Optional[datetime] = None,
until: Optional[datetime] = None,
filter_regex: Optional[str] = None,
) -> None:
client = self._get_docker_client()
for service_config in self._config.services:
@ -153,6 +155,12 @@ class DockerHost(Host):
logger.info(f"Got exception while dumping logs of '{container_name}': {exc}")
continue
if filter_regex:
logs = (
"\n".join(match[0] for match in re.findall(filter_regex, logs, re.IGNORECASE))
or f"No matches found in logs based on given filter '{filter_regex}'"
)
# Save logs to the directory
file_path = os.path.join(
directory_path,
@ -161,6 +169,28 @@ class DockerHost(Host):
with open(file_path, "wb") as file:
file.write(logs)
def is_message_in_logs(
self,
message_regex: str,
since: Optional[datetime] = None,
until: Optional[datetime] = None,
) -> bool:
client = self._get_docker_client()
for service_config in self._config.services:
container_name = self._get_service_attributes(service_config.name).container_name
try:
logs = client.logs(container_name, since=since, until=until)
except HTTPError as exc:
logger.info(f"Got exception while dumping logs of '{container_name}': {exc}")
continue
if message_regex:
matches = re.findall(message_regex, logs, re.IGNORECASE)
if matches:
return True
return False
def _get_service_attributes(self, service_name) -> ServiceAttributes:
service_config = self.get_service_config(service_name)
return ServiceAttributes.parse(service_config.attributes)

37
src/neofs_testlib/hosting/interfaces.py
View file

@ -118,11 +118,11 @@ class Host(ABC):
"""Detaches disk device to simulate disk offline/failover scenario.
Args:
device: Device name to detach
device: Device name to detach.
Returns:
internal service disk info related to host plugin (i.e. volume id for cloud devices),
which may be used to identify or re-attach existing volume back
which may be used to identify or re-attach existing volume back.
"""
@abstractmethod
@ -130,8 +130,8 @@ class Host(ABC):
"""Attaches disk device back.
Args:
device: Device name to attach
service_info: any info required for host plugin to identify/attach disk
device: Device name to attach.
service_info: any info required for host plugin to identify/attach disk.
"""
@abstractmethod
@ -139,12 +139,12 @@ class Host(ABC):
"""Checks if disk device is attached.
Args:
device: Device name to check
service_info: any info required for host plugin to identify disk
device: Device name to check.
service_info: any info required for host plugin to identify disk.
Returns:
True if attached
False if detached
True if attached.
False if detached.
"""
@abstractmethod
@ -153,6 +153,7 @@ class Host(ABC):
directory_path: str,
since: Optional[datetime] = None,
until: Optional[datetime] = None,
filter_regex: Optional[str] = None,
) -> None:
"""Dumps logs of all services on the host to specified directory.
@ -160,4 +161,24 @@ class Host(ABC):
directory_path: Path to the directory where logs should be stored.
since: If set, limits the time from which logs should be collected. Must be in UTC.
until: If set, limits the time until which logs should be collected. Must be in UTC.
filter_regex: regex to filter output
"""
@abstractmethod
def is_message_in_logs(
self,
message_regex: str,
since: Optional[datetime] = None,
until: Optional[datetime] = None,
) -> bool:
"""Checks logs on host for specified message regex.
Args:
message_regex: message to find.
since: If set, limits the time from which logs should be collected. Must be in UTC.
until: If set, limits the time until which logs should be collected. Must be in UTC.
Returns:
True if message found in logs in the given time frame.
False otherwise.
"""

8
src/neofs_testlib/shell/interfaces.py
View file

@ -2,6 +2,8 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Optional
from neofs_testlib.defaults import Options
@dataclass
class InteractiveInput:
@ -49,10 +51,14 @@ class CommandOptions:
interactive_inputs: Optional[list[InteractiveInput]] = None
close_stdin: bool = False
timeout: int = 30
timeout: Optional[int] = None
check: bool = True
no_log: bool = False
def __post_init__(self):
if self.timeout is None:
self.timeout = Options.get_default_shell_timeout()
@dataclass
class CommandResult: