forked from TrueCloudLab/frostfs-testcases
[#314] Fix tools config
Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir
parent
2452cccba0
commit
588292dfb5
18 changed files with 239 additions and 844 deletions
|
|
@ -16,8 +16,6 @@ import allure
|
|||
import pexpect
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
COLOR_GREEN = "\033[92m"
|
||||
COLOR_OFF = "\033[0m"
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import neofs_verbs
|
|||
from common import NEOFS_NETMAP, WALLET_CONFIG
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@allure.step("Get Link Object")
|
||||
|
|
@ -35,7 +34,7 @@ def get_link_object(
|
|||
Returns:
|
||||
(str): Link Object ID
|
||||
When no Link Object ID is found after all Storage Nodes polling,
|
||||
the function throws a native robot error.
|
||||
the function throws an error.
|
||||
"""
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
|
|
@ -68,7 +67,7 @@ def get_last_object(wallet: str, cid: str, oid: str):
|
|||
Returns:
|
||||
(str): Last Object ID
|
||||
When no Last Object ID is found after all Storage Nodes polling,
|
||||
the function throws a native robot error.
|
||||
the function throws an error.
|
||||
"""
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
This module contains keywords that utilize `neofs-cli container` commands.
|
||||
"""
|
||||
|
||||
import allure
|
||||
import json
|
||||
import logging
|
||||
from time import sleep
|
||||
|
|
@ -17,7 +16,6 @@ from common import NEOFS_ENDPOINT, WALLET_CONFIG
|
|||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
|
||||
|
||||
|
||||
|
|
@ -105,7 +103,7 @@ def wait_for_container_deletion(wallet: str, cid: str, attempts: int = 30, sleep
|
|||
raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.")
|
||||
|
||||
|
||||
@allure.step('List Containers')
|
||||
@allure.step("List Containers")
|
||||
def list_containers(wallet: str) -> list[str]:
|
||||
"""
|
||||
A wrapper for `neofs-cli container list` call. It returns all the
|
||||
|
|
@ -121,7 +119,7 @@ def list_containers(wallet: str) -> list[str]:
|
|||
return output.split()
|
||||
|
||||
|
||||
@allure.step('Get Container')
|
||||
@allure.step("Get Container")
|
||||
def get_container(wallet: str, cid: str, json_mode: bool = True) -> Union[dict, str]:
|
||||
"""
|
||||
A wrapper for `neofs-cli container get` call. It extracts container's
|
||||
|
|
@ -150,7 +148,7 @@ def get_container(wallet: str, cid: str, json_mode: bool = True) -> Union[dict,
|
|||
return container_info
|
||||
|
||||
|
||||
@allure.step('Delete Container')
|
||||
@allure.step("Delete Container")
|
||||
# TODO: make the error message about a non-found container more user-friendly
|
||||
# https://github.com/nspcc-dev/neofs-contract/issues/121
|
||||
def delete_container(wallet: str, cid: str, force: bool = False) -> None:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ from common import (
|
|||
)
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@allure.step("Get Epoch")
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from cli_helpers import _cmd_run
|
|||
from common import HTTP_GATE
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ import base64
|
|||
|
||||
import base58
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
def decode_simple_header(data: dict):
|
||||
"""
|
||||
|
|
@ -43,14 +41,8 @@ def decode_split_header(data: dict):
|
|||
"""
|
||||
try:
|
||||
data["splitId"] = json_reencode(data["splitId"])
|
||||
data["lastPart"] = (
|
||||
json_reencode(data["lastPart"]["value"])
|
||||
if data["lastPart"] else None
|
||||
)
|
||||
data["link"] = (
|
||||
json_reencode(data["link"]["value"])
|
||||
if data["link"] else None
|
||||
)
|
||||
data["lastPart"] = json_reencode(data["lastPart"]["value"]) if data["lastPart"] else None
|
||||
data["link"] = json_reencode(data["link"]["value"]) if data["link"] else None
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
|
||||
|
|
@ -66,16 +58,18 @@ def decode_linking_object(data: dict):
|
|||
data = decode_simple_header(data)
|
||||
# reencoding Child Object IDs
|
||||
# { 'value': <Base58 encoded OID> } -> <Base64 encoded OID>
|
||||
for ind, val in enumerate(data['header']['split']['children']):
|
||||
data['header']['split']['children'][ind] = json_reencode(val['value'])
|
||||
data['header']['split']['splitID'] = json_reencode(data['header']['split']['splitID'])
|
||||
data['header']['split']['previous'] = (
|
||||
json_reencode(data['header']['split']['previous']['value'])
|
||||
if data['header']['split']['previous'] else None
|
||||
for ind, val in enumerate(data["header"]["split"]["children"]):
|
||||
data["header"]["split"]["children"][ind] = json_reencode(val["value"])
|
||||
data["header"]["split"]["splitID"] = json_reencode(data["header"]["split"]["splitID"])
|
||||
data["header"]["split"]["previous"] = (
|
||||
json_reencode(data["header"]["split"]["previous"]["value"])
|
||||
if data["header"]["split"]["previous"]
|
||||
else None
|
||||
)
|
||||
data['header']['split']['parent'] = (
|
||||
json_reencode(data['header']['split']['parent']['value'])
|
||||
if data['header']['split']['parent'] else None
|
||||
data["header"]["split"]["parent"] = (
|
||||
json_reencode(data["header"]["split"]["parent"]["value"])
|
||||
if data["header"]["split"]["parent"]
|
||||
else None
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
|
|
@ -101,8 +95,7 @@ def decode_tombstone(data: dict):
|
|||
"""
|
||||
try:
|
||||
data = decode_simple_header(data)
|
||||
data['header']['sessionToken'] = decode_session_token(
|
||||
data['header']['sessionToken'])
|
||||
data["header"]["sessionToken"] = decode_session_token(data["header"]["sessionToken"])
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
return data
|
||||
|
|
@ -113,10 +106,12 @@ def decode_session_token(data: dict):
|
|||
This function reencodes a fragment of header which contains
|
||||
information about session token.
|
||||
"""
|
||||
data['body']['object']['address']['containerID'] = json_reencode(
|
||||
data['body']['object']['address']['containerID']['value'])
|
||||
data['body']['object']['address']['objectID'] = json_reencode(
|
||||
data['body']['object']['address']['objectID']['value'])
|
||||
data["body"]["object"]["address"]["containerID"] = json_reencode(
|
||||
data["body"]["object"]["address"]["containerID"]["value"]
|
||||
)
|
||||
data["body"]["object"]["address"]["objectID"] = json_reencode(
|
||||
data["body"]["object"]["address"]["objectID"]["value"]
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
|
|
@ -135,7 +130,7 @@ def encode_for_json(data: str):
|
|||
This function encodes binary data for sending them as protobuf
|
||||
structures.
|
||||
"""
|
||||
return base64.b64encode(base58.b58decode(data)).decode('utf-8')
|
||||
return base64.b64encode(base58.b58decode(data)).decode("utf-8")
|
||||
|
||||
|
||||
def decode_common_fields(data: dict):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
This module contains wrappers for NeoFS verbs executed via neofs-cli.
|
||||
"""
|
||||
|
||||
import allure
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
|
|
@ -19,8 +18,6 @@ from common import ASSETS_DIR, NEOFS_ENDPOINT, NEOFS_NETMAP, WALLET_CONFIG
|
|||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@allure.step("Get object")
|
||||
def get_object(
|
||||
|
|
@ -320,13 +317,13 @@ def search_object(
|
|||
if expected_objects_list:
|
||||
if sorted(found_objects) == sorted(expected_objects_list):
|
||||
logger.info(
|
||||
f"Found objects list '{found_objects}' ",
|
||||
f"is equal for expected list '{expected_objects_list}'",
|
||||
f"Found objects list '{found_objects}' "
|
||||
f"is equal for expected list '{expected_objects_list}'"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Found object list {found_objects} ",
|
||||
f"is not equal to expected list '{expected_objects_list}'",
|
||||
f"Found object list {found_objects} "
|
||||
f"is not equal to expected list '{expected_objects_list}'"
|
||||
)
|
||||
|
||||
return found_objects
|
||||
|
|
|
|||
|
|
@ -12,14 +12,13 @@ from dataclasses import dataclass
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from common import MAINNET_BLOCK_TIME, MORPH_BLOCK_TIME, NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS
|
||||
from common import MORPH_BLOCK_TIME, NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS
|
||||
from data_formatters import get_wallet_public_key
|
||||
from epoch import tick_epoch
|
||||
from service_helper import get_storage_service_helper
|
||||
from utility import robot_time_to_int
|
||||
from utility import parse_time
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -194,7 +193,7 @@ def delete_node_data(node_name: str) -> None:
|
|||
helper = get_storage_service_helper()
|
||||
helper.stop_node(node_name)
|
||||
helper.delete_node_data(node_name)
|
||||
time.sleep(robot_time_to_int(MORPH_BLOCK_TIME))
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
|
||||
|
||||
@allure.step("Exclude node {node_to_exclude} from network map")
|
||||
|
|
@ -204,7 +203,7 @@ def exclude_node_from_network_map(node_to_exclude, alive_node):
|
|||
|
||||
node_set_status(node_to_exclude, status="offline")
|
||||
|
||||
time.sleep(robot_time_to_int(MORPH_BLOCK_TIME))
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
tick_epoch()
|
||||
|
||||
snapshot = get_netmap_snapshot(node_name=alive_node)
|
||||
|
|
@ -217,7 +216,7 @@ def exclude_node_from_network_map(node_to_exclude, alive_node):
|
|||
def include_node_to_network_map(node_to_include: str, alive_node: str) -> None:
|
||||
node_set_status(node_to_include, status="online")
|
||||
|
||||
time.sleep(robot_time_to_int(MORPH_BLOCK_TIME))
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
tick_epoch()
|
||||
|
||||
check_node_in_map(node_to_include, alive_node)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ from wallet import nep17_transfer
|
|||
from wrappers import run_sh_with_passwd_contract
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
EMPTY_PASSWORD = ""
|
||||
TX_PERSIST_TIMEOUT = 15 # seconds
|
||||
|
|
|
|||
|
|
@ -1,226 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import allure
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from time import sleep
|
||||
from typing import Optional
|
||||
|
||||
import allure
|
||||
import boto3
|
||||
import urllib3
|
||||
from botocore.exceptions import ClientError
|
||||
from cli_helpers import _run_with_passwd, log_command_execution
|
||||
from common import NEOFS_ENDPOINT, S3_GATE, S3_GATE_WALLET_PASS, S3_GATE_WALLET_PATH
|
||||
from data_formatters import get_wallet_public_key
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env.
|
||||
urllib3.disable_warnings()
|
||||
##########################################################
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
CREDENTIALS_CREATE_TIMEOUT = "30s"
|
||||
NEOFS_EXEC = os.getenv("NEOFS_EXEC", "neofs-authmate")
|
||||
|
||||
# Artificial delay that we add after object deletion and container creation
|
||||
# Delay is added because sometimes immediately after deletion object still appears
|
||||
# to be existing (probably because tombstone object takes some time to replicate)
|
||||
# TODO: remove after https://github.com/nspcc-dev/neofs-s3-gw/issues/610 is fixed
|
||||
S3_SYNC_WAIT_TIME = 5
|
||||
|
||||
|
||||
class VersioningStatus(Enum):
|
||||
ENABLED = "Enabled"
|
||||
SUSPENDED = "Suspended"
|
||||
|
||||
|
||||
@allure.step("Init S3 Credentials")
|
||||
def init_s3_credentials(wallet_path, s3_bearer_rules_file: Optional[str] = None):
|
||||
bucket = str(uuid.uuid4())
|
||||
s3_bearer_rules = s3_bearer_rules_file or "robot/resources/files/s3_bearer_rules.json"
|
||||
gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS)
|
||||
cmd = (
|
||||
f"{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
|
||||
f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} "
|
||||
f"--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} "
|
||||
f"--bearer-rules {s3_bearer_rules}"
|
||||
)
|
||||
logger.info(f"Executing command: {cmd}")
|
||||
|
||||
try:
|
||||
output = _run_with_passwd(cmd)
|
||||
logger.info(f"Command completed with output: {output}")
|
||||
|
||||
# output contains some debug info and then several JSON structures, so we find each
|
||||
# JSON structure by curly brackets (naive approach, but works while JSON is not nested)
|
||||
# and then we take JSON containing secret_access_key
|
||||
json_blocks = re.findall(r"\{.*?\}", output, re.DOTALL)
|
||||
for json_block in json_blocks:
|
||||
try:
|
||||
parsed_json_block = json.loads(json_block)
|
||||
if "secret_access_key" in parsed_json_block:
|
||||
return (
|
||||
parsed_json_block["container_id"],
|
||||
bucket,
|
||||
parsed_json_block["access_key_id"],
|
||||
parsed_json_block["secret_access_key"],
|
||||
parsed_json_block["owner_private_key"],
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
raise AssertionError(f"Could not parse info from output\n{output}")
|
||||
raise AssertionError(f"Could not find AWS credentials in output:\n{output}")
|
||||
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Failed to init s3 credentials because of error\n{exc}") from exc
|
||||
|
||||
|
||||
@allure.step("Config S3 client")
|
||||
def config_s3_client(access_key_id: str, secret_access_key: str):
|
||||
try:
|
||||
|
||||
session = boto3.session.Session()
|
||||
|
||||
s3_client = session.client(
|
||||
service_name="s3",
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
endpoint_url=S3_GATE,
|
||||
verify=False,
|
||||
)
|
||||
|
||||
return s3_client
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Create bucket S3")
|
||||
def create_bucket_s3(s3_client):
|
||||
bucket_name = str(uuid.uuid4())
|
||||
|
||||
try:
|
||||
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
|
||||
log_command_execution(f"Created S3 bucket {bucket_name}", s3_bucket)
|
||||
sleep(S3_SYNC_WAIT_TIME)
|
||||
return bucket_name
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("List buckets S3")
|
||||
def list_buckets_s3(s3_client):
|
||||
found_buckets = []
|
||||
try:
|
||||
response = s3_client.list_buckets()
|
||||
log_command_execution("S3 List buckets result", response)
|
||||
|
||||
for bucket in response["Buckets"]:
|
||||
found_buckets.append(bucket["Name"])
|
||||
|
||||
return found_buckets
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Delete bucket S3")
|
||||
def delete_bucket_s3(s3_client, bucket: str):
|
||||
try:
|
||||
response = s3_client.delete_bucket(Bucket=bucket)
|
||||
log_command_execution("S3 Delete bucket result", response)
|
||||
sleep(S3_SYNC_WAIT_TIME)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
log_command_execution("S3 Delete bucket error", str(err))
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Head bucket S3")
|
||||
def head_bucket(s3_client, bucket: str):
|
||||
try:
|
||||
response = s3_client.head_bucket(Bucket=bucket)
|
||||
log_command_execution("S3 Head bucket result", response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
log_command_execution("S3 Head bucket error", str(err))
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Set bucket versioning status")
|
||||
def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus) -> None:
|
||||
try:
|
||||
response = s3_client.put_bucket_versioning(
|
||||
Bucket=bucket_name, VersioningConfiguration={"Status": status.value}
|
||||
)
|
||||
log_command_execution("S3 Set bucket versioning to", response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during set bucket versioning: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Get bucket versioning status")
|
||||
def get_bucket_versioning_status(s3_client, bucket_name: str) -> str:
|
||||
try:
|
||||
response = s3_client.get_bucket_versioning(Bucket=bucket_name)
|
||||
status = response.get("Status")
|
||||
log_command_execution("S3 Got bucket versioning status", response)
|
||||
return status
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during get bucket versioning status: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Put bucket tagging")
|
||||
def put_bucket_tagging(s3_client, bucket_name: str, tags: list):
|
||||
try:
|
||||
tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]
|
||||
tagging = {"TagSet": tags}
|
||||
response = s3_client.put_bucket_tagging(Bucket=bucket_name, Tagging=tagging)
|
||||
log_command_execution("S3 Put bucket tagging", response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during put bucket tagging: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Get bucket tagging")
|
||||
def get_bucket_tagging(s3_client, bucket_name: str) -> list:
|
||||
try:
|
||||
response = s3_client.get_bucket_tagging(Bucket=bucket_name)
|
||||
log_command_execution("S3 Get bucket tagging", response)
|
||||
return response.get("TagSet")
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during get bucket tagging: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Delete bucket tagging")
|
||||
def delete_bucket_tagging(s3_client, bucket_name: str) -> None:
|
||||
try:
|
||||
response = s3_client.delete_bucket_tagging(Bucket=bucket_name)
|
||||
log_command_execution("S3 Delete bucket tagging", response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during delete bucket tagging: {err}") from err
|
||||
|
|
@ -1,412 +0,0 @@
|
|||
#!/usr/bin/python3.9
|
||||
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from time import sleep
|
||||
from typing import Optional
|
||||
|
||||
import allure
|
||||
import urllib3
|
||||
from botocore.exceptions import ClientError
|
||||
from cli_helpers import log_command_execution
|
||||
from python_keywords.aws_cli_client import AwsCliClient
|
||||
from python_keywords.s3_gate_bucket import S3_SYNC_WAIT_TIME
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env.
|
||||
urllib3.disable_warnings()
|
||||
##########################################################
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
CREDENTIALS_CREATE_TIMEOUT = "30s"
|
||||
|
||||
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
|
||||
|
||||
|
||||
class VersioningStatus(Enum):
|
||||
ENABLED = "Enabled"
|
||||
SUSPENDED = "Suspended"
|
||||
|
||||
|
||||
@allure.step("List objects S3 v2")
|
||||
def list_objects_s3_v2(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket)
|
||||
content = response.get("Contents", [])
|
||||
log_command_execution("S3 v2 List objects result", response)
|
||||
obj_list = []
|
||||
for obj in content:
|
||||
obj_list.append(obj["Key"])
|
||||
logger.info(f"Found s3 objects: {obj_list}")
|
||||
return obj_list
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("List objects S3")
|
||||
def list_objects_s3(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_objects(Bucket=bucket)
|
||||
content = response.get("Contents", [])
|
||||
log_command_execution("S3 List objects result", response)
|
||||
obj_list = []
|
||||
for obj in content:
|
||||
obj_list.append(obj["Key"])
|
||||
logger.info(f"Found s3 objects: {obj_list}")
|
||||
return obj_list
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("List objects versions S3")
|
||||
def list_objects_versions_s3(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_object_versions(Bucket=bucket)
|
||||
versions = response.get("Versions", [])
|
||||
log_command_execution("S3 List objects versions result", response)
|
||||
return versions
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Put object S3")
|
||||
def put_object_s3(s3_client, bucket: str, filepath: str):
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
file_content = filepath
|
||||
else:
|
||||
with open(filepath, "rb") as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
||||
log_command_execution("S3 Put object result", response)
|
||||
return response.get("VersionId")
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Head object S3")
|
||||
def head_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||||
try:
|
||||
params = {"Bucket": bucket, "Key": object_key}
|
||||
if version_id:
|
||||
params["VersionId"] = version_id
|
||||
response = s3_client.head_object(**params)
|
||||
log_command_execution("S3 Head object result", response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Delete object S3")
|
||||
def delete_object_s3(s3_client, bucket, object_key, version_id: str = None):
|
||||
try:
|
||||
params = {"Bucket": bucket, "Key": object_key}
|
||||
if version_id:
|
||||
params["VersionId"] = version_id
|
||||
response = s3_client.delete_object(**params)
|
||||
log_command_execution("S3 Delete object result", response)
|
||||
sleep(S3_SYNC_WAIT_TIME)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Delete objects S3")
|
||||
def delete_objects_s3(s3_client, bucket: str, object_keys: list):
|
||||
try:
|
||||
response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys))
|
||||
log_command_execution("S3 Delete objects result", response)
|
||||
sleep(S3_SYNC_WAIT_TIME)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Delete object versions S3")
|
||||
def delete_object_versions_s3(s3_client, bucket: str, object_versions: list):
|
||||
try:
|
||||
# Build deletion list in S3 format
|
||||
delete_list = {
|
||||
"Objects": [
|
||||
{
|
||||
"Key": object_version["Key"],
|
||||
"VersionId": object_version["VersionId"],
|
||||
}
|
||||
for object_version in object_versions
|
||||
]
|
||||
}
|
||||
response = s3_client.delete_objects(Bucket=bucket, Delete=delete_list)
|
||||
log_command_execution("S3 Delete objects result", response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Copy object S3")
|
||||
def copy_object_s3(s3_client, bucket, object_key, bucket_dst=None):
|
||||
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
||||
try:
|
||||
response = s3_client.copy_object(
|
||||
Bucket=bucket_dst or bucket, CopySource=f"{bucket}/{object_key}", Key=filename
|
||||
)
|
||||
log_command_execution("S3 Copy objects result", response)
|
||||
return filename
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Get object S3")
|
||||
def get_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||||
filename = f"{ASSETS_DIR}/{uuid.uuid4()}"
|
||||
try:
|
||||
params = {"Bucket": bucket, "Key": object_key}
|
||||
if version_id:
|
||||
params["VersionId"] = version_id
|
||||
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
params["file_path"] = filename
|
||||
|
||||
response = s3_client.get_object(**params)
|
||||
log_command_execution("S3 Get objects result", response)
|
||||
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
with open(f"{filename}", "wb") as get_file:
|
||||
chunk = response["Body"].read(1024)
|
||||
while chunk:
|
||||
get_file.write(chunk)
|
||||
chunk = response["Body"].read(1024)
|
||||
|
||||
return filename
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Create multipart upload S3")
|
||||
def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str:
|
||||
try:
|
||||
response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution("S3 Created multipart upload", response)
|
||||
assert response.get("UploadId"), f"Expected UploadId in response:\n{response}"
|
||||
|
||||
return response.get("UploadId")
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("List multipart uploads S3")
|
||||
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[list[dict]]:
|
||||
try:
|
||||
response = s3_client.list_multipart_uploads(Bucket=bucket_name)
|
||||
log_command_execution("S3 List multipart upload", response)
|
||||
|
||||
return response.get("Uploads")
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Abort multipart upload S3")
|
||||
def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str):
|
||||
try:
|
||||
response = s3_client.abort_multipart_upload(
|
||||
Bucket=bucket_name, Key=object_key, UploadId=upload_id
|
||||
)
|
||||
log_command_execution("S3 Abort multipart upload", response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Upload part S3")
|
||||
def upload_part_s3(
|
||||
s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str
|
||||
) -> str:
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
file_content = filepath
|
||||
else:
|
||||
with open(filepath, "rb") as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.upload_part(
|
||||
UploadId=upload_id,
|
||||
Bucket=bucket_name,
|
||||
Key=object_key,
|
||||
PartNumber=part_num,
|
||||
Body=file_content,
|
||||
)
|
||||
log_command_execution("S3 Upload part", response)
|
||||
assert response.get("ETag"), f"Expected ETag in response:\n{response}"
|
||||
|
||||
return response.get("ETag")
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("List parts S3")
|
||||
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
|
||||
try:
|
||||
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution("S3 List part", response)
|
||||
assert response.get("Parts"), f"Expected Parts in response:\n{response}"
|
||||
|
||||
return response.get("Parts")
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Complete multipart upload S3")
|
||||
def complete_multipart_upload_s3(
|
||||
s3_client, bucket_name: str, object_key: str, upload_id: str, parts: list
|
||||
):
|
||||
try:
|
||||
parts = [{"ETag": etag, "PartNumber": part_num} for part_num, etag in parts]
|
||||
response = s3_client.complete_multipart_upload(
|
||||
Bucket=bucket_name, Key=object_key, UploadId=upload_id, MultipartUpload={"Parts": parts}
|
||||
)
|
||||
log_command_execution("S3 Complete multipart upload", response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("Put object tagging")
|
||||
def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list):
|
||||
try:
|
||||
tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]
|
||||
tagging = {"TagSet": tags}
|
||||
s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging)
|
||||
log_command_execution("S3 Put object tagging", str(tags))
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during put object tagging: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Get object tagging")
|
||||
def get_object_tagging(s3_client, bucket_name: str, object_key: str) -> list:
|
||||
try:
|
||||
response = s3_client.get_object_tagging(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution("S3 Get object tagging", response)
|
||||
return response.get("TagSet")
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during get object tagging: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Delete object tagging")
|
||||
def delete_object_tagging(s3_client, bucket_name: str, object_key: str):
|
||||
try:
|
||||
response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution("S3 Delete object tagging", response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during delete object tagging: {err}") from err
|
||||
|
||||
|
||||
@allure.step("Get object tagging")
|
||||
def get_object_attributes(
|
||||
s3_client,
|
||||
bucket_name: str,
|
||||
object_key: str,
|
||||
*attributes: str,
|
||||
version_id: str = None,
|
||||
max_parts: int = None,
|
||||
part_number: int = None,
|
||||
get_full_resp=True,
|
||||
) -> dict:
|
||||
try:
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
logger.warning("Method get_object_attributes is not supported by boto3 client")
|
||||
return {}
|
||||
response = s3_client.get_object_attributes(
|
||||
bucket_name,
|
||||
object_key,
|
||||
*attributes,
|
||||
version_id=version_id,
|
||||
max_parts=max_parts,
|
||||
part_number=part_number,
|
||||
)
|
||||
log_command_execution("S3 Get object attributes", response)
|
||||
for attr in attributes:
|
||||
assert attr in response, f"Expected attribute {attr} in {response}"
|
||||
|
||||
if get_full_resp:
|
||||
return response
|
||||
else:
|
||||
return response.get(attributes[0])
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f"Got error during get object attributes: {err}") from err
|
||||
|
||||
|
||||
def _make_objs_dict(key_names):
|
||||
objs_list = []
|
||||
for key in key_names:
|
||||
obj_dict = {"Key": key}
|
||||
objs_list.append(obj_dict)
|
||||
objs_dict = {"Objects": objs_list}
|
||||
return objs_dict
|
||||
|
|
@ -4,7 +4,6 @@
|
|||
This module contains keywords for work with session token.
|
||||
"""
|
||||
|
||||
import allure
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
|
|
@ -18,7 +17,6 @@ from common import ASSETS_DIR, NEOFS_ENDPOINT, WALLET_CONFIG
|
|||
from neo3 import wallet
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
# path to neofs-cli executable
|
||||
NEOFS_CLI_EXEC = os.getenv("NEOFS_CLI_EXEC", "neofs-cli")
|
||||
|
|
|
|||
|
|
@ -16,8 +16,6 @@ from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
|
|||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@allure.step("Get Object Copies")
|
||||
def get_object_copies(complexity: str, wallet: str, cid: str, oid: str):
|
||||
|
|
|
|||
|
|
@ -1,14 +1,11 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import allure
|
||||
import json
|
||||
|
||||
import allure
|
||||
import neofs_verbs
|
||||
from neo3 import wallet
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@allure.step("Verify Head Tombstone")
|
||||
def verify_head_tombstone(wallet_path: str, cid: str, oid_ts: str, oid: str):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue