Move shared code to testlib

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/resources/cli.py

@@ -0,0 +1,12 @@
+# Paths to CLI executables on machine that runs tests
+import os
+
+NEOGO_EXECUTABLE = os.getenv("FROSTFS_EXECUTABLE", "neo-go")
+FROSTFS_CLI_EXEC = os.getenv("FROSTFS_CLI_EXEC", "frostfs-cli")
+FROSTFS_AUTHMATE_EXEC = os.getenv("FROSTFS_AUTHMATE_EXEC", "frostfs-s3-authmate")
+FROSTFS_ADM_EXEC = os.getenv("FROSTFS_ADM_EXEC", "frostfs-adm")
+
+# Config for frostfs-adm utility. Optional if tests are running against devenv
+FROSTFS_ADM_CONFIG_PATH = os.getenv("FROSTFS_ADM_CONFIG_PATH")
+
+CLI_DEFAULT_TIMEOUT = os.getenv("CLI_DEFAULT_TIMEOUT", None)

src/frostfs_testlib/resources/common.py

@@ -1,37 +1,46 @@
-# ACLs with final flag
-PUBLIC_ACL_F = "1FBFBFFF"
-PRIVATE_ACL_F = "1C8C8CCC"
-READONLY_ACL_F = "1FBF8CFF"
+import os
 
-# ACLs without final flag set
-PUBLIC_ACL = "0FBFBFFF"
-INACCESSIBLE_ACL = "40000000"
-STICKY_BIT_PUB_ACL = "3FFFFFFF"
+import yaml
 
-EACL_PUBLIC_READ_WRITE = "eacl-public-read-write"
+CONTAINER_WAIT_INTERVAL = "1m"
 
-# Regex patterns of status codes of Container service
-CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
+SIMPLE_OBJECT_SIZE = os.getenv("SIMPLE_OBJECT_SIZE", "1000")
+COMPLEX_OBJECT_CHUNKS_COUNT = os.getenv("COMPLEX_OBJECT_CHUNKS_COUNT", "3")
+COMPLEX_OBJECT_TAIL_SIZE = os.getenv("COMPLEX_OBJECT_TAIL_SIZE", "1000")
 
+SERVICE_MAX_STARTUP_TIME = os.getenv("SERVICE_MAX_STARTUP_TIME", "5m")
 
-# Regex patterns of status codes of Object service
-MALFORMED_REQUEST = "code = 1024.*message = malformed request"
-OBJECT_ACCESS_DENIED = "code = 2048.*message = access to object operation denied"
-OBJECT_NOT_FOUND = "code = 2049.*message = object not found"
-OBJECT_ALREADY_REMOVED = "code = 2052.*message = object already removed"
-SESSION_NOT_FOUND = "code = 4096.*message = session token not found"
-OUT_OF_RANGE = "code = 2053.*message = out of range"
-EXPIRED_SESSION_TOKEN = "code = 4097.*message = expired session token"
-# TODO: Due to https://github.com/nspcc-dev/neofs-node/issues/2092 we have to check only codes until fixed
-# OBJECT_IS_LOCKED = "code = 2050.*message = object is locked"
-# LOCK_NON_REGULAR_OBJECT = "code = 2051.*message = ..." will be available once 2092 is fixed
-OBJECT_IS_LOCKED = "code = 2050"
-LOCK_NON_REGULAR_OBJECT = "code = 2051"
+MORPH_TIMEOUT = os.getenv("MORPH_BLOCK_TIME", "8s")
+MORPH_BLOCK_TIME = os.getenv("MORPH_BLOCK_TIME", "1s")
+FROSTFS_CONTRACT_CACHE_TIMEOUT = os.getenv("FROSTFS_CONTRACT_CACHE_TIMEOUT", "30s")
 
-LIFETIME_REQUIRED = "either expiration epoch of a lifetime is required"
-LOCK_OBJECT_REMOVAL = "lock object removal"
-LOCK_OBJECT_EXPIRATION = "lock object expiration: {expiration_epoch}; current: {current_epoch}"
-INVALID_RANGE_ZERO_LENGTH = "invalid '{range}' range: zero length"
-INVALID_RANGE_OVERFLOW = "invalid '{range}' range: uint64 overflow"
-INVALID_OFFSET_SPECIFIER = "invalid '{range}' range offset specifier"
-INVALID_LENGTH_SPECIFIER = "invalid '{range}' range length specifier"
+# Time interval that allows a GC pass on storage node (this includes GC sleep interval
+# of 1min plus 15 seconds for GC pass itself)
+STORAGE_GC_TIME = os.getenv("STORAGE_GC_TIME", "75s")
+
+GAS_HASH = os.getenv("GAS_HASH", "0xd2a4cff31913016155e38e474a2c06d08be276cf")
+FROSTFS_CONTRACT = os.getenv("FROSTFS_IR_CONTRACTS_FROSTFS")
+
+ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir")
+
+# Password of wallet owned by user on behalf of whom we are running tests
+# Default wallet password is empty
+DEFAULT_WALLET_PASS = os.getenv("WALLET_PASS", "")
+
+# Artificial delay that we add after object deletion and container creation
+# Delay is added because sometimes immediately after deletion object still appears
+# to be existing (probably because tombstone object takes some time to replicate)
+# TODO: remove this wait
+S3_SYNC_WAIT_TIME = 5
+
+# Generate wallet config
+# TODO: we should move all info about wallet configs to fixtures
+DEFAULT_WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")
+with open(DEFAULT_WALLET_CONFIG, "w") as file:
+    yaml.dump({"password": DEFAULT_WALLET_PASS}, file)
+
+# Number of attempts that S3 clients will attempt per each request (1 means single attempt
+# without any retries)
+MAX_REQUEST_ATTEMPTS = 1
+RETRY_MODE = "standard"
+CREDENTIALS_CREATE_TIMEOUT = "1m"

src/frostfs_testlib/resources/error_patterns.py

@@ -0,0 +1,28 @@
+# Regex patterns of status codes of Container service
+CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
+
+# Regex patterns of status codes of Object service
+MALFORMED_REQUEST = "code = 1024.*message = malformed request"
+OBJECT_ACCESS_DENIED = "code = 2048.*message = access to object operation denied"
+OBJECT_NOT_FOUND = "code = 2049.*message = object not found"
+OBJECT_ALREADY_REMOVED = "code = 2052.*message = object already removed"
+SESSION_NOT_FOUND = "code = 4096.*message = session token not found"
+OUT_OF_RANGE = "code = 2053.*message = out of range"
+EXPIRED_SESSION_TOKEN = "code = 4097.*message = expired session token"
+# TODO: Change to codes with message
+# OBJECT_IS_LOCKED = "code = 2050.*message = object is locked"
+# LOCK_NON_REGULAR_OBJECT = "code = 2051.*message = ..." will be available once 2092 is fixed
+OBJECT_IS_LOCKED = "code = 2050"
+LOCK_NON_REGULAR_OBJECT = "code = 2051"
+
+LIFETIME_REQUIRED = "either expiration epoch of a lifetime is required"
+LOCK_OBJECT_REMOVAL = "lock object removal"
+LOCK_OBJECT_EXPIRATION = "lock object expiration: {expiration_epoch}; current: {current_epoch}"
+INVALID_RANGE_ZERO_LENGTH = "invalid '{range}' range: zero length"
+INVALID_RANGE_OVERFLOW = "invalid '{range}' range: uint64 overflow"
+INVALID_OFFSET_SPECIFIER = "invalid '{range}' range offset specifier"
+INVALID_LENGTH_SPECIFIER = "invalid '{range}' range length specifier"
+
+S3_MALFORMED_XML_REQUEST = (
+    "The XML you provided was not well-formed or did not validate against our published schema."
+)

src/frostfs_testlib/resources/load_params.py

@@ -0,0 +1,30 @@
+import os
+
+# Background load node parameters
+LOAD_NODES = os.getenv("LOAD_NODES", "").split()
+# Must hardcode for now
+LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER", "service")
+LOAD_NODE_SSH_PASSWORD = os.getenv("LOAD_NODE_SSH_PASSWORD")
+LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH")
+LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE")
+BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 4)
+BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 4)
+BACKGROUND_DELETERS_COUNT = os.getenv("BACKGROUND_DELETERS_COUNT", 0)
+BACKGROUND_LOAD_DEFAULT_TIME = os.getenv("BACKGROUND_LOAD_DEFAULT_TIME", 600)
+BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE = os.getenv("BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE", 32)
+
+# This will decrease load params for some weak environments
+BACKGROUND_LOAD_VUS_COUNT_DIVISOR = os.getenv("BACKGROUND_LOAD_VUS_COUNT_DIVISOR", 1)
+
+# Wait for 1 hour for xk6 verify scenario by default (in practice means like "unlimited" time)
+BACKGROUND_LOAD_MAX_VERIFY_TIME = os.getenv("BACKGROUND_LOAD_VERIFY_MAX_TIME", 3600)
+BACKGROUND_LOAD_CONTAINER_PLACEMENT_POLICY = os.getenv(
+    "BACKGROUND_LOAD_CONTAINER_PLACEMENT_POLICY", "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
+)
+BACKGROUND_LOAD_S3_LOCATION = os.getenv("BACKGROUND_LOAD_S3_LOCATION", "node-off")
+PRESET_CONTAINERS_COUNT = os.getenv("CONTAINERS_COUNT", "40")
+# TODO: At lease one object is required due to bug in xk6 (buckets with no objects produce millions exceptions in read)
+PRESET_OBJECTS_COUNT = os.getenv("OBJ_COUNT", "10")
+K6_DIRECTORY = os.getenv("K6_DIRECTORY", "/etc/k6")
+K6_TEARDOWN_PERIOD = os.getenv("K6_TEARDOWN_PERIOD", "30")
+LOAD_CONFIG_YAML_PATH = os.getenv("LOAD_CONFIG_YAML_PATH", "load_config_yaml_file.yml")

src/frostfs_testlib/resources/optionals.py

@@ -0,0 +1,26 @@
+import os
+
+
+def str_to_bool(input: str) -> bool:
+    return input in ["true", "True", "1"]
+
+
+# Override these optional params to not comment/modify code during local development. Use with caution.
+# Node under test. Set this to occupy exact node.
+OPTIONAL_NODE_UNDER_TEST = os.getenv("OPTIONAL_NODE_UNDER_TEST")
+
+# Node under load. Set this to target load on exact node.
+OPTIONAL_NODE_UNDER_LOAD = os.getenv("OPTIONAL_NODE_UNDER_LOAD")
+
+# Set this to True to disable failover commands. I.E. node which supposed to be stopped will not be actually stopped.
+OPTIONAL_FAILOVER_ENABLED = str_to_bool(os.getenv("OPTIONAL_FAILOVER_ENABLED", "true"))
+
+# Set this to True to disable background load. I.E. node which supposed to be stopped will not be actually stopped.
+OPTIONAL_BACKGROUND_LOAD_ENABLED = str_to_bool(
+    os.getenv("OPTIONAL_BACKGROUND_LOAD_ENABLED", "true")
+)
+
+# Set this to False for disable autouse fixture like node healthcheck during developing time.
+OPTIONAL_AUTOUSE_FIXTURES_ENABLED = str_to_bool(
+    os.getenv("OPTIONAL_AUTOUSE_FIXTURES_ENABLED", "true")
+)

src/frostfs_testlib/resources/wellknown_acl.py

@@ -0,0 +1,11 @@
+# ACLs with final flag
+PUBLIC_ACL_F = "1FBFBFFF"
+PRIVATE_ACL_F = "1C8C8CCC"
+READONLY_ACL_F = "1FBF8CFF"
+
+# ACLs without final flag set
+PUBLIC_ACL = "0FBFBFFF"
+INACCESSIBLE_ACL = "40000000"
+STICKY_BIT_PUB_ACL = "3FFFFFFF"
+
+EACL_PUBLIC_READ_WRITE = "eacl-public-read-write"