Fix s3_client fixture

Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>

.devenv.hosting.yaml

@@ -14,8 +14,6 @@ hosts:
       endpoint_data0: s01.frostfs.devenv:8080
       control_endpoint: s01.frostfs.devenv:8081
       un_locode: "RU MOW"
-      http_hostname: "no_hostname"
-      s3_hostname: "no_hostname"
   - name: s02
     attributes:
       container_name: s02
@@ -28,8 +26,6 @@ hosts:
       endpoint_data0: s02.frostfs.devenv:8080
       control_endpoint: s02.frostfs.devenv:8081
       un_locode: "RU LED"
-      http_hostname: "no_hostname"
-      s3_hostname: "no_hostname"
   - name: s03
     attributes:
       container_name: s03
@@ -42,8 +38,6 @@ hosts:
       endpoint_data0: s03.frostfs.devenv:8080
       control_endpoint: s03.frostfs.devenv:8081
       un_locode: "SE STO"
-      http_hostname: "no_hostname"
-      s3_hostname: "no_hostname"
   - name: s04
     attributes:
       container_name: s04
@@ -56,8 +50,6 @@ hosts:
       endpoint_data0: s04.frostfs.devenv:8080
       control_endpoint: s04.frostfs.devenv:8081
       un_locode: "FI HEL"
-      http_hostname: "no_hostname"
-      s3_hostname: "no_hostname"
   - name: s3-gate01
     attributes:
       container_name: s3_gate

.gitignore

@@ -3,7 +3,7 @@
 .idea
 
 .DS_Store
-venv.*
+
 venv_macos
 
 

Makefile

@@ -4,39 +4,26 @@ VENV_NAME = frostfs-testcases-${PYTHON_VERSION}
 VENV_DIR := venv.${VENV_NAME}
 
 current_dir := $(shell pwd)
-FROM_VENV := . ${VENV_DIR}/bin/activate &&
 
 venv: create requirements paths precommit
 	@echo Ready
 
 precommit:
 	@echo Isntalling pre-commit hooks
-	${FROM_VENV} pre-commit install
+	. ${VENV_DIR}/bin/activate && pre-commit install
 
 paths:
 	@echo Append paths for project
 	@echo Virtual environment: ${VENV_DIR}
-	@rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
-	@touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
-	@echo ${current_dir} | tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
+	@sudo rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
+	@sudo touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
+	@echo ${current_dir} | sudo tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
 
-create: ${VENV_DIR}
-
-${VENV_DIR}:
-	@echo Create virtual environment ${VENV_DIR}
-	virtualenv --python=python${PYTHON_VERSION} --prompt=frostfs-testlib ${VENV_DIR}
+create:
+	@echo Create virtual environment for 
+	virtualenv --python=python${PYTHON_VERSION} --prompt=${VENV_NAME} ${VENV_DIR}
 
 requirements:
 	@echo Isntalling pip requirements
-	${FROM_VENV} pip install -e ../frostfs-testlib
-	${FROM_VENV} pip install -Ur requirements.txt
-	${FROM_VENV} pip install -Ur requirements_dev.txt
-
-
-#### VALIDATION SECTION ####
-lint: create requirements
-	${FROM_VENV} pip install -e ../frostfs-testlib;
-	${FROM_VENV} pylint --disable R,C,W pytest_tests
-
-validation: lint
-	${FROM_VENV} pytest --collect-only
+	. ${VENV_DIR}/bin/activate && pip install -e ../frostfs-testlib
+	. ${VENV_DIR}/bin/activate && pip install -Ur pytest_tests/requirements.txt

pytest_tests/testsuites/conftest.py

@@ -165,6 +165,7 @@ def s3_client(
     cluster: Cluster,
     request: pytest.FixtureRequest,
 ) -> S3ClientWrapper:
+    s3_bearer_rules_file = f"{os.getcwd()}/pytest_tests/resources/files/s3_bearer_rules.json"
     wallet = WalletInfo(path=default_wallet, password=DEFAULT_WALLET_PASS)
 
     (cid, access_key_id, secret_access_key) = s3_helper.init_s3_credentials(
@@ -172,6 +173,7 @@ def s3_client(
         client_shell,
         cluster,
         s3gates=[cluster_node.s3_gate for cluster_node in cluster.cluster_nodes],
+        s3_bearer_rules_file=s3_bearer_rules_file,
         policy=s3_policy,
     )
     containers_list = list_containers(

pytest_tests/testsuites/failovers/test_failover_network.py

@@ -28,11 +28,11 @@ blocked_nodes: list[StorageNode] = []
 @pytest.mark.failover_network
 class TestFailoverNetwork(ClusterTestBase):
     @pytest.fixture(autouse=True)
-    @allure.title("Restore network")
+    @allure.step("Restore network")
     def restore_network(self):
         yield
-        with allure.step(f"Count blocked nodes {len(blocked_nodes)}"):
-            not_empty = len(blocked_nodes) != 0
+
+        not_empty = len(blocked_nodes) != 0
         for node in list(blocked_nodes):
             with allure.step(f"Restore network at host for {node.label}"):
                 IpTablesHelper.restore_input_traffic_to_port(node.host.get_shell(), PORTS_TO_BLOCK)
@@ -42,10 +42,7 @@ class TestFailoverNetwork(ClusterTestBase):
 
     @allure.title("Block Storage node traffic")
     def test_block_storage_node_traffic(
-        self,
-        default_wallet: str,
-        require_multiple_hosts,
-        simple_object_size: int,
+        self, default_wallet: str, require_multiple_hosts, simple_object_size: int
     ):
         """
         Block storage nodes traffic using iptables and wait for replication for objects.

pytest_tests/testsuites/services/http_gate/test_http_bearer.py

@@ -20,7 +20,6 @@ from frostfs_testlib.utils.file_utils import generate_file
 logger = logging.getLogger("NeoLogger")
 
 
-@pytest.mark.skip("Temporarly skip for v0.37")
 @pytest.mark.sanity
 @pytest.mark.http_gate
 class Test_http_bearer(ClusterTestBase):
@@ -92,7 +91,6 @@ class Test_http_bearer(ClusterTestBase):
             error_pattern="access to object operation denied",
         )
 
-    @pytest.mark.skip("Temp disable for v0.37")
     @pytest.mark.parametrize(
         "object_size",
         [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],

pytest_tests/testsuites/services/http_gate/test_http_gate.py

@@ -94,13 +94,11 @@ class TestHttpGate(ClusterTestBase):
                 shell=self.shell,
                 nodes=self.cluster.storage_nodes,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )
 
     @allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
     @allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
     @allure.title("Test Put over HTTP, Get over HTTP")
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     @pytest.mark.smoke
     def test_put_http_get_http(self, complex_object_size: int, simple_object_size: int):
         """
@@ -143,14 +141,12 @@ class TestHttpGate(ClusterTestBase):
                 shell=self.shell,
                 nodes=self.cluster.storage_nodes,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                hostname=self.cluster.default_http_hostname,
             )
 
     @allure.link(
         "https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes",
         name="download by attributes",
     )
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     @allure.title("Test Put over HTTP, Get over HTTP with headers")
     @pytest.mark.parametrize(
         "attributes",
@@ -198,10 +194,8 @@ class TestHttpGate(ClusterTestBase):
             cid=cid,
             attrs=attributes,
             endpoint=self.cluster.default_http_gate_endpoint,
-            http_hostname=self.cluster.default_http_hostname,
         )
 
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     @allure.title("Test Expiration-Epoch in HTTP header")
     @pytest.mark.parametrize("epoch_gap", [0, 1])
     def test_expiration_epoch_in_http(self, simple_object_size: int, epoch_gap):
@@ -226,23 +220,14 @@ class TestHttpGate(ClusterTestBase):
 
             with allure.step("Put objects using HTTP with attribute Expiration-Epoch"):
                 oid = upload_via_http_gate(
-                    cid=cid,
-                    path=file_path,
-                    headers=headers,
-                    endpoint=http_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
+                    cid=cid, path=file_path, headers=headers, endpoint=http_endpoint
                 )
                 if get_epoch(self.shell, self.cluster) + 1 <= valid_until:
                     oids_to_be_valid.append(oid)
                 else:
                     oids_to_be_expired.append(oid)
                 with allure.step("This object can be got"):
-                    get_via_http_gate(
-                        cid=cid,
-                        oid=oid,
-                        endpoint=http_endpoint,
-                        http_hostname=self.cluster.default_http_hostname,
-                    )
+                    get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint)
 
         self.tick_epoch()
 
@@ -256,18 +241,11 @@ class TestHttpGate(ClusterTestBase):
                     oid=oid,
                     error_pattern=OBJECT_NOT_FOUND_ERROR,
                     endpoint=self.cluster.default_http_gate_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
                 )
         for oid in oids_to_be_valid:
             with allure.step(f"{oid} shall be valid and can be got"):
-                get_via_http_gate(
-                    cid=cid,
-                    oid=oid,
-                    endpoint=http_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
-                )
+                get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint)
 
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     @allure.title("Test Zip in HTTP header")
     def test_zip_in_http(self, complex_object_size: int, simple_object_size: int):
         cid = create_container(
@@ -299,10 +277,7 @@ class TestHttpGate(ClusterTestBase):
         )
 
         dir_path = get_via_zip_http_gate(
-            cid=cid,
-            prefix=common_prefix,
-            endpoint=self.cluster.default_http_gate_endpoint,
-            http_hostname=self.cluster.default_http_hostname,
+            cid=cid, prefix=common_prefix, endpoint=self.cluster.default_http_gate_endpoint
         )
 
         with allure.step("Verify hashes"):
@@ -310,7 +285,6 @@ class TestHttpGate(ClusterTestBase):
             assert get_file_hash(f"{dir_path}/file2") == get_file_hash(file_path_large)
 
     @pytest.mark.long
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object")
     def test_put_http_get_http_large_file(self, complex_object_size: int):
         """
@@ -346,7 +320,6 @@ class TestHttpGate(ClusterTestBase):
             shell=self.shell,
             nodes=self.cluster.storage_nodes,
             endpoint=self.cluster.default_http_gate_endpoint,
-            hostname=self.cluster.default_http_hostname,
         )
         verify_object_hash(
             oid=oid_curl,
@@ -356,11 +329,9 @@ class TestHttpGate(ClusterTestBase):
             shell=self.shell,
             nodes=self.cluster.storage_nodes,
             endpoint=self.cluster.default_http_gate_endpoint,
-            http_hostname=self.cluster.default_http_hostname,
             object_getter=get_via_http_curl,
         )
 
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     @allure.title("Test Put/Get over HTTP using Curl utility")
     def test_put_http_get_http_curl(self, complex_object_size: int, simple_object_size: int):
         """
@@ -396,6 +367,5 @@ class TestHttpGate(ClusterTestBase):
                 shell=self.shell,
                 nodes=self.cluster.storage_nodes,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
                 object_getter=get_via_http_curl,
             )

pytest_tests/testsuites/services/http_gate/test_http_headers.py

@@ -47,12 +47,9 @@ class Test_http_headers(ClusterTestBase):
     @pytest.fixture(
         params=[
             pytest.lazy_fixture("simple_object_size"),
-            # TODO: Temp disable for v0.37
-            # pytest.lazy_fixture("complex_object_size"),
+            pytest.lazy_fixture("complex_object_size"),
         ],
-        # TODO: Temp disable for v0.37
-        # ids=["simple object", "complex object"],
-        ids=["simple object"],
+        ids=["simple object", "complex object"],
         scope="class",
     )
     def storage_objects_with_attributes(self, request: FixtureRequest) -> list[StorageObjectInfo]:
@@ -105,7 +102,6 @@ class Test_http_headers(ClusterTestBase):
                 cid=storage_object_1.cid,
                 attrs={"Chapter2": storage_object_1.attributes["Chapter2"]},
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )
 
     @allure.title("Test get object2 with different attributes, then delete object2 and get object1")
@@ -139,7 +135,6 @@ class Test_http_headers(ClusterTestBase):
                     cid=storage_object_2.cid,
                     attrs=attributes,
                     endpoint=self.cluster.default_http_gate_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
                 )
         with allure.step("Delete object#2 and verify is the container deleted"):
             delete_object(
@@ -154,7 +149,6 @@ class Test_http_headers(ClusterTestBase):
                 oid=storage_object_2.oid,
                 error_pattern=OBJECT_ALREADY_REMOVED_ERROR,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )
             storage_objects_with_attributes.remove(storage_object_2)
 
@@ -168,11 +162,9 @@ class Test_http_headers(ClusterTestBase):
                 cid=storage_object_1.cid,
                 attrs=key_value_pair,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )
 
     @allure.title("[Negative] Try to put object and get right after container is deleted")
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     def test_negative_put_and_get_object3(
         self, storage_objects_with_attributes: list[StorageObjectInfo]
     ):
@@ -232,5 +224,4 @@ class Test_http_headers(ClusterTestBase):
                 attrs=attrs_obj3,
                 http_request_path=request,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )

pytest_tests/testsuites/services/http_gate/test_http_object.py

@@ -91,7 +91,6 @@ class Test_http_object(ClusterTestBase):
                 shell=self.shell,
                 nodes=self.cluster.storage_nodes,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )
         with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"):
             attrs = {obj_key1: obj_value1, obj_key2: obj_value2}
@@ -104,7 +103,6 @@ class Test_http_object(ClusterTestBase):
                 http_request_path=request,
                 attrs=attrs,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                http_hostname=self.cluster.default_http_hostname,
             )
 
         with allure.step(
@@ -116,7 +114,6 @@ class Test_http_object(ClusterTestBase):
                 cid=cid,
                 attrs=attrs,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                hostname=self.cluster.default_http_hostname,
             )
         with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
             request = f"/get_by_attribute/{cid}/{oid}"
@@ -126,5 +123,4 @@ class Test_http_object(ClusterTestBase):
                 error_pattern=expected_err_msg,
                 http_request_path=request,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                hostname=self.cluster.default_http_hostname,
             )

pytest_tests/testsuites/services/http_gate/test_http_streaming.py

@@ -27,7 +27,6 @@ class Test_http_streaming(ClusterTestBase):
         [pytest.lazy_fixture("complex_object_size")],
         ids=["complex object"],
     )
-    @pytest.mark.skip("Skipped due to deprecated PUT via http")
     def test_object_can_be_put_get_by_streaming(self, object_size: int):
         """
         Test that object can be put using gRPC interface and get using HTTP.
@@ -67,5 +66,4 @@ class Test_http_streaming(ClusterTestBase):
                 shell=self.shell,
                 nodes=self.cluster.storage_nodes,
                 endpoint=self.cluster.default_http_gate_endpoint,
-                hostname=self.cluster.default_http_hostname,
             )

pytest_tests/testsuites/services/http_gate/test_http_system_header.py

@@ -130,7 +130,6 @@ class Test_http_system_header(ClusterTestBase):
             shell=self.shell,
             nodes=self.cluster.storage_nodes,
             endpoint=self.cluster.default_http_gate_endpoint,
-            hostname=self.cluster.default_http_hostname,
         )
         head = head_object(
             wallet=self.wallet,
@@ -210,7 +209,6 @@ class Test_http_system_header(ClusterTestBase):
         [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
         ids=["simple object", "complex object"],
     )
-    @pytest.mark.skip("Temp disable for v0.37")
     def test_http_attr_priority_epoch_duration(
         self, user_container: str, object_size: int, epoch_duration: int
     ):
@@ -243,7 +241,6 @@ class Test_http_system_header(ClusterTestBase):
                     oid=oid,
                     error_pattern="404 Not Found",
                     endpoint=self.cluster.default_http_gate_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
                 )
                 # check that object is not available via grpc
                 with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@@ -259,7 +256,6 @@ class Test_http_system_header(ClusterTestBase):
         [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
         ids=["simple object", "complex object"],
     )
-    @pytest.mark.skip("Temp disable for v0.37")
     def test_http_attr_priority_dur_timestamp(
         self, user_container: str, object_size: int, epoch_duration: int
     ):
@@ -299,7 +295,6 @@ class Test_http_system_header(ClusterTestBase):
                     oid=oid,
                     error_pattern="404 Not Found",
                     endpoint=self.cluster.default_http_gate_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
                 )
                 # check that object is not available via grpc
                 with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@@ -315,7 +310,6 @@ class Test_http_system_header(ClusterTestBase):
         [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
         ids=["simple object", "complex object"],
     )
-    @pytest.mark.skip("Temp disable for v0.37")
     def test_http_attr_priority_timestamp_rfc(
         self, user_container: str, object_size: int, epoch_duration: int
     ):
@@ -355,7 +349,6 @@ class Test_http_system_header(ClusterTestBase):
                     oid=oid,
                     error_pattern="404 Not Found",
                     endpoint=self.cluster.default_http_gate_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
                 )
                 # check that object is not available via grpc
                 with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
@@ -366,11 +359,8 @@ class Test_http_system_header(ClusterTestBase):
     @allure.title("Test that object is automatically delete when expiration passed")
     @pytest.mark.parametrize(
         "object_size",
-        # TODO: Temp disabled for v0.37
-        # [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
-        [pytest.lazy_fixture("simple_object_size")],
-        # ids=["simple object", "complex object"],
-        ids=["simple object"],
+        [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
+        ids=["simple object", "complex object"],
     )
     def test_http_rfc_object_unavailable_after_expir(
         self, user_container: str, object_size: int, epoch_duration: int
@@ -411,7 +401,6 @@ class Test_http_system_header(ClusterTestBase):
                     oid=oid,
                     error_pattern="404 Not Found",
                     endpoint=self.cluster.default_http_gate_endpoint,
-                    http_hostname=self.cluster.default_http_hostname,
                 )
                 # check that object is not available via grpc
                 with pytest.raises(Exception, match=OBJECT_NOT_FOUND):

pytest_tests/testsuites/special/test_logs.py

@@ -18,7 +18,7 @@ class TestLogs(ClusterTestBase):
         end_time = datetime.utcnow()
         logs_dir = os.path.join(temp_directory, "logs")
         os.makedirs(logs_dir)
-        issues_regex = r"\Wpanic\W|\Woom\W|\Wtoo many\W|\Winsufficient funds\W"
+        issues_regex = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
 
         hosts_with_problems = []
         for host in self.cluster.hosts:

requirements_dev.txt

@@ -1,3 +1,2 @@
 pre-commit==2.20.0
-isort==5.12.0
-pylint==2.17.4
+isort==5.12.0