From f5e23584bfa822cdd626ae2c09f97806890763ba Mon Sep 17 00:00:00 2001 From: Yaroslava Lukoyanova Date: Mon, 3 Feb 2025 12:42:02 +0300 Subject: [PATCH] [#366] Set of presigned URL test cases Signed-off-by: Yaroslava Lukoyanova --- pytest_tests/testsuites/conftest.py | 12 +- .../s3_gate/test_s3_presigned_urls.py | 114 ++++++++++++++++++ 2 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 pytest_tests/testsuites/services/s3_gate/test_s3_presigned_urls.py diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index b23471d5..9ea2841c 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -7,7 +7,7 @@ import allure import pytest from frostfs_testlib import plugins, reporter from frostfs_testlib.cli import FrostfsCli -from frostfs_testlib.clients import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, S3HttpClient +from frostfs_testlib.clients import AwsCliClient, Boto3ClientWrapper, HttpClient, S3ClientWrapper, S3HttpClient from frostfs_testlib.clients.s3 import BucketContainerResolver, VersioningStatus from frostfs_testlib.credentials.interfaces import CredentialsProvider, User from frostfs_testlib.healthcheck.interfaces import Healthcheck @@ -226,6 +226,12 @@ def remote_grpc_client(remote_frostfs_cli: FrostfsCli) -> GrpcClientWrapper: return CliClientWrapper(remote_frostfs_cli) +@reporter.step("[Session]: Create HTTP client") +@pytest.fixture(scope="session") +def http_client() -> HttpClient: + return HttpClient() + + # By default we want all tests to be executed with both storage policies. # This can be overriden in choosen tests if needed. @pytest.fixture(scope="session", params=[pytest.param("rep", marks=pytest.mark.rep), pytest.param("ec", marks=pytest.mark.ec)]) @@ -520,7 +526,7 @@ def container_request(request: pytest.FixtureRequest) -> ContainerRequest: if not container_request: raise RuntimeError( - f"""Container specification is empty. + f"""Container specification is empty. Add @pytest.mark.parametrize("container_request", [ContainerRequest(...)], indirect=True) decorator.""" ) @@ -533,7 +539,7 @@ def multiple_containers_request(request: pytest.FixtureRequest) -> ContainerRequ return request.param raise RuntimeError( - f"""Container specification is empty. + f"""Container specification is empty. Add @pytest.mark.parametrize("container_requests", [[ContainerRequest(...), ..., ContainerRequest(...)]], indirect=True) decorator.""" ) diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_presigned_urls.py b/pytest_tests/testsuites/services/s3_gate/test_s3_presigned_urls.py new file mode 100644 index 00000000..bdc6638c --- /dev/null +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_presigned_urls.py @@ -0,0 +1,114 @@ +import time + +import allure +import pytest +from frostfs_testlib import reporter +from frostfs_testlib.clients import Boto3ClientWrapper, HttpClient, S3ClientWrapper +from frostfs_testlib.clients.s3 import VersioningStatus +from frostfs_testlib.steps import s3_helper +from frostfs_testlib.steps.http_gate import get_via_http_gate +from frostfs_testlib.storage.cluster import ClusterNode +from frostfs_testlib.storage.dataclasses.object_size import ObjectSize +from frostfs_testlib.utils.file_utils import generate_file, get_file_hash + + +@pytest.mark.nightly +@pytest.mark.s3_gate +@pytest.mark.s3_gate_object +class TestS3PresignUrls: + @pytest.mark.parametrize( + "versioning_status", + [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], + ) + @allure.title("Presigned URLs for GET method (s3_client={s3_client}, bucket versioning_status={versioning_status})") + def test_get_presigned_urls( + self, + s3_client: S3ClientWrapper, + bucket: str, + simple_object_size: ObjectSize, + node_under_test: ClusterNode, + versioning_status: VersioningStatus, + ): + s3_helper.set_bucket_versioning(s3_client, bucket, versioning_status) + + with reporter.step("Put object1 into the bucket"): + file_path1 = generate_file(simple_object_size.value) + file_name = s3_helper.object_key_from_file_path(file_path1) + s3_client.put_object(bucket=bucket, filepath=file_path1, key=file_name) + + with reporter.step("Create a presign url to the object without expiration time"): + presigned_url_no_expiration = s3_client.create_presign_url("get_object", bucket, file_name) + with reporter.step("Download the object via HTTP GW"): + # Let's pretend here that bucket is CID and key is OID. They are used only in tmp file name here, request is made via URL + downloaded_object = get_via_http_gate( + cid=bucket, oid=file_name, node=node_under_test, presigned_url=presigned_url_no_expiration + ) + assert get_file_hash(downloaded_object) == get_file_hash(file_path1), f"Objects are not equal" + + with reporter.step("Create a presign url to the object with expiration time of 60 seconds"): + presigned_url_expiration = s3_client.create_presign_url("get_object", bucket, file_name, expires_in=60) + + with reporter.step("Download the object via HTTP GW"): + downloaded_object2 = get_via_http_gate(cid=bucket, oid=file_name, node=node_under_test, presigned_url=presigned_url_expiration) + assert get_file_hash(downloaded_object2) == get_file_hash(file_path1), f"Objects are not equal" + + with reporter.step("Wait for 60 seconds and check that download is not available"): + time.sleep(65) + with pytest.raises(Exception, match=r"403 Forbidden"): + get_via_http_gate(cid=bucket, oid=file_name, node=node_under_test, presigned_url=presigned_url_expiration) + + with reporter.step("Put object2 into the bucket with the same key"): + file_path2 = generate_file(simple_object_size.value) + s3_client.put_object(bucket=bucket, filepath=file_path2, key=file_name) + + with reporter.step("Download the object via HTTP GW and check that it is the second object"): + downloaded_object3 = get_via_http_gate( + cid=bucket, oid=file_name, node=node_under_test, presigned_url=presigned_url_no_expiration + ) + assert get_file_hash(downloaded_object3) == get_file_hash(file_path2), f"Objects are not equal" + + @pytest.mark.parametrize( + "versioning_status", + [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], + ) + @pytest.mark.parametrize("s3_client", [Boto3ClientWrapper], indirect=True) + @allure.title("Presigned URLs for PUT method (s3_client={s3_client}, bucket versioning_status={versioning_status})") + def test_put_presigned_urls( + self, + s3_client: S3ClientWrapper, + bucket: str, + simple_object_size: ObjectSize, + versioning_status: VersioningStatus, + http_client: HttpClient, + ): + s3_helper.set_bucket_versioning(s3_client, bucket, versioning_status) + + with reporter.step("Create presign URL for put object1"): + file_path1 = generate_file(simple_object_size.value) + file_name = s3_helper.object_key_from_file_path(file_path1) + presigned_url = s3_client.create_presign_url("put_object", bucket, file_name) + + with reporter.step("Put object1 into the bucket using presign URL"): + with open(file_path1, "rb") as object_file: + object_read = object_file.read() + http_client.send("PUT", presigned_url, data=object_read) + + with reporter.step("Download the object"): + downloaded_object1 = s3_client.get_object(bucket, file_name) + downloaded_object1_hash = get_file_hash(downloaded_object1) + assert downloaded_object1_hash == get_file_hash(file_path1), f"Objects are not equal" + + with reporter.step("Create presign URL for put object2 with the same key"): + file_path2 = generate_file(simple_object_size.value) + file_name = s3_helper.object_key_from_file_path(file_path2) + presigned_url = s3_client.create_presign_url("put_object", bucket, file_name) + + with reporter.step("Put object2 into the bucket using presign URL"): + with open(file_path2, "rb") as object_file: + object_read = object_file.read() + http_client.send("PUT", presigned_url, data=object_read) + + with reporter.step("Download the object"): + downloaded_object2 = s3_client.get_object(bucket, file_name) + downloaded_object2_hash = get_file_hash(downloaded_object2) + assert downloaded_object2_hash == get_file_hash(file_path2), f"Objects are not equal"