Fix s3 tests

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-03-29 19:59:14 +03:00
parent 4779d2be88
commit b995bfca41

View file

@ -6,6 +6,7 @@ from random import choices, sample
import allure import allure
import pytest import pytest
from frostfs_testlib.utils import wallet_utils
from pytest_tests.helpers.aws_cli_client import AwsCliClient from pytest_tests.helpers.aws_cli_client import AwsCliClient
from pytest_tests.helpers.file_helper import ( from pytest_tests.helpers.file_helper import (
@ -14,14 +15,13 @@ from pytest_tests.helpers.file_helper import (
generate_file_with_content, generate_file_with_content,
get_file_hash, get_file_hash,
) )
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
from pytest_tests.helpers.s3_helper import ( from pytest_tests.helpers.s3_helper import (
assert_object_lock_mode, assert_object_lock_mode,
assert_s3_acl, assert_s3_acl,
check_objects_in_bucket, check_objects_in_bucket,
set_bucket_versioning, set_bucket_versioning,
) )
from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS from pytest_tests.resources.common import ASSETS_DIR, WALLET_PASS
from pytest_tests.steps import s3_gate_bucket, s3_gate_object from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase from pytest_tests.steps.s3_gate_base import TestS3GateBase
@ -39,6 +39,13 @@ class TestS3GateObject(TestS3GateBase):
def object_key_from_file_path(full_path: str) -> str: def object_key_from_file_path(full_path: str) -> str:
return os.path.basename(full_path) return os.path.basename(full_path)
@pytest.fixture
def second_wallet_public_key(self):
second_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
wallet_utils.init_wallet(second_wallet, WALLET_PASS)
public_key = wallet_utils.get_wallet_public_key(second_wallet, WALLET_PASS)
yield public_key
@allure.title("Test S3: Copy object") @allure.title("Test S3: Copy object")
def test_s3_copy_object(self, two_buckets, simple_object_size): def test_s3_copy_object(self, two_buckets, simple_object_size):
file_path = generate_file(simple_object_size) file_path = generate_file(simple_object_size)
@ -131,7 +138,7 @@ class TestS3GateObject(TestS3GateBase):
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
check_objects_in_bucket(self.s3_client, bucket, [obj_key]) check_objects_in_bucket(self.s3_client, bucket, [obj_key])
with allure.step("Copy object and check acl attribute"): with allure.step("Copy object and check acl attribute"):
@ -202,7 +209,7 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
version_id_1 = s3_gate_object.put_object_tagging( s3_gate_object.put_object_tagging(
self.s3_client, bucket, file_name_simple, tags=object_tagging self.s3_client, bucket, file_name_simple, tags=object_tagging
) )
bucket_1_objects = [file_name_simple] bucket_1_objects = [file_name_simple]
@ -267,7 +274,7 @@ class TestS3GateObject(TestS3GateBase):
assert obj_versions == { assert obj_versions == {
version_id_1, version_id_1,
version_id_2, version_id_2,
}, f"Expected object has versions: {version_id_1, version_id_2}" }, f"Object should have versions: {version_id_1, version_id_2}"
with allure.step("Delete 1 version of object"): with allure.step("Delete 1 version of object"):
delete_obj = s3_gate_object.delete_object_s3( delete_obj = s3_gate_object.delete_object_s3(
@ -277,8 +284,8 @@ class TestS3GateObject(TestS3GateBase):
obj_versions = { obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key version.get("VersionId") for version in versions if version.get("Key") == obj_key
} }
assert obj_versions == {version_id_2}, f"Expected object has versions: {version_id_2}" assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found" assert "DeleteMarkers" not in delete_obj.keys(), "Delete markers not found"
with allure.step("Delete second version of object"): with allure.step("Delete second version of object"):
delete_obj = s3_gate_object.delete_object_s3( delete_obj = s3_gate_object.delete_object_s3(
@ -289,18 +296,18 @@ class TestS3GateObject(TestS3GateBase):
version.get("VersionId") for version in versions if version.get("Key") == obj_key version.get("VersionId") for version in versions if version.get("Key") == obj_key
} }
assert not obj_versions, "Expected object not found" assert not obj_versions, "Expected object not found"
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found" assert "DeleteMarkers" not in delete_obj.keys(), "Delete markers not found"
with allure.step("Put new object into bucket"): with allure.step("Put new object into bucket"):
file_name_simple = generate_file(complex_object_size) file_name_simple = generate_file(complex_object_size)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
with allure.step("Delete last object"): with allure.step("Delete last object"):
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key) delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key)
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket, True) versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket, True)
assert versions.get("DeleteMarkers", None), f"Expected delete Marker" assert versions.get("DeleteMarkers", None), "Expected delete Marker"
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Test S3: bulk delete version of object") @allure.title("Test S3: bulk delete version of object")
def test_s3_bulk_delete_versioning(self, bucket, simple_object_size): def test_s3_bulk_delete_versioning(self, bucket, simple_object_size):
@ -334,7 +341,7 @@ class TestS3GateObject(TestS3GateBase):
obj_versions = { obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key version.get("VersionId") for version in versions if version.get("Key") == obj_key
} }
assert obj_versions == version_ids, f"Expected object has versions: {version_ids}" assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
with allure.step("Delete two objects from bucket one by one"): with allure.step("Delete two objects from bucket one by one"):
version_to_delete_b1 = sample( version_to_delete_b1 = sample(
@ -351,7 +358,7 @@ class TestS3GateObject(TestS3GateBase):
] ]
assert ( assert (
obj_versions.sort() == version_to_save.sort() obj_versions.sort() == version_to_save.sort()
), f"Expected object has versions: {version_to_save}" ), f"Object should have versions: {version_to_save}"
@allure.title("Test S3: Get versions of object") @allure.title("Test S3: Get versions of object")
def test_s3_get_versioning(self, bucket, simple_object_size): def test_s3_get_versioning(self, bucket, simple_object_size):
@ -531,10 +538,10 @@ class TestS3GateObject(TestS3GateBase):
list_obj = s3_gate_object.list_objects_s3(self.s3_client, bucket) list_obj = s3_gate_object.list_objects_s3(self.s3_client, bucket)
elif list_type == "v2": elif list_type == "v2":
list_obj = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket) list_obj = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket)
assert len(list_obj) == 2, f"bucket have 2 objects" assert len(list_obj) == 2, "bucket should have 2 objects"
assert ( assert (
list_obj.sort() == [file_name, file_name_2].sort() list_obj.sort() == [file_name, file_name_2].sort()
), f"bucket have object key {file_name, file_name_2}" ), f"bucket should have object key {file_name, file_name_2}"
with allure.step("Delete object"): with allure.step("Delete object"):
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name) delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name)
@ -547,9 +554,11 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, full_output=True self.s3_client, bucket, full_output=True
) )
contents = list_obj_1.get("Contents", []) contents = list_obj_1.get("Contents", [])
assert len(contents) == 1, f"bucket have only 1 object" assert len(contents) == 1, "bucket should have only 1 object"
assert contents[0].get("Key") == file_name_2, f"bucket has object key {file_name_2}" assert (
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" contents[0].get("Key") == file_name_2
), f"bucket should have object key {file_name_2}"
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Test S3: put object") @allure.title("Test S3: put object")
def test_s3_put_object(self, bucket, complex_object_size, simple_object_size): def test_s3_put_object(self, bucket, complex_object_size, simple_object_size):
@ -570,7 +579,7 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, file_path_1, Metadata=object_1_metadata, Tagging=tag_1 self.s3_client, bucket, file_path_1, Metadata=object_1_metadata, Tagging=tag_1
) )
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name) obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
assert obj_head.get("Metadata") == object_1_metadata, "Matadata must be the same" assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name) got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
assert got_tags, f"Expected tags, got {got_tags}" assert got_tags, f"Expected tags, got {got_tags}"
assert got_tags == [ assert got_tags == [
@ -583,7 +592,7 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2 self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2
) )
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name) obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
assert obj_head.get("Metadata") == object_2_metadata, "Matadata must be the same" assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same"
got_tags_1 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name) got_tags_1 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
assert got_tags_1, f"Expected tags, got {got_tags_1}" assert got_tags_1, f"Expected tags, got {got_tags_1}"
assert got_tags_1 == [ assert got_tags_1 == [
@ -624,9 +633,9 @@ class TestS3GateObject(TestS3GateBase):
assert obj_versions == { assert obj_versions == {
version_id_1, version_id_1,
version_id_2, version_id_2,
}, f"Expected object has versions: {version_id_1, version_id_2}" }, f"Object should have versions: {version_id_1, version_id_2}"
got_tags_4 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3) got_tags_4 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3)
assert not got_tags_4, f"No expected tags" assert not got_tags_4, "No tags expected"
with allure.step("Get object"): with allure.step("Get object"):
object_3 = s3_gate_object.get_object_s3( object_3 = s3_gate_object.get_object_s3(
@ -652,7 +661,7 @@ class TestS3GateObject(TestS3GateBase):
obj_head_3 = s3_gate_object.head_object_s3( obj_head_3 = s3_gate_object.head_object_s3(
self.s3_client, bucket, file_name_3, version_id_1 self.s3_client, bucket, file_name_3, version_id_1
) )
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same" assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same"
got_tags_3 = s3_gate_object.get_object_tagging( got_tags_3 = s3_gate_object.get_object_tagging(
self.s3_client, bucket, file_name_3, version_id_1 self.s3_client, bucket, file_name_3, version_id_1
) )
@ -669,6 +678,7 @@ class TestS3GateObject(TestS3GateBase):
bucket, bucket,
complex_object_size, complex_object_size,
simple_object_size, simple_object_size,
second_wallet_public_key,
): ):
file_path_1 = generate_file(complex_object_size) file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1) file_name = self.object_key_from_file_path(file_path_1)
@ -717,32 +727,32 @@ class TestS3GateObject(TestS3GateBase):
file_name_5 = self.object_key_from_file_path(file_path_5) file_name_5 = self.object_key_from_file_path(file_path_5)
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"): with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5) generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_path_6, file_path_5,
GrantFullControl=f"id={self.other_public_key}", GrantFullControl=f"id={second_wallet_public_key}",
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5) object_5 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same" assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
with allure.step( with allure.step(
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers" "Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
): ):
file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5) generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_path_7, file_path_5,
GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers", GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5) object_6 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same" assert get_file_hash(file_path_5) == get_file_hash(object_6), "Hashes must be the same"
@allure.title("Test S3: put object with lock-mode") @allure.title("Test S3: put object with lock-mode")
def test_s3_put_object_lock_mode(self, complex_object_size, simple_object_size): def test_s3_put_object_lock_mode(self, complex_object_size, simple_object_size):
@ -772,7 +782,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]" "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
): ):
date_obj = datetime.utcnow() + timedelta(days=2) date_obj = datetime.utcnow() + timedelta(days=2)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1) generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
@ -788,7 +798,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]" "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
): ):
date_obj = datetime.utcnow() + timedelta(days=3) date_obj = datetime.utcnow() + timedelta(days=3)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1) generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,