amaks/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Fix s3 tests

Browse source 
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-03-29 19:59:14 +03:00
parent 4779d2be88
commit b995bfca41
1 changed files with 44 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
pytest_tests/testsuites/services/s3_gate/test_s3_object.py
View file

@ -6,6 +6,7 @@ from random import choices, sample
import allure
import pytest
from frostfs_testlib.utils import wallet_utils
from pytest_tests.helpers.aws_cli_client import AwsCliClient
from pytest_tests.helpers.file_helper import (
@ -14,14 +15,13 @@ from pytest_tests.helpers.file_helper import (
generate_file_with_content,
get_file_hash,
)
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
from pytest_tests.helpers.s3_helper import (
assert_object_lock_mode,
assert_s3_acl,
check_objects_in_bucket,
set_bucket_versioning,
)
from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from pytest_tests.resources.common import ASSETS_DIR, WALLET_PASS
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
@ -39,6 +39,13 @@ class TestS3GateObject(TestS3GateBase):
def object_key_from_file_path(full_path: str) -> str:
return os.path.basename(full_path)
@pytest.fixture
def second_wallet_public_key(self):
second_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
wallet_utils.init_wallet(second_wallet, WALLET_PASS)
public_key = wallet_utils.get_wallet_public_key(second_wallet, WALLET_PASS)
yield public_key
@allure.title("Test S3: Copy object")
def test_s3_copy_object(self, two_buckets, simple_object_size):
file_path = generate_file(simple_object_size)
@ -131,7 +138,7 @@ class TestS3GateObject(TestS3GateBase):
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
check_objects_in_bucket(self.s3_client, bucket, [obj_key])
with allure.step("Copy object and check acl attribute"):
@ -202,7 +209,7 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put several versions of object into bucket"):
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
version_id_1 = s3_gate_object.put_object_tagging(
s3_gate_object.put_object_tagging(
self.s3_client, bucket, file_name_simple, tags=object_tagging
)
bucket_1_objects = [file_name_simple]
@ -267,7 +274,7 @@ class TestS3GateObject(TestS3GateBase):
assert obj_versions == {
version_id_1,
version_id_2,
}, f"Expected object has versions: {version_id_1, version_id_2}"
}, f"Object should have versions: {version_id_1, version_id_2}"
with allure.step("Delete 1 version of object"):
delete_obj = s3_gate_object.delete_object_s3(
@ -277,8 +284,8 @@ class TestS3GateObject(TestS3GateBase):
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert obj_versions == {version_id_2}, f"Expected object has versions: {version_id_2}"
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found"
assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
assert "DeleteMarkers" not in delete_obj.keys(), "Delete markers not found"
with allure.step("Delete second version of object"):
delete_obj = s3_gate_object.delete_object_s3(
@ -289,18 +296,18 @@ class TestS3GateObject(TestS3GateBase):
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert not obj_versions, "Expected object not found"
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found"
assert "DeleteMarkers" not in delete_obj.keys(), "Delete markers not found"
with allure.step("Put new object into bucket"):
file_name_simple = generate_file(complex_object_size)
obj_key = os.path.basename(file_name_simple)
version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
with allure.step("Delete last object"):
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key)
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket, True)
assert versions.get("DeleteMarkers", None), f"Expected delete Marker"
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
assert versions.get("DeleteMarkers", None), "Expected delete Marker"
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Test S3: bulk delete version of object")
def test_s3_bulk_delete_versioning(self, bucket, simple_object_size):
@ -334,7 +341,7 @@ class TestS3GateObject(TestS3GateBase):
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert obj_versions == version_ids, f"Expected object has versions: {version_ids}"
assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
with allure.step("Delete two objects from bucket one by one"):
version_to_delete_b1 = sample(
@ -351,7 +358,7 @@ class TestS3GateObject(TestS3GateBase):
]
assert (
obj_versions.sort() == version_to_save.sort()
), f"Expected object has versions: {version_to_save}"
), f"Object should have versions: {version_to_save}"
@allure.title("Test S3: Get versions of object")
def test_s3_get_versioning(self, bucket, simple_object_size):
@ -531,10 +538,10 @@ class TestS3GateObject(TestS3GateBase):
list_obj = s3_gate_object.list_objects_s3(self.s3_client, bucket)
elif list_type == "v2":
list_obj = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket)
assert len(list_obj) == 2, f"bucket have 2 objects"
assert len(list_obj) == 2, "bucket should have 2 objects"
assert (
list_obj.sort() == [file_name, file_name_2].sort()
), f"bucket have object key {file_name, file_name_2}"
), f"bucket should have object key {file_name, file_name_2}"
with allure.step("Delete object"):
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name)
@ -547,9 +554,11 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, full_output=True
)
contents = list_obj_1.get("Contents", [])
assert len(contents) == 1, f"bucket have only 1 object"
assert contents[0].get("Key") == file_name_2, f"bucket has object key {file_name_2}"
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
assert len(contents) == 1, "bucket should have only 1 object"
assert (
contents[0].get("Key") == file_name_2
), f"bucket should have object key {file_name_2}"
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Test S3: put object")
def test_s3_put_object(self, bucket, complex_object_size, simple_object_size):
@ -570,7 +579,7 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, file_path_1, Metadata=object_1_metadata, Tagging=tag_1
)
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
assert obj_head.get("Metadata") == object_1_metadata, "Matadata must be the same"
assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
assert got_tags, f"Expected tags, got {got_tags}"
assert got_tags == [
@ -583,7 +592,7 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2
)
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
assert obj_head.get("Metadata") == object_2_metadata, "Matadata must be the same"
assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same"
got_tags_1 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
assert got_tags_1, f"Expected tags, got {got_tags_1}"
assert got_tags_1 == [
@ -624,9 +633,9 @@ class TestS3GateObject(TestS3GateBase):
assert obj_versions == {
version_id_1,
version_id_2,
}, f"Expected object has versions: {version_id_1, version_id_2}"
}, f"Object should have versions: {version_id_1, version_id_2}"
got_tags_4 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3)
assert not got_tags_4, f"No expected tags"
assert not got_tags_4, "No tags expected"
with allure.step("Get object"):
object_3 = s3_gate_object.get_object_s3(
@ -652,7 +661,7 @@ class TestS3GateObject(TestS3GateBase):
obj_head_3 = s3_gate_object.head_object_s3(
self.s3_client, bucket, file_name_3, version_id_1
)
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same"
assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same"
got_tags_3 = s3_gate_object.get_object_tagging(
self.s3_client, bucket, file_name_3, version_id_1
)
@ -669,6 +678,7 @@ class TestS3GateObject(TestS3GateBase):
bucket,
complex_object_size,
simple_object_size,
second_wallet_public_key,
):
file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1)
@ -717,32 +727,32 @@ class TestS3GateObject(TestS3GateBase):
file_name_5 = self.object_key_from_file_path(file_path_5)
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5)
generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
file_path_6,
GrantFullControl=f"id={self.other_public_key}",
file_path_5,
GrantFullControl=f"id={second_wallet_public_key}",
)
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same"
object_5 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
with allure.step(
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5)
generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
file_path_7,
file_path_5,
GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers",
)
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same"
object_6 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_6), "Hashes must be the same"
@allure.title("Test S3: put object with lock-mode")
def test_s3_put_object_lock_mode(self, complex_object_size, simple_object_size):
@ -772,7 +782,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
):
date_obj = datetime.utcnow() + timedelta(days=2)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
@ -788,7 +798,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
):
date_obj = datetime.utcnow() + timedelta(days=3)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,