forked from TrueCloudLab/frostfs-testcases
Fix s3 tests
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
4779d2be88
commit
b995bfca41
1 changed files with 44 additions and 34 deletions
|
@ -6,6 +6,7 @@ from random import choices, sample
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
|
from frostfs_testlib.utils import wallet_utils
|
||||||
|
|
||||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||||
from pytest_tests.helpers.file_helper import (
|
from pytest_tests.helpers.file_helper import (
|
||||||
|
@ -14,14 +15,13 @@ from pytest_tests.helpers.file_helper import (
|
||||||
generate_file_with_content,
|
generate_file_with_content,
|
||||||
get_file_hash,
|
get_file_hash,
|
||||||
)
|
)
|
||||||
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
|
|
||||||
from pytest_tests.helpers.s3_helper import (
|
from pytest_tests.helpers.s3_helper import (
|
||||||
assert_object_lock_mode,
|
assert_object_lock_mode,
|
||||||
assert_s3_acl,
|
assert_s3_acl,
|
||||||
check_objects_in_bucket,
|
check_objects_in_bucket,
|
||||||
set_bucket_versioning,
|
set_bucket_versioning,
|
||||||
)
|
)
|
||||||
from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
|
from pytest_tests.resources.common import ASSETS_DIR, WALLET_PASS
|
||||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||||
|
|
||||||
|
@ -39,6 +39,13 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
def object_key_from_file_path(full_path: str) -> str:
|
def object_key_from_file_path(full_path: str) -> str:
|
||||||
return os.path.basename(full_path)
|
return os.path.basename(full_path)
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def second_wallet_public_key(self):
|
||||||
|
second_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
|
||||||
|
wallet_utils.init_wallet(second_wallet, WALLET_PASS)
|
||||||
|
public_key = wallet_utils.get_wallet_public_key(second_wallet, WALLET_PASS)
|
||||||
|
yield public_key
|
||||||
|
|
||||||
@allure.title("Test S3: Copy object")
|
@allure.title("Test S3: Copy object")
|
||||||
def test_s3_copy_object(self, two_buckets, simple_object_size):
|
def test_s3_copy_object(self, two_buckets, simple_object_size):
|
||||||
file_path = generate_file(simple_object_size)
|
file_path = generate_file(simple_object_size)
|
||||||
|
@ -131,7 +138,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
|
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
|
||||||
|
|
||||||
with allure.step("Put several versions of object into bucket"):
|
with allure.step("Put several versions of object into bucket"):
|
||||||
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
||||||
check_objects_in_bucket(self.s3_client, bucket, [obj_key])
|
check_objects_in_bucket(self.s3_client, bucket, [obj_key])
|
||||||
|
|
||||||
with allure.step("Copy object and check acl attribute"):
|
with allure.step("Copy object and check acl attribute"):
|
||||||
|
@ -202,7 +209,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
|
|
||||||
with allure.step("Put several versions of object into bucket"):
|
with allure.step("Put several versions of object into bucket"):
|
||||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
|
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
|
||||||
version_id_1 = s3_gate_object.put_object_tagging(
|
s3_gate_object.put_object_tagging(
|
||||||
self.s3_client, bucket, file_name_simple, tags=object_tagging
|
self.s3_client, bucket, file_name_simple, tags=object_tagging
|
||||||
)
|
)
|
||||||
bucket_1_objects = [file_name_simple]
|
bucket_1_objects = [file_name_simple]
|
||||||
|
@ -267,7 +274,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
assert obj_versions == {
|
assert obj_versions == {
|
||||||
version_id_1,
|
version_id_1,
|
||||||
version_id_2,
|
version_id_2,
|
||||||
}, f"Expected object has versions: {version_id_1, version_id_2}"
|
}, f"Object should have versions: {version_id_1, version_id_2}"
|
||||||
|
|
||||||
with allure.step("Delete 1 version of object"):
|
with allure.step("Delete 1 version of object"):
|
||||||
delete_obj = s3_gate_object.delete_object_s3(
|
delete_obj = s3_gate_object.delete_object_s3(
|
||||||
|
@ -277,8 +284,8 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
obj_versions = {
|
obj_versions = {
|
||||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||||
}
|
}
|
||||||
assert obj_versions == {version_id_2}, f"Expected object has versions: {version_id_2}"
|
assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
|
||||||
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found"
|
assert "DeleteMarkers" not in delete_obj.keys(), "Delete markers not found"
|
||||||
|
|
||||||
with allure.step("Delete second version of object"):
|
with allure.step("Delete second version of object"):
|
||||||
delete_obj = s3_gate_object.delete_object_s3(
|
delete_obj = s3_gate_object.delete_object_s3(
|
||||||
|
@ -289,18 +296,18 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||||
}
|
}
|
||||||
assert not obj_versions, "Expected object not found"
|
assert not obj_versions, "Expected object not found"
|
||||||
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found"
|
assert "DeleteMarkers" not in delete_obj.keys(), "Delete markers not found"
|
||||||
|
|
||||||
with allure.step("Put new object into bucket"):
|
with allure.step("Put new object into bucket"):
|
||||||
file_name_simple = generate_file(complex_object_size)
|
file_name_simple = generate_file(complex_object_size)
|
||||||
obj_key = os.path.basename(file_name_simple)
|
obj_key = os.path.basename(file_name_simple)
|
||||||
version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
||||||
|
|
||||||
with allure.step("Delete last object"):
|
with allure.step("Delete last object"):
|
||||||
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key)
|
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key)
|
||||||
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket, True)
|
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket, True)
|
||||||
assert versions.get("DeleteMarkers", None), f"Expected delete Marker"
|
assert versions.get("DeleteMarkers", None), "Expected delete Marker"
|
||||||
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
|
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
|
||||||
|
|
||||||
@allure.title("Test S3: bulk delete version of object")
|
@allure.title("Test S3: bulk delete version of object")
|
||||||
def test_s3_bulk_delete_versioning(self, bucket, simple_object_size):
|
def test_s3_bulk_delete_versioning(self, bucket, simple_object_size):
|
||||||
|
@ -334,7 +341,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
obj_versions = {
|
obj_versions = {
|
||||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||||
}
|
}
|
||||||
assert obj_versions == version_ids, f"Expected object has versions: {version_ids}"
|
assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
|
||||||
|
|
||||||
with allure.step("Delete two objects from bucket one by one"):
|
with allure.step("Delete two objects from bucket one by one"):
|
||||||
version_to_delete_b1 = sample(
|
version_to_delete_b1 = sample(
|
||||||
|
@ -351,7 +358,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
]
|
]
|
||||||
assert (
|
assert (
|
||||||
obj_versions.sort() == version_to_save.sort()
|
obj_versions.sort() == version_to_save.sort()
|
||||||
), f"Expected object has versions: {version_to_save}"
|
), f"Object should have versions: {version_to_save}"
|
||||||
|
|
||||||
@allure.title("Test S3: Get versions of object")
|
@allure.title("Test S3: Get versions of object")
|
||||||
def test_s3_get_versioning(self, bucket, simple_object_size):
|
def test_s3_get_versioning(self, bucket, simple_object_size):
|
||||||
|
@ -531,10 +538,10 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
list_obj = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
list_obj = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||||
elif list_type == "v2":
|
elif list_type == "v2":
|
||||||
list_obj = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket)
|
list_obj = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket)
|
||||||
assert len(list_obj) == 2, f"bucket have 2 objects"
|
assert len(list_obj) == 2, "bucket should have 2 objects"
|
||||||
assert (
|
assert (
|
||||||
list_obj.sort() == [file_name, file_name_2].sort()
|
list_obj.sort() == [file_name, file_name_2].sort()
|
||||||
), f"bucket have object key {file_name, file_name_2}"
|
), f"bucket should have object key {file_name, file_name_2}"
|
||||||
|
|
||||||
with allure.step("Delete object"):
|
with allure.step("Delete object"):
|
||||||
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name)
|
delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name)
|
||||||
|
@ -547,9 +554,11 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
self.s3_client, bucket, full_output=True
|
self.s3_client, bucket, full_output=True
|
||||||
)
|
)
|
||||||
contents = list_obj_1.get("Contents", [])
|
contents = list_obj_1.get("Contents", [])
|
||||||
assert len(contents) == 1, f"bucket have only 1 object"
|
assert len(contents) == 1, "bucket should have only 1 object"
|
||||||
assert contents[0].get("Key") == file_name_2, f"bucket has object key {file_name_2}"
|
assert (
|
||||||
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
|
contents[0].get("Key") == file_name_2
|
||||||
|
), f"bucket should have object key {file_name_2}"
|
||||||
|
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
|
||||||
|
|
||||||
@allure.title("Test S3: put object")
|
@allure.title("Test S3: put object")
|
||||||
def test_s3_put_object(self, bucket, complex_object_size, simple_object_size):
|
def test_s3_put_object(self, bucket, complex_object_size, simple_object_size):
|
||||||
|
@ -570,7 +579,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
self.s3_client, bucket, file_path_1, Metadata=object_1_metadata, Tagging=tag_1
|
self.s3_client, bucket, file_path_1, Metadata=object_1_metadata, Tagging=tag_1
|
||||||
)
|
)
|
||||||
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
|
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
|
||||||
assert obj_head.get("Metadata") == object_1_metadata, "Matadata must be the same"
|
assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
|
||||||
got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
|
got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
|
||||||
assert got_tags, f"Expected tags, got {got_tags}"
|
assert got_tags, f"Expected tags, got {got_tags}"
|
||||||
assert got_tags == [
|
assert got_tags == [
|
||||||
|
@ -583,7 +592,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2
|
self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2
|
||||||
)
|
)
|
||||||
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
|
obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
|
||||||
assert obj_head.get("Metadata") == object_2_metadata, "Matadata must be the same"
|
assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same"
|
||||||
got_tags_1 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
|
got_tags_1 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name)
|
||||||
assert got_tags_1, f"Expected tags, got {got_tags_1}"
|
assert got_tags_1, f"Expected tags, got {got_tags_1}"
|
||||||
assert got_tags_1 == [
|
assert got_tags_1 == [
|
||||||
|
@ -624,9 +633,9 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
assert obj_versions == {
|
assert obj_versions == {
|
||||||
version_id_1,
|
version_id_1,
|
||||||
version_id_2,
|
version_id_2,
|
||||||
}, f"Expected object has versions: {version_id_1, version_id_2}"
|
}, f"Object should have versions: {version_id_1, version_id_2}"
|
||||||
got_tags_4 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3)
|
got_tags_4 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3)
|
||||||
assert not got_tags_4, f"No expected tags"
|
assert not got_tags_4, "No tags expected"
|
||||||
|
|
||||||
with allure.step("Get object"):
|
with allure.step("Get object"):
|
||||||
object_3 = s3_gate_object.get_object_s3(
|
object_3 = s3_gate_object.get_object_s3(
|
||||||
|
@ -652,7 +661,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
obj_head_3 = s3_gate_object.head_object_s3(
|
obj_head_3 = s3_gate_object.head_object_s3(
|
||||||
self.s3_client, bucket, file_name_3, version_id_1
|
self.s3_client, bucket, file_name_3, version_id_1
|
||||||
)
|
)
|
||||||
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same"
|
assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same"
|
||||||
got_tags_3 = s3_gate_object.get_object_tagging(
|
got_tags_3 = s3_gate_object.get_object_tagging(
|
||||||
self.s3_client, bucket, file_name_3, version_id_1
|
self.s3_client, bucket, file_name_3, version_id_1
|
||||||
)
|
)
|
||||||
|
@ -669,6 +678,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
bucket,
|
bucket,
|
||||||
complex_object_size,
|
complex_object_size,
|
||||||
simple_object_size,
|
simple_object_size,
|
||||||
|
second_wallet_public_key,
|
||||||
):
|
):
|
||||||
file_path_1 = generate_file(complex_object_size)
|
file_path_1 = generate_file(complex_object_size)
|
||||||
file_name = self.object_key_from_file_path(file_path_1)
|
file_name = self.object_key_from_file_path(file_path_1)
|
||||||
|
@ -717,32 +727,32 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
file_name_5 = self.object_key_from_file_path(file_path_5)
|
file_name_5 = self.object_key_from_file_path(file_path_5)
|
||||||
|
|
||||||
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
|
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
|
||||||
file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5)
|
generate_file_with_content(simple_object_size, file_path=file_path_5)
|
||||||
s3_gate_object.put_object_s3(
|
s3_gate_object.put_object_s3(
|
||||||
self.s3_client,
|
self.s3_client,
|
||||||
bucket,
|
bucket,
|
||||||
file_path_6,
|
file_path_5,
|
||||||
GrantFullControl=f"id={self.other_public_key}",
|
GrantFullControl=f"id={second_wallet_public_key}",
|
||||||
)
|
)
|
||||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
|
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
|
||||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||||
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
|
object_5 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
|
||||||
assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same"
|
assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
|
||||||
|
|
||||||
with allure.step(
|
with allure.step(
|
||||||
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
||||||
):
|
):
|
||||||
file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5)
|
generate_file_with_content(simple_object_size, file_path=file_path_5)
|
||||||
s3_gate_object.put_object_s3(
|
s3_gate_object.put_object_s3(
|
||||||
self.s3_client,
|
self.s3_client,
|
||||||
bucket,
|
bucket,
|
||||||
file_path_7,
|
file_path_5,
|
||||||
GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||||
)
|
)
|
||||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
|
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
|
||||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||||
object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
|
object_6 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
|
||||||
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same"
|
assert get_file_hash(file_path_5) == get_file_hash(object_6), "Hashes must be the same"
|
||||||
|
|
||||||
@allure.title("Test S3: put object with lock-mode")
|
@allure.title("Test S3: put object with lock-mode")
|
||||||
def test_s3_put_object_lock_mode(self, complex_object_size, simple_object_size):
|
def test_s3_put_object_lock_mode(self, complex_object_size, simple_object_size):
|
||||||
|
@ -772,7 +782,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
|
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
|
||||||
):
|
):
|
||||||
date_obj = datetime.utcnow() + timedelta(days=2)
|
date_obj = datetime.utcnow() + timedelta(days=2)
|
||||||
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
|
generate_file_with_content(simple_object_size, file_path=file_path_1)
|
||||||
s3_gate_object.put_object_s3(
|
s3_gate_object.put_object_s3(
|
||||||
self.s3_client,
|
self.s3_client,
|
||||||
bucket,
|
bucket,
|
||||||
|
@ -788,7 +798,7 @@ class TestS3GateObject(TestS3GateBase):
|
||||||
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
|
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
|
||||||
):
|
):
|
||||||
date_obj = datetime.utcnow() + timedelta(days=3)
|
date_obj = datetime.utcnow() + timedelta(days=3)
|
||||||
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
|
generate_file_with_content(simple_object_size, file_path=file_path_1)
|
||||||
s3_gate_object.put_object_s3(
|
s3_gate_object.put_object_s3(
|
||||||
self.s3_client,
|
self.s3_client,
|
||||||
bucket,
|
bucket,
|
||||||
|
|
Loading…
Reference in a new issue