forked from TrueCloudLab/frostfs-testcases
[#322] Remove basic acl part one
Signed-off-by: a.berezin <a.berezin@yadro.com>
This commit is contained in:
parent
1ee5f73243
commit
35c3d4657f
4 changed files with 76 additions and 64 deletions
|
|
@ -42,3 +42,7 @@ class ContainerSpec:
|
|||
spec_info.append(f"ape_rules=[{ape_rules_list}]")
|
||||
|
||||
return f"ContainerSpec({', '.join(spec_info)})"
|
||||
|
||||
|
||||
class ContainerSpecs:
|
||||
PublicReadWrite = ContainerSpec(ape_rules=APE_PUBLIC_READ_WRITE)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
|
@ -17,7 +15,6 @@ from frostfs_testlib.healthcheck.interfaces import Healthcheck
|
|||
from frostfs_testlib.hosting import Hosting
|
||||
from frostfs_testlib.resources import optionals
|
||||
from frostfs_testlib.resources.common import COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, MORPH_BLOCK_TIME, SIMPLE_OBJECT_SIZE
|
||||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3.interfaces import BucketContainerResolver
|
||||
from frostfs_testlib.shell import LocalShell, Shell
|
||||
|
|
@ -42,11 +39,11 @@ from frostfs_testlib.storage.grpc_operations.client_wrappers import CliClientWra
|
|||
from frostfs_testlib.storage.grpc_operations.interfaces import GrpcClientWrapper
|
||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
from frostfs_testlib.testing.parallel import parallel
|
||||
from frostfs_testlib.testing.test_control import run_optionally, wait_for_success
|
||||
from frostfs_testlib.testing.test_control import cached_fixture, run_optionally, wait_for_success
|
||||
from frostfs_testlib.utils import datetime_utils, env_utils, string_utils, version_utils
|
||||
from frostfs_testlib.utils.file_utils import TestFile, generate_file
|
||||
from workspace.frostfs_testcases.pytest_tests.helpers.container_spec import ContainerSpec
|
||||
|
||||
from ..helpers.container_spec import ContainerSpec, ContainerSpecs
|
||||
from ..resources.common import TEST_CYCLES_COUNT
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
|
@ -153,15 +150,16 @@ def require_multiple_interfaces(cluster: Cluster):
|
|||
interfaces = cluster.cluster_nodes[0].host.config.interfaces
|
||||
if "internal1" not in interfaces or "data1" not in interfaces:
|
||||
pytest.skip("This test requires multiple internal and data interfaces")
|
||||
yield
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
|
||||
def max_object_size(cluster: Cluster, client_shell: Shell) -> int:
|
||||
storage_node = cluster.storage_nodes[0]
|
||||
wallet = WalletInfo.from_node(storage_node)
|
||||
net_info = get_netmap_netinfo(wallet=wallet, endpoint=storage_node.get_rpc_endpoint(), shell=client_shell)
|
||||
yield net_info["maximum_object_size"]
|
||||
return net_info["maximum_object_size"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
|
@ -436,6 +434,7 @@ def readiness_on_node(cluster_node: ClusterNode):
|
|||
|
||||
@reporter.step("Prepare default user with wallet")
|
||||
@pytest.fixture(scope="session")
|
||||
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
|
||||
def default_user(credentials_provider: CredentialsProvider, cluster: Cluster) -> User:
|
||||
user = User(string_utils.unique_name("user-"))
|
||||
node = cluster.cluster_nodes[0]
|
||||
|
|
@ -512,6 +511,38 @@ def container(
|
|||
if container_spec.ape_rules:
|
||||
_apply_ape_rules(frostfs_cli, cluster, cid, container_spec.ape_rules)
|
||||
|
||||
# Add marker if we want to run all tests with container
|
||||
request.node.add_marker("requires_container")
|
||||
|
||||
return cid
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def container_module_scope(
|
||||
default_wallet: WalletInfo,
|
||||
frostfs_cli: FrostfsCli,
|
||||
client_shell: Shell,
|
||||
cluster: Cluster,
|
||||
request: pytest.FixtureRequest,
|
||||
rpc_endpoint: str,
|
||||
) -> str:
|
||||
with reporter.step("Get container specification for test"):
|
||||
container_spec = _get_container_spec(request)
|
||||
|
||||
with reporter.step("Create container"):
|
||||
cid = _create_container_by_spec(default_wallet, client_shell, cluster, rpc_endpoint, container_spec)
|
||||
# TODO: deprecate this. Use generic ContainerSpec.ape_rule param
|
||||
if container_spec.allow_owner_via_ape:
|
||||
with reporter.step("Allow owner via APE on container"):
|
||||
_allow_owner_via_ape(frostfs_cli, cluster, cid)
|
||||
|
||||
with reporter.step("Apply APE rules for container"):
|
||||
if container_spec.ape_rules:
|
||||
_apply_ape_rules(frostfs_cli, cluster, cid, container_spec.ape_rules)
|
||||
|
||||
# Add marker if we want to run all tests with container
|
||||
request.node.add_marker("requires_container")
|
||||
|
||||
return cid
|
||||
|
||||
|
||||
|
|
@ -534,11 +565,8 @@ def _apply_ape_rules(frostfs_cli: FrostfsCli, cluster: Cluster, container: str,
|
|||
def _create_container_by_spec(
|
||||
default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, rpc_endpoint: str, container_spec: ContainerSpec
|
||||
) -> str:
|
||||
# TODO: add container spec to step message
|
||||
with reporter.step("Create container"):
|
||||
cid = create_container(
|
||||
default_wallet, client_shell, rpc_endpoint, basic_acl=container_spec.basic_acl, rule=container_spec.parsed_rule(cluster)
|
||||
)
|
||||
with reporter.step(f"Create container by spec {container_spec}"):
|
||||
cid = create_container(default_wallet, client_shell, rpc_endpoint, container_spec.parsed_rule(cluster))
|
||||
|
||||
with reporter.step("Search nodes holding the container"):
|
||||
container_holder_nodes = search_nodes_with_container(default_wallet, cid, client_shell, cluster.default_rpc_endpoint, cluster)
|
||||
|
|
@ -552,7 +580,7 @@ def _create_container_by_spec(
|
|||
def _get_container_spec(request: pytest.FixtureRequest) -> ContainerSpec:
|
||||
container_marker = request.node.get_closest_marker("container")
|
||||
# let default container to be public at the moment
|
||||
container_spec = ContainerSpec(basic_acl=PUBLIC_ACL)
|
||||
container_spec = ContainerSpecs.PublicReadWrite
|
||||
|
||||
if container_marker:
|
||||
if len(container_marker.args) != 1:
|
||||
|
|
@ -592,3 +620,8 @@ def _allow_owner_via_ape(frostfs_cli: FrostfsCli, cluster: Cluster, container: s
|
|||
@pytest.fixture()
|
||||
def new_epoch(client_shell: Shell, cluster: Cluster) -> int:
|
||||
return ensure_fresh_epoch(client_shell, cluster)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def new_epoch_module_scope(client_shell: Shell, cluster: Cluster) -> int:
|
||||
return ensure_fresh_epoch(client_shell, cluster)
|
||||
|
|
|
|||
|
|
@ -4,14 +4,15 @@ import allure
|
|||
import pytest
|
||||
from frostfs_testlib import reporter
|
||||
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
|
||||
from frostfs_testlib.steps.cli.container import create_container
|
||||
from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node
|
||||
from frostfs_testlib.steps.epoch import get_epoch
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
|
||||
from frostfs_testlib.testing.test_control import expect_not_raises
|
||||
from frostfs_testlib.utils.file_utils import TestFile
|
||||
|
||||
from ...helpers.container_spec import ContainerSpecs
|
||||
from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
|
@ -21,36 +22,32 @@ logger = logging.getLogger("NeoLogger")
|
|||
@pytest.mark.sanity
|
||||
@pytest.mark.grpc_api
|
||||
class TestObjectApiLifetime(ClusterTestBase):
|
||||
@pytest.mark.container(ContainerSpecs.PublicReadWrite)
|
||||
@allure.title("Object is removed when lifetime expired (obj_size={object_size})")
|
||||
def test_object_api_lifetime(self, default_wallet: WalletInfo, object_size: ObjectSize):
|
||||
def test_object_api_lifetime(self, container: str, test_file: TestFile, default_wallet: WalletInfo, object_size: ObjectSize):
|
||||
"""
|
||||
Test object deleted after expiration epoch.
|
||||
"""
|
||||
|
||||
wallet = default_wallet
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
cid = create_container(wallet, self.shell, endpoint)
|
||||
|
||||
file_path = generate_file(object_size.value)
|
||||
file_hash = get_file_hash(file_path)
|
||||
epoch = get_epoch(self.shell, self.cluster)
|
||||
|
||||
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster, expire_at=epoch + 1)
|
||||
got_file = get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster)
|
||||
assert get_file_hash(got_file) == file_hash
|
||||
oid = put_object_to_random_node(wallet, test_file.path, container, self.shell, self.cluster, expire_at=epoch + 1)
|
||||
with expect_not_raises():
|
||||
head_object(wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
|
||||
with reporter.step("Tick two epochs"):
|
||||
for _ in range(2):
|
||||
self.tick_epoch()
|
||||
self.tick_epochs(2)
|
||||
|
||||
# Wait for GC, because object with expiration is counted as alive until GC removes it
|
||||
wait_for_gc_pass_on_storage_nodes()
|
||||
|
||||
with reporter.step("Check object deleted because it expires on epoch"):
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
head_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
head_object(wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster)
|
||||
get_object_from_random_node(wallet, container, oid, self.shell, self.cluster)
|
||||
|
||||
with reporter.step("Tick additional epoch"):
|
||||
self.tick_epoch()
|
||||
|
|
@ -59,6 +56,6 @@ class TestObjectApiLifetime(ClusterTestBase):
|
|||
|
||||
with reporter.step("Check object deleted because it expires on previous epoch"):
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
head_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
head_object(wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster)
|
||||
get_object_from_random_node(wallet, container, oid, self.shell, self.cluster)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import logging
|
||||
import re
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
|
|
@ -11,15 +10,14 @@ from frostfs_testlib.resources.error_patterns import (
|
|||
LOCK_NON_REGULAR_OBJECT,
|
||||
LOCK_OBJECT_EXPIRATION,
|
||||
LOCK_OBJECT_REMOVAL,
|
||||
OBJECT_ALREADY_REMOVED,
|
||||
OBJECT_IS_LOCKED,
|
||||
OBJECT_NOT_FOUND,
|
||||
)
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
|
||||
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo
|
||||
from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object
|
||||
from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks
|
||||
from frostfs_testlib.steps.epoch import ensure_fresh_epoch, get_epoch, tick_epoch
|
||||
from frostfs_testlib.steps.epoch import ensure_fresh_epoch
|
||||
from frostfs_testlib.steps.node_management import drop_object
|
||||
from frostfs_testlib.steps.storage_object import delete_objects
|
||||
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
|
||||
|
|
@ -31,6 +29,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
|||
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
|
||||
from frostfs_testlib.utils import datetime_utils, string_utils
|
||||
|
||||
from ...helpers.container_spec import ContainerSpecs
|
||||
from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
|
@ -38,22 +37,24 @@ logger = logging.getLogger("NeoLogger")
|
|||
FIXTURE_LOCK_LIFETIME = 5
|
||||
FIXTURE_OBJECT_LIFETIME = 10
|
||||
|
||||
pytestmark = pytest.mark.container(ContainerSpecs.PublicReadWrite)
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
|
||||
with reporter.step("Create user wallet with container"):
|
||||
user = User(string_utils.unique_name("user-"))
|
||||
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
|
||||
container_id = create_container(user_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
|
||||
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
|
||||
@pytest.fixture(scope="class")
|
||||
def user_container(container_module_scope: str, user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
|
||||
return StorageContainer(StorageContainerInfo(container_module_scope, user_wallet), client_shell, cluster)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@pytest.fixture(scope="class")
|
||||
def locked_storage_object(
|
||||
new_epoch_module_scope: int,
|
||||
user_container: StorageContainer,
|
||||
client_shell: Shell,
|
||||
cluster: Cluster,
|
||||
|
|
@ -63,7 +64,7 @@ def locked_storage_object(
|
|||
Intention of this fixture is to provide storage object which is NOT expected to be deleted during test act phase
|
||||
"""
|
||||
with reporter.step("Creating locked object"):
|
||||
current_epoch = ensure_fresh_epoch(client_shell, cluster)
|
||||
current_epoch = new_epoch_module_scope
|
||||
expiration_epoch = current_epoch + FIXTURE_LOCK_LIFETIME
|
||||
|
||||
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME)
|
||||
|
|
@ -77,30 +78,7 @@ def locked_storage_object(
|
|||
)
|
||||
storage_object.locks = [LockObjectInfo(storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch)]
|
||||
|
||||
yield storage_object
|
||||
|
||||
with reporter.step("Delete created locked object"):
|
||||
current_epoch = get_epoch(client_shell, cluster)
|
||||
epoch_diff = expiration_epoch - current_epoch + 1
|
||||
|
||||
if epoch_diff > 0:
|
||||
with reporter.step(f"Tick {epoch_diff} epochs"):
|
||||
for _ in range(epoch_diff):
|
||||
tick_epoch(client_shell, cluster)
|
||||
try:
|
||||
delete_object(
|
||||
storage_object.wallet,
|
||||
storage_object.cid,
|
||||
storage_object.oid,
|
||||
client_shell,
|
||||
cluster.default_rpc_endpoint,
|
||||
)
|
||||
except Exception as ex:
|
||||
ex_message = str(ex)
|
||||
# It's okay if object already removed
|
||||
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(OBJECT_ALREADY_REMOVED, ex_message):
|
||||
raise ex
|
||||
logger.debug(ex_message)
|
||||
return storage_object
|
||||
|
||||
|
||||
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
|
||||
|
|
|
|||
Loading…
Reference in a new issue