frostfs-testcases/pytest_tests/testsuites/network/test_node_management.py

394 lines
16 KiB
Python
Raw Normal View History

import datetime
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
import logging
import random
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
from time import sleep
from typing import Optional, Tuple
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
import allure
import pytest
from frostfs_testlib.resources.common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container, get_container
from frostfs_testlib.steps.cli.object import (
delete_object,
get_object,
get_object_from_random_node,
head_object,
neo_go_query_height,
put_object,
put_object_to_random_node,
)
from frostfs_testlib.steps.epoch import tick_epoch
from frostfs_testlib.steps.node_management import (
check_node_in_map,
delete_node_data,
drop_object,
exclude_node_from_network_map,
get_locode_from_random_node,
include_node_to_network_map,
node_shard_list,
node_shard_set_mode,
storage_node_healthcheck,
storage_node_set_status,
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.storage_policy import get_nodes_with_object, get_simple_object_copies
from frostfs_testlib.storage.cluster import StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import datetime_utils, string_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = []
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
@allure.title("Add one node to cluster")
@pytest.mark.add_nodes
@pytest.mark.node_mgmt
class TestNodeManagement(ClusterTestBase):
@pytest.fixture
@allure.title("Create container and pick the node with data")
def create_container_and_pick_node(
self, default_wallet: str, simple_object_size: ObjectSize
) -> Tuple[str, StorageNode]:
file_path = generate_file(simple_object_size.value)
placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
endpoint = self.cluster.default_rpc_endpoint
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
cid = create_container(
default_wallet,
shell=self.shell,
endpoint=endpoint,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, self.cluster)
nodes = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert len(nodes) == 1
node = nodes[0]
yield cid, node
shards = node_shard_list(node)
assert shards
for shard in shards:
node_shard_set_mode(node, shard, "read-write")
node_shard_list(node)
@allure.step("Tick epoch with retries")
def tick_epoch_with_retries(self, attempts: int = 3, timeout: int = 3, wait_block: int = None):
for attempt in range(attempts):
try:
self.tick_epoch(wait_block=wait_block)
except RuntimeError:
sleep(timeout)
if attempt >= attempts - 1:
raise
continue
return
@pytest.fixture
def after_run_start_all_nodes(self):
yield
self.return_nodes()
@pytest.fixture
def return_nodes_after_test_run(self):
yield
self.return_nodes()
@allure.step("Return node to cluster")
def return_nodes(self, alive_node: Optional[StorageNode] = None) -> None:
for node in list(check_nodes):
with allure.step(f"Start node {node}"):
node.start_service()
with allure.step(f"Waiting status ready for node {node}"):
wait_for_node_to_be_ready(node)
# We need to wait for node to establish notifications from morph-chain
# Otherwise it will hang up when we will try to set status
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
with allure.step(f"Move node {node} to online state"):
storage_node_set_status(node, status="online", retries=2)
check_nodes.remove(node)
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
self.tick_epoch_with_retries(3, wait_block=2)
check_node_in_map(node, shell=self.shell, alive_node=alive_node)
@allure.title("Add one node to cluster")
@pytest.mark.add_nodes
def test_add_nodes(
self,
default_wallet: str,
simple_object_size: ObjectSize,
return_nodes_after_test_run,
):
"""
This test remove one node from frostfs_testlib.storage.cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
wallet = default_wallet
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
alive_node = random.choice(
[storage_node for storage_node in storage_nodes if storage_node.id != random_node.id]
)
check_node_in_map(random_node, shell=self.shell, alive_node=alive_node)
# Add node to recovery list before messing with it
check_nodes.append(random_node)
exclude_node_from_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
delete_node_data(random_node)
cid = create_container(
wallet,
rule=placement_rule_3,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
oid = put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
wait_object_replication(cid, oid, 3, shell=self.shell, nodes=storage_nodes)
self.return_nodes(alive_node)
with allure.step("Check data could be replicated to new node"):
random_node = random.choice(list(set(storage_nodes) - {random_node, alive_node}))
# Add node to recovery list before messing with it
check_nodes.append(random_node)
exclude_node_from_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
wait_object_replication(
cid,
oid,
3,
shell=self.shell,
nodes=list(set(storage_nodes) - {random_node}),
)
include_node_to_network_map(random_node, alive_node, shell=self.shell, cluster=self.cluster)
wait_object_replication(cid, oid, 3, shell=self.shell, nodes=storage_nodes)
with allure.step("Check container could be created with new node"):
cid = create_container(
wallet,
rule=placement_rule_4,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
oid = put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
wait_object_replication(cid, oid, 4, shell=self.shell, nodes=storage_nodes)
@pytest.mark.node_mgmt
2023-09-08 10:35:34 +00:00
@allure.title("Drop object using control command")
def test_drop_object(self, default_wallet, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
Test checks object could be dropped using `frostfs-cli control drop-objects` command.
"""
wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint
file_path_simple = generate_file(simple_object_size.value)
file_path_complex = generate_file(complex_object_size.value)
locode = get_locode_from_random_node(self.cluster)
rule = f"REP 1 IN SE CBF 1 SELECT 1 FROM LOC AS SE FILTER 'UN-LOCODE' EQ '{locode}' AS LOC"
cid = create_container(wallet, rule=rule, shell=self.shell, endpoint=endpoint)
oid_simple = put_object_to_random_node(wallet, file_path_simple, cid, shell=self.shell, cluster=self.cluster)
oid_complex = put_object_to_random_node(wallet, file_path_complex, cid, shell=self.shell, cluster=self.cluster)
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
for oid in (oid_simple, oid_complex):
get_object_from_random_node(wallet, cid, oid, shell=self.shell, cluster=self.cluster)
head_object(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
Add test suites for acl, container and node management Signed-off-by: Vladimir Domnich <v.domnich@yadro.com> commit f7c68cfb423e3213179521954dccb6053fc6382d Merge: e234b61 99bfe6b Merge branch 'avolkov/add_ssh' into internal_tmp_b commit 99bfe6b56cd75590f868313910068cf1a80bd43f Tick one more epoch. commit bd70bc49391d578cdda727edb4dcd181b832bf1e Start nodes in case of test fail. commit b3888ec62cfc3c18b1dff58962a94a3094342186 Catch json decode error. commit c18e415b783ec3e4ce804f43c19246240c186a97 Add ssh-key access. commit 7dbdeb653b7d5b7ab3874b546e05a48b502c2460 Add some tests. commit 844367c68638c7f97ba4860dd0069c07f499d66d Add some tests for nodes management. commit 1b84b37048dcd3cc0888aa54639975fc11fb2d75 Add some tests for nodes management. commit b30c1336a6919e0c8e500bdf2a9be3d5a14470ea Add ssh execution option. commit 2df40eca74ee20bd668778715185ffddda63cb05 Change AWS cli v1 to cli v2. commit 7403da3d7c2a5963cfbb12b7c0f3d1d641f52a7e Change AWS cli v1 to cli v2. commit b110dcdb655a585e6c53e6ebc3eae7bf1f1e792f Change AWS cli v1 to cli v2. commit 6183756a4c064c932ee193c2e08a79343017fa49 Change AWS cli v1 to cli v2. commit 398006544d60896faa3fc6e6a9dbb51ada06759c Fix container run. commit e7202136dabbe7e2d3da508e0a2ec55a0d5cb67a Added tests with AWS CLI. commit 042e1478ee1fd700c8572cbc6d0d9e6b312b8e8d Fix PR comments. commit e234b61dbb9b8b10812e069322ab03615af0d44e Add debug for env. commit 14febd06713dc03a8207bb80384acb4a7d32df0e Move env variables for pytest docker into env file. commit bafdc6131b5ac855a43b672be194cde2ccf6f75b Move env variables for pytest docker into env file. commit 27c2c6b11f51d2e3c085d44b814cb4c00f81b376 Move env variables for pytest docker into env file. commit e4db4948978e092adb83aeacdf06619f5ca2f242 Merge branch 'master' into avolkov/try_pytest commit c83a7e625e8daba3a40b65a1d69b2b1323e9ae28 WIP. commit 42489bbf8058acd2926cdb04074dc9a8ff86a0a0 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit 62526d94dc2bf72372125bea119fa66f670cf7e1 Improve allure attachments. commit 4564dae697cb069ac45bc4ba7eb0b5bbdcf1d153 Merge branch 'avolkov/try_pytest' into internal_tmp_b commit ab65810b23410ca7382ed4bdd257addfa6619659 Added tests for S3 API. commit 846c495a846c977f3e5f0bada01e5a9691a81e3d Let's get NEOFS_IR_CONTRACTS_NEOFS from env. commit c39bd88568b70ffcb76b76d68531b17d3747829d Added S3 test for versioning. commit d7c9f351abc7e02d4ebf162475604a2d6b46e712 Merge remote-tracking branch 'origin/avolkov/try_pytest' into internal_tmp_b commit bfbed22a50ce4cb6a49de383cfef66452ba9f4c1 Added some tests for S3 API and curl tests for HTTP. commit 1c49def3ddd0b3f7cf97f131e269ad465c70a680 Add yadro submodule commit 2a91685f9108101ab523e05cc9287d0f5a20196b Fix. commit 33fc2813e205766e69ef74a42a10850db6c63ce6 Add debug. commit aaaceca59e4c67253ecd4a741667b7327d1fb679 Add env variables for data nodes. commit 001cb26bcc22c8543fb2672564e898928d20622b Merge: b48a87d c70da26 Merge branch 'avolkov/try_pytest' into tmp_b commit b48a87d9a09309fea671573ba6cf303c31b11b6a Added submodule commit c70da265d319950977774e34740276f324eb57a7 Added tests for S3 bucket API. commit 3d335abe6de45d1859454f1ddf85a97514667b8f Added tests for S3 object API. commit 2ac829c700f5bc20c28953f1d40cd953fed8b390 flake8 changes for python_keywords module. commit 2de5963e96b13a5e944906b695e5d9c0829de9ad Add pytest tests. commit 4472c079b9dfd979b7c101bea32893c80cb1fe57 Add pytest tests. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
2022-07-08 17:24:55 +00:00
nodes_with_object = get_nodes_with_object(cid, oid_simple, shell=self.shell, nodes=self.cluster.storage_nodes)
random_node = random.choice(nodes_with_object)
for oid in (oid_simple, oid_complex):
with allure.step(f"Drop object {oid}"):
get_object_from_random_node(wallet, cid, oid, shell=self.shell, cluster=self.cluster)
head_object(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
drop_object(random_node, cid, oid)
self.wait_for_obj_dropped(wallet, cid, oid, endpoint, get_object)
self.wait_for_obj_dropped(wallet, cid, oid, endpoint, head_object)
@pytest.mark.node_mgmt
@pytest.mark.skip(reason="Need to clarify scenario")
@allure.title("Control Operations with storage nodes")
def test_shards(
self,
default_wallet,
create_container_and_pick_node,
simple_object_size: ObjectSize,
):
wallet = default_wallet
file_path = generate_file(simple_object_size.value)
cid, node = create_container_and_pick_node
original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
# for mode in ('read-only', 'degraded'):
for mode in ("degraded",):
shards = node_shard_list(node)
assert shards
for shard in shards:
node_shard_set_mode(node, shard, mode)
shards = node_shard_list(node)
assert shards
with pytest.raises(RuntimeError):
put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
with pytest.raises(RuntimeError):
delete_object(wallet, cid, original_oid, self.shell, self.cluster.default_rpc_endpoint)
get_object_from_random_node(wallet, cid, original_oid, self.shell, self.cluster)
for shard in shards:
node_shard_set_mode(node, shard, "read-write")
shards = node_shard_list(node)
assert shards
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
delete_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
@pytest.mark.node_mgmt
@allure.title("Put object with stopped node")
def test_stop_node(self, default_wallet, return_nodes_after_test_run, simple_object_size: ObjectSize):
wallet = default_wallet
placement_rule = "REP 3 IN X SELECT 4 FROM * AS X"
source_file_path = generate_file(simple_object_size.value)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
alive_node = random.choice(
[storage_node for storage_node in storage_nodes if storage_node.id != random_node.id]
)
cid = create_container(
wallet,
rule=placement_rule,
basic_acl=PUBLIC_ACL,
shell=self.shell,
endpoint=random_node.get_rpc_endpoint(),
)
with allure.step("Stop the random node"):
check_nodes.append(random_node)
random_node.stop_service()
with allure.step("Try to put an object and expect success"):
put_object(
wallet,
source_file_path,
cid,
shell=self.shell,
endpoint=alive_node.get_rpc_endpoint(),
)
self.return_nodes(alive_node)
@allure.step("Wait for object to be dropped")
def wait_for_obj_dropped(self, wallet: str, cid: str, oid: str, endpoint: str, checker) -> None:
for _ in range(3):
try:
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
wait_for_gc_pass_on_storage_nodes()
except Exception as err:
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
return
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')
raise AssertionError(f"Object {oid} was not dropped from node")
@pytest.mark.time
class TestTime(ClusterTestBase):
@allure.step("Neo-go should continue to release blocks")
def check_nodes_block(self, cluster_state_controller: ClusterStateController):
count_blocks = {}
with allure.step("Get current block id"):
for cluster_node in self.cluster.cluster_nodes:
cluster_state_controller.get_node_date(cluster_node)
count_blocks[cluster_node] = neo_go_query_height(
shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint()
)["Latest block"]
with allure.step("Wait for 3 blocks"):
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 3)
with allure.step("Current block id should be higher than before"):
for cluster_node in self.cluster.cluster_nodes:
shell = cluster_node.host.get_shell()
now_block = neo_go_query_height(shell=shell, endpoint=cluster_node.morph_chain.get_http_endpoint())[
"Latest block"
]
assert count_blocks[cluster_node] < now_block
@pytest.fixture()
def node_time_synchronizer(self, cluster_state_controller: ClusterStateController) -> None:
cluster_state_controller.set_sync_date_all_nodes(status="inactive")
yield
cluster_state_controller.set_sync_date_all_nodes(status="active")
def test_system_time(self, cluster_state_controller: ClusterStateController, node_time_synchronizer: None):
cluster_nodes = self.cluster.cluster_nodes
timezone_utc = datetime.timezone.utc
node_1, node_2, node_3 = cluster_nodes[0:3]
with allure.step("On node 1, move the system time forward by 2 years"):
cluster_state_controller.change_node_date(
node_1, (datetime.datetime.now(timezone_utc) + datetime.timedelta(days=(365 * 2)))
)
self.check_nodes_block(cluster_state_controller)
with allure.step("On node 2, move the system time back 2 years."):
cluster_state_controller.change_node_date(
node_2, (datetime.datetime.now(timezone_utc) - datetime.timedelta(days=(365 * 2)))
)
self.check_nodes_block(cluster_state_controller)
with allure.step("On node 3, move the system time forward by 3 years"):
cluster_state_controller.change_node_date(
node_3, (datetime.datetime.now(timezone_utc) + datetime.timedelta(days=(365 * 3)))
)
self.check_nodes_block(cluster_state_controller)
with allure.step("Return the time on all nodes to the current one"):
for cluster_node in self.cluster.cluster_nodes:
cluster_state_controller.restore_node_date(cluster_node)
self.check_nodes_block(cluster_state_controller)
with allure.step("Reboot all nodes"):
cluster_state_controller.shutdown_cluster(mode="soft")
cluster_state_controller.start_stopped_hosts()
self.check_nodes_block(cluster_state_controller)