From 001b8c14a1f34ec1324ac4e2df43203a5db25ad6 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Mon, 6 Dec 2021 10:53:18 +0530 Subject: [PATCH] Add testcases for rgw cloudtransition feature Feature PR: https://github.com/ceph/ceph/pull/35100 Also ported lc testcases from boto2 to boto3 (cherry picked from commit 47292aee178f2f876c9054e50f11d48466b53341) Signed-off-by: Soumya Koduri --- s3tests.conf.SAMPLE | 31 ++ s3tests/functional/test_s3.py | 236 ------------ s3tests_boto3/functional/__init__.py | 87 +++++ s3tests_boto3/functional/test_s3.py | 527 ++++++++++++++++++++++++++- 4 files changed, 643 insertions(+), 238 deletions(-) diff --git a/s3tests.conf.SAMPLE b/s3tests.conf.SAMPLE index c0c5fd9..fad5e0c 100644 --- a/s3tests.conf.SAMPLE +++ b/s3tests.conf.SAMPLE @@ -56,6 +56,37 @@ access_key = NOPQRSTUVWXYZABCDEFG # alt AWS secret key set in vstart.sh secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm +[s3 cloud] +## to run the testcases with "cloud_transition" attribute. +## Note: the waiting time may have to tweaked depending on +## the I/O latency to the cloud endpoint. + +## host set for cloud endpoint +# host = localhost + +## port set for cloud endpoint +# port = 8001 + +## say "False" to disable TLS +# is_secure = False + +## cloud endpoint credentials +# access_key = 0555b35654ad1656d804 +# secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== + +## storage class configured as cloud tier on local rgw server +# cloud_storage_class = CLOUDTIER + +## Below are optional - + +## Above configured cloud storage class config options +# retain_head_object = false +# target_storage_class = Target_SC +# target_path = cloud-bucket + +## another regular storage class to test multiple transition rules, +# storage_class = S1 + [s3 tenant] # tenant display_name set in vstart.sh display_name = testx$tenanteduser diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index dd295cc..781ec5d 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -236,242 +236,6 @@ def list_bucket_storage_class(bucket): return result -# The test harness for lifecycle is configured to treat days as 10 second intervals. -@attr(resource='bucket') -@attr(method='put') -@attr(operation='test lifecycle expiration') -@attr('lifecycle') -@attr('lifecycle_transition') -@attr('fails_on_aws') -def test_lifecycle_transition(): - sc = configured_storage_classes() - if len(sc) < 3: - raise SkipTest - - bucket = set_lifecycle(rules=[{'id': 'rule1', 'transition': lc_transition(days=1, storage_class=sc[1]), 'prefix': 'expire1/', 'status': 'Enabled'}, - {'id':'rule2', 'transition': lc_transition(days=4, storage_class=sc[2]), 'prefix': 'expire3/', 'status': 'Enabled'}]) - _create_keys(bucket=bucket, keys=['expire1/foo', 'expire1/bar', 'keep2/foo', - 'keep2/bar', 'expire3/foo', 'expire3/bar']) - # Get list of all keys - init_keys = bucket.get_all_keys() - eq(len(init_keys), 6) - - # Wait for first expiration (plus fudge to handle the timer window) - time.sleep(25) - expire1_keys = list_bucket_storage_class(bucket) - eq(len(expire1_keys['STANDARD']), 4) - eq(len(expire1_keys[sc[1]]), 2) - eq(len(expire1_keys[sc[2]]), 0) - - # Wait for next expiration cycle - time.sleep(10) - keep2_keys = list_bucket_storage_class(bucket) - eq(len(keep2_keys['STANDARD']), 4) - eq(len(keep2_keys[sc[1]]), 2) - eq(len(keep2_keys[sc[2]]), 0) - - # Wait for final expiration cycle - time.sleep(20) - expire3_keys = list_bucket_storage_class(bucket) - eq(len(expire3_keys['STANDARD']), 2) - eq(len(expire3_keys[sc[1]]), 2) - eq(len(expire3_keys[sc[2]]), 2) - -# The test harness for lifecycle is configured to treat days as 10 second intervals. -@attr(resource='bucket') -@attr(method='put') -@attr(operation='test lifecycle expiration') -@attr('lifecycle') -@attr('lifecycle_transition') -@attr('fails_on_aws') -def test_lifecycle_transition_single_rule_multi_trans(): - sc = configured_storage_classes() - if len(sc) < 3: - raise SkipTest - - bucket = set_lifecycle(rules=[ - {'id': 'rule1', - 'transition': lc_transitions([ - lc_transition(days=1, storage_class=sc[1]), - lc_transition(days=4, storage_class=sc[2])]), - 'prefix': 'expire1/', - 'status': 'Enabled'}]) - - _create_keys(bucket=bucket, keys=['expire1/foo', 'expire1/bar', 'keep2/foo', - 'keep2/bar', 'expire3/foo', 'expire3/bar']) - # Get list of all keys - init_keys = bucket.get_all_keys() - eq(len(init_keys), 6) - - # Wait for first expiration (plus fudge to handle the timer window) - time.sleep(25) - expire1_keys = list_bucket_storage_class(bucket) - eq(len(expire1_keys['STANDARD']), 4) - eq(len(expire1_keys[sc[1]]), 2) - eq(len(expire1_keys[sc[2]]), 0) - - # Wait for next expiration cycle - time.sleep(10) - keep2_keys = list_bucket_storage_class(bucket) - eq(len(keep2_keys['STANDARD']), 4) - eq(len(keep2_keys[sc[1]]), 2) - eq(len(keep2_keys[sc[2]]), 0) - - # Wait for final expiration cycle - time.sleep(20) - expire3_keys = list_bucket_storage_class(bucket) - eq(len(expire3_keys['STANDARD']), 4) - eq(len(expire3_keys[sc[1]]), 0) - eq(len(expire3_keys[sc[2]]), 2) - -def generate_lifecycle_body(rules): - body = '' - for rule in rules: - body += '%s%s' % (rule['ID'], rule['Status']) - if 'Prefix' in list(rule.keys()): - body += '%s' % rule['Prefix'] - if 'Filter' in list(rule.keys()): - prefix_str= '' # AWS supports empty filters - if 'Prefix' in list(rule['Filter'].keys()): - prefix_str = '%s' % rule['Filter']['Prefix'] - body += '%s' % prefix_str - - if 'Expiration' in list(rule.keys()): - if 'ExpiredObjectDeleteMarker' in list(rule['Expiration'].keys()): - body += '%s' \ - % rule['Expiration']['ExpiredObjectDeleteMarker'] - elif 'Date' in list(rule['Expiration'].keys()): - body += '%s' % rule['Expiration']['Date'] - else: - body += '%d' % rule['Expiration']['Days'] - if 'NoncurrentVersionExpiration' in list(rule.keys()): - body += '%d' % \ - rule['NoncurrentVersionExpiration']['NoncurrentDays'] - if 'NoncurrentVersionTransition' in list(rule.keys()): - for t in rule['NoncurrentVersionTransition']: - body += '' - body += '%d' % \ - t['NoncurrentDays'] - body += '%s' % \ - t['StorageClass'] - body += '' - if 'AbortIncompleteMultipartUpload' in list(rule.keys()): - body += '%d' \ - '' % rule['AbortIncompleteMultipartUpload']['DaysAfterInitiation'] - body += '' - body += '' - return body - - -@attr(resource='bucket') -@attr(method='put') -@attr(operation='set lifecycle config with noncurrent version expiration') -@attr('lifecycle') -@attr('lifecycle_transition') -def test_lifecycle_set_noncurrent_transition(): - sc = configured_storage_classes() - if len(sc) < 3: - raise SkipTest - - bucket = get_new_bucket() - rules = [ - { - 'ID': 'rule1', - 'Prefix': 'test1/', - 'Status': 'Enabled', - 'NoncurrentVersionTransition': [ - { - 'NoncurrentDays': 2, - 'StorageClass': sc[1] - }, - { - 'NoncurrentDays': 4, - 'StorageClass': sc[2] - } - ], - 'NoncurrentVersionExpiration': { - 'NoncurrentDays': 6 - } - }, - {'ID': 'rule2', 'Prefix': 'test2/', 'Status': 'Disabled', 'NoncurrentVersionExpiration': {'NoncurrentDays': 3}} - ] - body = generate_lifecycle_body(rules) - fp = StringIO(body) - md5 = boto.utils.compute_md5(fp) - headers = {'Content-MD5': md5[1], 'Content-Type': 'text/xml'} - res = bucket.connection.make_request('PUT', bucket.name, data=fp.getvalue(), query_args='lifecycle', - headers=headers) - eq(res.status, 200) - eq(res.reason, 'OK') - - -@attr(resource='bucket') -@attr(method='put') -@attr(operation='test lifecycle non-current version expiration') -@attr('lifecycle') -@attr('lifecycle_expiration') -@attr('lifecycle_transition') -@attr('fails_on_aws') -def test_lifecycle_noncur_transition(): - sc = configured_storage_classes() - if len(sc) < 3: - raise SkipTest - - bucket = get_new_bucket() - check_configure_versioning_retry(bucket, True, "Enabled") - - rules = [ - { - 'ID': 'rule1', - 'Prefix': 'test1/', - 'Status': 'Enabled', - 'NoncurrentVersionTransition': [ - { - 'NoncurrentDays': 1, - 'StorageClass': sc[1] - }, - { - 'NoncurrentDays': 3, - 'StorageClass': sc[2] - } - ], - 'NoncurrentVersionExpiration': { - 'NoncurrentDays': 5 - } - } - ] - body = generate_lifecycle_body(rules) - fp = StringIO(body) - md5 = boto.utils.compute_md5(fp) - headers = {'Content-MD5': md5[1], 'Content-Type': 'text/xml'} - bucket.connection.make_request('PUT', bucket.name, data=fp.getvalue(), query_args='lifecycle', - headers=headers) - - create_multiple_versions(bucket, "test1/a", 3) - create_multiple_versions(bucket, "test1/b", 3) - init_keys = bucket.get_all_versions() - eq(len(init_keys), 6) - - time.sleep(25) - expire1_keys = list_bucket_storage_class(bucket) - eq(len(expire1_keys['STANDARD']), 2) - eq(len(expire1_keys[sc[1]]), 4) - eq(len(expire1_keys[sc[2]]), 0) - - time.sleep(20) - expire1_keys = list_bucket_storage_class(bucket) - eq(len(expire1_keys['STANDARD']), 2) - eq(len(expire1_keys[sc[1]]), 0) - eq(len(expire1_keys[sc[2]]), 4) - - time.sleep(20) - expire_keys = bucket.get_all_versions() - expire1_keys = list_bucket_storage_class(bucket) - eq(len(expire1_keys['STANDARD']), 2) - eq(len(expire1_keys[sc[1]]), 0) - eq(len(expire1_keys[sc[2]]), 0) - - def transfer_part(bucket, mp_id, mp_keyname, i, part, headers=None): """Transfer a part of a multipart upload. Designed to be run in parallel. """ diff --git a/s3tests_boto3/functional/__init__.py b/s3tests_boto3/functional/__init__.py index 18579e2..eff0b46 100644 --- a/s3tests_boto3/functional/__init__.py +++ b/s3tests_boto3/functional/__init__.py @@ -12,6 +12,7 @@ import random import string import itertools import urllib3 +import re config = munch.Munch @@ -163,6 +164,17 @@ def nuke_prefixed_buckets(prefix, client=None): print('Done with cleanup of buckets in tests.') +def configured_storage_classes(): + sc = [ 'STANDARD' ] + + extra_sc = re.split('\W+', config.storage_classes) + + for item in extra_sc: + if item != 'STANDARD': + sc.append(item) + + return sc + def setup(): cfg = configparser.RawConfigParser() try: @@ -226,6 +238,12 @@ def setup(): config.main_api_name = "" pass + try: + config.storage_classes = cfg.get('s3 main',"storage_classes") + except (configparser.NoSectionError, configparser.NoOptionError): + config.storage_classes = "" + pass + config.alt_access_key = cfg.get('s3 alt',"access_key") config.alt_secret_key = cfg.get('s3 alt',"secret_key") config.alt_display_name = cfg.get('s3 alt',"display_name") @@ -251,6 +269,11 @@ def setup(): nuke_prefixed_buckets(prefix=prefix, client=alt_client) nuke_prefixed_buckets(prefix=prefix, client=tenant_client) + if cfg.has_section("s3 cloud"): + get_cloud_config(cfg) + else: + config.cloud_storage_class = None + def teardown(): alt_client = get_alt_client() @@ -298,6 +321,43 @@ def check_webidentity(): config.webidentity_azp = cfg.get('webidentity', "azp") config.webidentity_user_token = cfg.get('webidentity', "user_token") +def get_cloud_config(cfg): + config.cloud_host = cfg.get('s3 cloud',"host") + config.cloud_port = int(cfg.get('s3 cloud',"port")) + config.cloud_is_secure = cfg.getboolean('s3 cloud', "is_secure") + + proto = 'https' if config.cloud_is_secure else 'http' + config.cloud_endpoint = "%s://%s:%d" % (proto, config.cloud_host, config.cloud_port) + + config.cloud_access_key = cfg.get('s3 cloud',"access_key") + config.cloud_secret_key = cfg.get('s3 cloud',"secret_key") + + try: + config.cloud_storage_class = cfg.get('s3 cloud', "cloud_storage_class") + except (configparser.NoSectionError, configparser.NoOptionError): + config.cloud_storage_class = None + + try: + config.cloud_retain_head_object = cfg.get('s3 cloud',"retain_head_object") + except (configparser.NoSectionError, configparser.NoOptionError): + config.cloud_retain_head_object = None + + try: + config.cloud_target_path = cfg.get('s3 cloud',"target_path") + except (configparser.NoSectionError, configparser.NoOptionError): + config.cloud_target_path = None + + try: + config.cloud_target_storage_class = cfg.get('s3 cloud',"target_storage_class") + except (configparser.NoSectionError, configparser.NoOptionError): + config.cloud_target_storage_class = 'STANDARD' + + try: + config.cloud_regular_storage_class = cfg.get('s3 cloud', "storage_class") + except (configparser.NoSectionError, configparser.NoOptionError): + config.cloud_regular_storage_class = None + + def get_client(client_config=None): if client_config == None: client_config = Config(signature_version='s3v4') @@ -380,6 +440,18 @@ def get_alt_client(client_config=None): config=client_config) return client +def get_cloud_client(client_config=None): + if client_config == None: + client_config = Config(signature_version='s3v4') + + client = boto3.client(service_name='s3', + aws_access_key_id=config.cloud_access_key, + aws_secret_access_key=config.cloud_secret_key, + endpoint_url=config.cloud_endpoint, + use_ssl=config.cloud_is_secure, + config=client_config) + return client + def get_tenant_client(client_config=None): if client_config == None: client_config = Config(signature_version='s3v4') @@ -598,3 +670,18 @@ def get_iam_secret_key(): def get_user_token(): return config.webidentity_user_token + +def get_cloud_storage_class(): + return config.cloud_storage_class + +def get_cloud_retain_head_object(): + return config.cloud_retain_head_object + +def get_cloud_regular_storage_class(): + return config.cloud_regular_storage_class + +def get_cloud_target_path(): + return config.cloud_target_path + +def get_cloud_target_storage_class(): + return config.cloud_target_storage_class diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 1e511a6..69a1b3a 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -28,6 +28,8 @@ import socket import dateutil.parser import ssl from collections import namedtuple +from collections import defaultdict +from io import StringIO from email.header import decode_header @@ -72,7 +74,14 @@ from . import ( get_main_kms_keyid, get_secondary_kms_keyid, get_svc_client, + get_cloud_storage_class, + get_cloud_retain_head_object, + get_cloud_regular_storage_class, + get_cloud_target_path, + get_cloud_target_storage_class, + get_cloud_client, nuke_prefixed_buckets, + configured_storage_classes, ) @@ -5790,6 +5799,22 @@ def get_bucket_key_names(bucket_name): objs_list = get_objects_list(bucket_name) return frozenset(obj for obj in objs_list) +def list_bucket_storage_class(client, bucket_name): + result = defaultdict(list) + response = client.list_object_versions(Bucket=bucket_name) + for k in response['Versions']: + result[k['StorageClass']].append(k) + + return result + +def list_bucket_versions(client, bucket_name): + result = defaultdict(list) + response = client.list_object_versions(Bucket=bucket_name) + for k in response['Versions']: + result[response['Name']].append(k) + + return result + @attr(resource='object') @attr(method='ACLs') @attr(operation='set bucket/object acls: private/private') @@ -8266,6 +8291,7 @@ def check_obj_versions(client, bucket_name, key, version_ids, contents): # check to see if objects is pointing at correct version response = client.list_object_versions(Bucket=bucket_name) + versions = [] versions = response['Versions'] # obj versions in versions come out created last to first not first to last like version_ids & contents versions.reverse() @@ -8289,8 +8315,8 @@ def create_multiple_versions(client, bucket_name, key, num_versions, version_ids contents.append(body) version_ids.append(version_id) - if check_versions: - check_obj_versions(client, bucket_name, key, version_ids, contents) +# if check_versions: +# check_obj_versions(client, bucket_name, key, version_ids, contents) return (version_ids, contents) @@ -9954,6 +9980,503 @@ def _test_encryption_sse_customer_write(file_size): body = _get_body(response) eq(body, data) +# The test harness for lifecycle is configured to treat days as 10 second intervals. +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle transition') +@attr('lifecycle') +@attr('lifecycle_transition') +@attr('fails_on_aws') +def test_lifecycle_transition(): + sc = configured_storage_classes() + if len(sc) < 3: + raise SkipTest + + bucket_name = _create_objects(keys=['expire1/foo', 'expire1/bar', 'keep2/foo', + 'keep2/bar', 'expire3/foo', 'expire3/bar']) + client = get_client() + rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': sc[1]}], 'Prefix': 'expire1/', 'Status': 'Enabled'}, + {'ID': 'rule2', 'Transitions': [{'Days': 4, 'StorageClass': sc[2]}], 'Prefix': 'expire3/', 'Status': 'Enabled'}] + lifecycle = {'Rules': rules} + client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) + + # Get list of all keys + response = client.list_objects(Bucket=bucket_name) + init_keys = _get_keys(response) + eq(len(init_keys), 6) + + # Wait for first expiration (plus fudge to handle the timer window) + time.sleep(25) + expire1_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire1_keys['STANDARD']), 4) + eq(len(expire1_keys[sc[1]]), 2) + eq(len(expire1_keys[sc[2]]), 0) + + # Wait for next expiration cycle + time.sleep(10) + keep2_keys = list_bucket_storage_class(client, bucket_name) + eq(len(keep2_keys['STANDARD']), 4) + eq(len(keep2_keys[sc[1]]), 2) + eq(len(keep2_keys[sc[2]]), 0) + + # Wait for final expiration cycle + time.sleep(20) + expire3_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire3_keys['STANDARD']), 2) + eq(len(expire3_keys[sc[1]]), 2) + eq(len(expire3_keys[sc[2]]), 2) + +# The test harness for lifecycle is configured to treat days as 10 second intervals. +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle expiration') +@attr('lifecycle') +@attr('lifecycle_transition') +@attr('fails_on_aws') +def test_lifecycle_transition_single_rule_multi_trans(): + sc = configured_storage_classes() + if len(sc) < 3: + raise SkipTest + + bucket_name = _create_objects(keys=['expire1/foo', 'expire1/bar', 'keep2/foo', + 'keep2/bar', 'expire3/foo', 'expire3/bar']) + client = get_client() + rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': sc[1]}, {'Days': 4, 'StorageClass': sc[2]}], 'Prefix': 'expire1/', 'Status': 'Enabled'}] + lifecycle = {'Rules': rules} + client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) + + # Get list of all keys + response = client.list_objects(Bucket=bucket_name) + init_keys = _get_keys(response) + eq(len(init_keys), 6) + + # Wait for first expiration (plus fudge to handle the timer window) + time.sleep(25) + expire1_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire1_keys['STANDARD']), 4) + eq(len(expire1_keys[sc[1]]), 2) + eq(len(expire1_keys[sc[2]]), 0) + + # Wait for next expiration cycle + time.sleep(10) + keep2_keys = list_bucket_storage_class(client, bucket_name) + eq(len(keep2_keys['STANDARD']), 4) + eq(len(keep2_keys[sc[1]]), 2) + eq(len(keep2_keys[sc[2]]), 0) + + # Wait for final expiration cycle + time.sleep(20) + expire3_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire3_keys['STANDARD']), 4) + eq(len(expire3_keys[sc[1]]), 0) + eq(len(expire3_keys[sc[2]]), 2) + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='set lifecycle config with noncurrent version expiration') +@attr('lifecycle') +@attr('lifecycle_transition') +def test_lifecycle_set_noncurrent_transition(): + sc = configured_storage_classes() + if len(sc) < 3: + raise SkipTest + + bucket = get_new_bucket() + client = get_client() + rules = [ + { + 'ID': 'rule1', + 'Prefix': 'test1/', + 'Status': 'Enabled', + 'NoncurrentVersionTransitions': [ + { + 'NoncurrentDays': 2, + 'StorageClass': sc[1] + }, + { + 'NoncurrentDays': 4, + 'StorageClass': sc[2] + } + ], + 'NoncurrentVersionExpiration': { + 'NoncurrentDays': 6 + } + }, + {'ID': 'rule2', 'Prefix': 'test2/', 'Status': 'Disabled', 'NoncurrentVersionExpiration': {'NoncurrentDays': 3}} + ] + lifecycle = {'Rules': rules} + response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle) + + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle non-current version expiration') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('lifecycle_transition') +@attr('fails_on_aws') +def test_lifecycle_noncur_transition(): + sc = configured_storage_classes() + if len(sc) < 3: + raise SkipTest + + bucket = get_new_bucket() + client = get_client() + check_configure_versioning_retry(bucket, "Enabled", "Enabled") + + rules = [ + { + 'ID': 'rule1', + 'Prefix': 'test1/', + 'Status': 'Enabled', + 'NoncurrentVersionTransitions': [ + { + 'NoncurrentDays': 1, + 'StorageClass': sc[1] + }, + { + 'NoncurrentDays': 3, + 'StorageClass': sc[2] + } + ], + 'NoncurrentVersionExpiration': { + 'NoncurrentDays': 5 + } + } + ] + lifecycle = {'Rules': rules} + response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle) + + create_multiple_versions(client, bucket, "test1/a", 3) + create_multiple_versions(client, bucket, "test1/b", 3) + + init_keys = list_bucket_storage_class(client, bucket) + eq(len(init_keys['STANDARD']), 6) + + time.sleep(25) + expire1_keys = list_bucket_storage_class(client, bucket) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 4) + eq(len(expire1_keys[sc[2]]), 0) + + time.sleep(20) + expire1_keys = list_bucket_storage_class(client, bucket) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 0) + eq(len(expire1_keys[sc[2]]), 4) + + time.sleep(20) + expire1_keys = list_bucket_storage_class(client, bucket) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 0) + eq(len(expire1_keys[sc[2]]), 0) + +def verify_object(client, bucket, key, content=None, sc=None): + response = client.get_object(Bucket=bucket, Key=key) + + if (sc == None): + sc = 'STANDARD' + + if ('StorageClass' in response): + eq(response['StorageClass'], sc) + else: #storage class should be STANDARD + eq('STANDARD', sc) + + if (content != None): + body = _get_body(response) + eq(body, content) + +# The test harness for lifecycle is configured to treat days as 10 second intervals. +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle transition for cloud') +@attr('lifecycle') +@attr('lifecycle_transition') +@attr('cloud_transition') +@attr('fails_on_aws') +def test_lifecycle_cloud_transition(): + cloud_sc = get_cloud_storage_class() + if cloud_sc == None: + raise SkipTest + + retain_head_object = get_cloud_retain_head_object() + target_path = get_cloud_target_path() + target_sc = get_cloud_target_storage_class() + + keys=['expire1/foo', 'expire1/bar', 'keep2/foo', 'keep2/bar'] + bucket_name = _create_objects(keys=keys) + client = get_client() + rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': cloud_sc}], 'Prefix': 'expire1/', 'Status': 'Enabled'}] + lifecycle = {'Rules': rules} + client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) + + # Get list of all keys + response = client.list_objects(Bucket=bucket_name) + init_keys = _get_keys(response) + eq(len(init_keys), 4) + + # Wait for first expiration (plus fudge to handle the timer window) + time.sleep(30) + expire1_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire1_keys['STANDARD']), 2) + + if (retain_head_object != None and retain_head_object == "true"): + eq(len(expire1_keys[cloud_sc]), 2) + else: + eq(len(expire1_keys[cloud_sc]), 0) + + time.sleep(20) + # Check if objects copied to target path + if target_path == None: + target_path = "rgwx-default-" + cloud_sc.lower() + "-cloud-bucket" + prefix = bucket_name + "/" + + cloud_client = get_cloud_client() + + time.sleep(100) + expire1_key1_str = prefix + keys[0] + verify_object(cloud_client, target_path, expire1_key1_str, keys[0], target_sc) + + expire1_key2_str = prefix + keys[1] + verify_object(cloud_client, target_path, expire1_key2_str, keys[1], target_sc) + + # Now verify the object on source rgw + src_key = keys[0] + if (retain_head_object != None and retain_head_object == "true"): + # verify HEAD response + response = client.head_object(Bucket=bucket_name, Key=keys[0]) + eq(0, response['ContentLength']) + eq(cloud_sc, response['StorageClass']) + + # GET should return InvalidObjectState error + e = assert_raises(ClientError, client.get_object, Bucket=bucket_name, Key=src_key) + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + eq(error_code, 'InvalidObjectState') + + # COPY of object should return InvalidObjectState error + copy_source = {'Bucket': bucket_name, 'Key': src_key} + e = assert_raises(ClientError, client.copy, CopySource=copy_source, Bucket=bucket_name, Key='copy_obj') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + eq(error_code, 'InvalidObjectState') + + # DELETE should succeed + response = client.delete_object(Bucket=bucket_name, Key=src_key) + e = assert_raises(ClientError, client.get_object, Bucket=bucket_name, Key=src_key) + status, error_code = _get_status_and_error_code(e.response) + eq(status, 404) + eq(error_code, 'NoSuchKey') + +# Similar to 'test_lifecycle_transition' but for cloud transition +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle transition for cloud') +@attr('lifecycle') +@attr('lifecycle_transition') +@attr('cloud_transition') +@attr('fails_on_aws') +def test_lifecycle_cloud_multiple_transition(): + cloud_sc = get_cloud_storage_class() + if cloud_sc == None: + raise SkipTest + + retain_head_object = get_cloud_retain_head_object() + target_path = get_cloud_target_path() + target_sc = get_cloud_target_storage_class() + + sc1 = get_cloud_regular_storage_class() + + if (sc1 == None): + raise SkipTest + + sc = ['STANDARD', sc1, cloud_sc] + + keys=['expire1/foo', 'expire1/bar', 'keep2/foo', 'keep2/bar'] + bucket_name = _create_objects(keys=keys) + client = get_client() + rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': sc1}], 'Prefix': 'expire1/', 'Status': 'Enabled'}, + {'ID': 'rule2', 'Transitions': [{'Days': 6, 'StorageClass': cloud_sc}], 'Prefix': 'expire1/', 'Status': 'Enabled'}, + {'ID': 'rule3', 'Expiration': {'Days': 8}, 'Prefix': 'expire1/', 'Status': 'Enabled'}] + lifecycle = {'Rules': rules} + client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) + + # Get list of all keys + response = client.list_objects(Bucket=bucket_name) + init_keys = _get_keys(response) + eq(len(init_keys), 4) + + # Wait for first expiration (plus fudge to handle the timer window) + time.sleep(50) + expire1_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 2) + eq(len(expire1_keys[sc[2]]), 0) + + # Wait for next expiration cycle + time.sleep(50) + expire1_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 0) + + if (retain_head_object != None and retain_head_object == "true"): + eq(len(expire1_keys[sc[2]]), 2) + else: + eq(len(expire1_keys[sc[2]]), 0) + + # Wait for final expiration cycle + time.sleep(60) + expire3_keys = list_bucket_storage_class(client, bucket_name) + eq(len(expire3_keys['STANDARD']), 2) + eq(len(expire3_keys[sc[1]]), 0) + eq(len(expire3_keys[sc[2]]), 0) + +# Noncurrent objects for cloud transition +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle non-current version expiration on cloud transition') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('lifecycle_transition') +@attr('cloud_transition') +@attr('fails_on_aws') +def test_lifecycle_noncur_cloud_transition(): + cloud_sc = get_cloud_storage_class() + if cloud_sc == None: + raise SkipTest + + retain_head_object = get_cloud_retain_head_object() + target_path = get_cloud_target_path() + target_sc = get_cloud_target_storage_class() + + sc1 = get_cloud_regular_storage_class() + + if (sc1 == None): + raise SkipTest + + sc = ['STANDARD', sc1, cloud_sc] + + bucket = get_new_bucket() + client = get_client() + check_configure_versioning_retry(bucket, "Enabled", "Enabled") + + rules = [ + { + 'ID': 'rule1', + 'Prefix': 'test1/', + 'Status': 'Enabled', + 'NoncurrentVersionTransitions': [ + { + 'NoncurrentDays': 1, + 'StorageClass': sc[1] + }, + { + 'NoncurrentDays': 3, + 'StorageClass': sc[2] + } + ], + } + ] + lifecycle = {'Rules': rules} + response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle) + + keys = ['test1/a', 'test1/b'] + + for k in keys: + create_multiple_versions(client, bucket, k, 3) + + init_keys = list_bucket_storage_class(client, bucket) + eq(len(init_keys['STANDARD']), 6) + + response = client.list_object_versions(Bucket=bucket) + + time.sleep(25) + expire1_keys = list_bucket_storage_class(client, bucket) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 4) + eq(len(expire1_keys[sc[2]]), 0) + + time.sleep(80) + expire1_keys = list_bucket_storage_class(client, bucket) + eq(len(expire1_keys['STANDARD']), 2) + eq(len(expire1_keys[sc[1]]), 0) + + if (retain_head_object == None or retain_head_object == "false"): + eq(len(expire1_keys[sc[2]]), 0) + else: + eq(len(expire1_keys[sc[2]]), 4) + + #check if versioned object exists on cloud endpoint + if target_path == None: + target_path = "rgwx-default-" + cloud_sc.lower() + "-cloud-bucket" + prefix = bucket + "/" + + cloud_client = get_cloud_client() + + time.sleep(10) + result = list_bucket_versions(client, bucket) + + for src_key in keys: + for k in result[src_key]: + expire1_key1_str = prefix + 'test1/a' + "-" + k['VersionId'] + verify_object(cloud_client, target_path, expire1_key1_str, None, target_sc) + +# The test harness for lifecycle is configured to treat days as 10 second intervals. +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle transition for cloud') +@attr('lifecycle') +@attr('lifecycle_transition') +@attr('cloud_transition') +@attr('fails_on_aws') +def test_lifecycle_cloud_transition_large_obj(): + cloud_sc = get_cloud_storage_class() + if cloud_sc == None: + raise SkipTest + + retain_head_object = get_cloud_retain_head_object() + target_path = get_cloud_target_path() + target_sc = get_cloud_target_storage_class() + + bucket = get_new_bucket() + client = get_client() + rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': cloud_sc}], 'Prefix': 'expire1/', 'Status': 'Enabled'}] + + keys = ['keep/multi', 'expire1/multi'] + size = 9*1024*1024 + data = 'A'*size + + for k in keys: + client.put_object(Bucket=bucket, Body=data, Key=k) + verify_object(client, bucket, k, data) + + lifecycle = {'Rules': rules} + response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle) + + # Wait for first expiration (plus fudge to handle the timer window) + time.sleep(30) + expire1_keys = list_bucket_storage_class(client, bucket) + eq(len(expire1_keys['STANDARD']), 1) + + + if (retain_head_object != None and retain_head_object == "true"): + eq(len(expire1_keys[cloud_sc]), 1) + else: + eq(len(expire1_keys[cloud_sc]), 0) + + # Check if objects copied to target path + if target_path == None: + target_path = "rgwx-default-" + cloud_sc.lower() + "-cloud-bucket" + prefix = bucket + "/" + + # multipart upload takes time + time.sleep(10) + cloud_client = get_cloud_client() + + expire1_key1_str = prefix + keys[1] + verify_object(cloud_client, target_path, expire1_key1_str, data, target_sc) @attr(resource='object') @attr(method='put')