object-lock: test changes between retention modes

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit 8662815ebe)

README.rst

@@ -54,3 +54,20 @@ You can run only the boto3 tests with::
 
         S3TEST_CONF=your.conf ./virtualenv/bin/nosetests -v -s -A 'not fails_on_rgw' s3tests_boto3.functional
 
+========================
+ STS compatibility tests
+========================
+
+This section contains some basic tests for the AssumeRole, GetSessionToken and AssumeRoleWithWebIdentity API's. The test file is located under ``s3tests_boto3/functional``.
+
+You can run only the sts tests (all the three API's) with::
+
+        S3TEST_CONF=your.conf ./virtualenv/bin/nosetests s3tests_boto3.functional.test_sts
+
+You can filter tests based on the attributes. There is a attribute named ``test_of_sts`` to run AssumeRole and GetSessionToken tests and ``webidentity_test`` to run the AssumeRoleWithWebIdentity tests. If you want to execute only ``test_of_sts`` tests you can apply that filter as below::
+
+        S3TEST_CONF=your.conf ./virtualenv/bin/nosetests -v -s -A 'test_of_sts' s3tests_boto3.functional.test_sts
+
+For running ``webidentity_test`` you'll need have Keycloak running.
+
+In order to run any STS test you'll need to add "iam" section to the config file. For further reference on how your config file should look check ``s3tests.conf.SAMPLE``.

bootstrap

@@ -59,13 +59,13 @@ esac
 
 
 # s3-tests only works on python 3.6 not newer versions of python3
-${virtualenv} --python=$(which python3.6) --no-site-packages --distribute virtualenv
+${virtualenv} --python=$(which python3.6) virtualenv
 
 # avoid pip bugs
 ./virtualenv/bin/pip3 install --upgrade pip
 
-# slightly old version of setuptools; newer fails w/ requests 0.14.0
+# latest setuptools supporting python 2.7
-./virtualenv/bin/pip3 install setuptools==32.3.1
+./virtualenv/bin/pip install setuptools==44.1.0
 
 ./virtualenv/bin/pip3 install -r requirements.txt
 

requirements.txt

@@ -6,7 +6,7 @@ munch >=2.0.0
 # 0.14 switches to libev, that means bootstrap needs to change too
 gevent >=1.0
 isodate >=0.4.4
-requests >=0.14.0
+requests >=2.23.0
 pytz >=2011k
 httplib2
 lxml

s3tests.conf.SAMPLE

@@ -10,6 +10,9 @@ port = 8000
 ## say "False" to disable TLS
 is_secure = False
 
+## say "False" to disable SSL Verify
+ssl_verify = False
+
 [fixtures]
 ## all the buckets created will start with this prefix;
 ## {random} will be filled with random characters to pad
@@ -68,3 +71,33 @@ secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
 
 # tenant email set in vstart.sh
 email = tenanteduser@example.com
+
+#following section needs to be added for all sts-tests
+[iam]
+#used for iam operations in sts-tests
+#email from vstart.sh
+email = s3@example.com
+
+#user_id from vstart.sh
+user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+#access_key from vstart.sh
+access_key = ABCDEFGHIJKLMNOPQRST
+
+#secret_key vstart.sh
+secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn
+
+#display_name from vstart.sh
+display_name = youruseridhere
+
+#following section needs to be added when you want to run Assume Role With Webidentity test
+[webidentity]
+#used for assume role with web identity test in sts-tests
+#all parameters will be obtained from ceph/qa/tasks/keycloak.py
+token=<access_token>
+
+aud=<obtained after introspecting token>
+
+thumbprint=<obtained from x509 certificate>
+
+KC_REALM=<name of the realm>

s3tests/functional/test_s3_website.py

@@ -7,6 +7,7 @@ import random
 from pprint import pprint
 import time
 import boto.exception
+import socket
 
 from urllib.parse import urlparse
 
@@ -520,6 +521,57 @@ def test_website_private_bucket_list_empty_blockederrordoc():
     errorhtml.delete()
     bucket.delete()
 
+@attr(resource='bucket')
+@attr(method='get')
+@attr(operation='list')
+@attr(assertion='check if there is an invalid payload after serving error doc')
+@attr('s3website')
+@nose.with_setup(setup=check_can_test_website, teardown=common.teardown)
+def test_website_public_bucket_list_pubilc_errordoc():
+    bucket = get_new_bucket()
+    f = _test_website_prep(bucket, WEBSITE_CONFIGS_XMLFRAG['IndexDocErrorDoc'])
+    bucket.make_public()
+    errorhtml = bucket.new_key(f['ErrorDocument_Key'])
+    errorstring = choose_bucket_prefix(template=ERRORDOC_TEMPLATE, max_len=256)
+    errorhtml.set_contents_from_string(errorstring)
+    errorhtml.set_canned_acl('public-read')
+
+    url = get_website_url(proto='http', bucket=bucket.name, path='')
+    o = urlparse(url)
+    host = o.hostname
+    port = s3.main.port
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.connect((host, port))
+
+    request = "GET / HTTP/1.1\r\nHost:%s.%s:%s\r\n\r\n" % (bucket.name, host, port)
+    sock.send(request.encode())
+    
+    #receive header
+    resp = sock.recv(4096)
+    print(resp)
+
+    #receive body
+    resp = sock.recv(4096)
+    print('payload length=%d' % len(resp))
+    print(resp)
+
+    #check if any additional payload is left
+    resp_len = 0
+    sock.settimeout(2)
+    try:
+        resp = sock.recv(4096)
+        resp_len = len(resp)
+        print('invalid payload length=%d' % resp_len)
+        print(resp)
+    except socket.timeout:
+        print('no invalid payload')
+
+    ok(resp_len == 0, 'invalid payload')
+
+    errorhtml.delete()
+    bucket.delete()
+
 @attr(resource='bucket')
 @attr(method='get')
 @attr(operation='list')

s3tests_boto3/functional/__init__.py

@@ -4,11 +4,14 @@ from botocore.client import Config
 from botocore.exceptions import ClientError
 from botocore.handlers import disable_signing
 import configparser
+import datetime
+import time
 import os
 import munch
 import random
 import string
 import itertools
+import urllib3
 
 config = munch.Munch
 
@@ -74,38 +77,69 @@ def get_objects_list(bucket, client=None, prefix=None):
 
     return objects_list
 
-def get_versioned_objects_list(bucket, client=None):
+# generator function that returns object listings in batches, where each
-    if client == None:
+# batch is a list of dicts compatible with delete_objects()
-        client = get_client()
+def list_versions(client, bucket, batch_size):
-    response = client.list_object_versions(Bucket=bucket)
+    key_marker = ''
-    versioned_objects_list = []
+    version_marker = ''
+    truncated = True
+    while truncated:
+        listing = client.list_object_versions(
+                Bucket=bucket,
+                KeyMarker=key_marker,
+                VersionIdMarker=version_marker,
+                MaxKeys=batch_size)
 
-    if 'Versions' in response:
+        key_marker = listing.get('NextKeyMarker')
-        contents = response['Versions']
+        version_marker = listing.get('NextVersionIdMarker')
-        for obj in contents:
+        truncated = listing['IsTruncated']
-            key = obj['Key']
-            version_id = obj['VersionId']
-            versioned_obj = (key,version_id)
-            versioned_objects_list.append(versioned_obj)
 
-    return versioned_objects_list
+        objs = listing.get('Versions', []) + listing.get('DeleteMarkers', [])
+        if len(objs):
+            yield [{'Key': o['Key'], 'VersionId': o['VersionId']} for o in objs]
 
-def get_delete_markers_list(bucket, client=None):
+def nuke_bucket(client, bucket):
-    if client == None:
+    batch_size = 128
-        client = get_client()
+    max_retain_date = None
-    response = client.list_object_versions(Bucket=bucket)
-    delete_markers = []
 
-    if 'DeleteMarkers' in response:
+    # list and delete objects in batches
-        contents = response['DeleteMarkers']
+    for objects in list_versions(client, bucket, batch_size):
-        for obj in contents:
+        delete = client.delete_objects(Bucket=bucket,
-            key = obj['Key']
+                Delete={'Objects': objects, 'Quiet': True},
-            version_id = obj['VersionId']
+                BypassGovernanceRetention=True)
-            versioned_obj = (key,version_id)
-            delete_markers.append(versioned_obj)
 
-    return delete_markers
+        # check for object locks on 403 AccessDenied errors
+        for err in delete.get('Errors', []):
+            if err.get('Code') != 'AccessDenied':
+                continue
+            try:
+                res = client.get_object_retention(Bucket=bucket,
+                        Key=err['Key'], VersionId=err['VersionId'])
+                retain_date = res['Retention']['RetainUntilDate']
+                if not max_retain_date or max_retain_date < retain_date:
+                    max_retain_date = retain_date
+            except ClientError:
+                pass
 
+    if max_retain_date:
+        # wait out the retention period (up to 60 seconds)
+        now = datetime.datetime.now(max_retain_date.tzinfo)
+        if max_retain_date > now:
+            delta = max_retain_date - now
+            if delta.total_seconds() > 60:
+                raise RuntimeError('bucket {} still has objects \
+locked for {} more seconds, not waiting for \
+bucket cleanup'.format(bucket, delta.total_seconds()))
+            print('nuke_bucket', bucket, 'waiting', delta.total_seconds(),
+                    'seconds for object locks to expire')
+            time.sleep(delta.total_seconds())
+
+        for objects in list_versions(client, bucket, batch_size):
+            client.delete_objects(Bucket=bucket,
+                    Delete={'Objects': objects, 'Quiet': True},
+                    BypassGovernanceRetention=True)
+
+    client.delete_bucket(Bucket=bucket)
 
 def nuke_prefixed_buckets(prefix, client=None):
     if client == None:
@@ -113,24 +147,19 @@ def nuke_prefixed_buckets(prefix, client=None):
 
     buckets = get_buckets_list(client, prefix)
 
-    if buckets != []:
+    err = None
     for bucket_name in buckets:
-            objects_list = get_objects_list(bucket_name, client)
-            for obj in objects_list:
-                response = client.delete_object(Bucket=bucket_name,Key=obj)
-            versioned_objects_list = get_versioned_objects_list(bucket_name, client)
-            for obj in versioned_objects_list:
-                response = client.delete_object(Bucket=bucket_name,Key=obj[0],VersionId=obj[1])
-            delete_markers = get_delete_markers_list(bucket_name, client)
-            for obj in delete_markers:
-                response = client.delete_object(Bucket=bucket_name,Key=obj[0],VersionId=obj[1])
         try:
-                response = client.delete_bucket(Bucket=bucket_name)
+            nuke_bucket(client, bucket_name)
-            except ClientError:
+        except Exception as e:
-                # if DELETE times out, the retry may see NoSuchBucket
+            # The exception shouldn't be raised when doing cleanup. Pass and continue
-                if response['Error']['Code'] != 'NoSuchBucket':
+            # the bucket cleanup process. Otherwise left buckets wouldn't be cleared
-                    raise ClientError
+            # resulting in some kind of resource leak. err is used to hint user some
+            # exception once occurred.
+            err = e
             pass
+    if err:
+        raise err
 
     print('Done with cleanup of buckets in tests.')
 
@@ -166,6 +195,15 @@ def setup():
     proto = 'https' if config.default_is_secure else 'http'
     config.default_endpoint = "%s://%s:%d" % (proto, config.default_host, config.default_port)
 
+    try:
+        config.default_ssl_verify = cfg.getboolean('DEFAULT', "ssl_verify")
+    except configparser.NoOptionError:
+        config.default_ssl_verify = False
+
+    # Disable InsecureRequestWarning reported by urllib3 when ssl_verify is False
+    if not config.default_ssl_verify:
+        urllib3.disable_warnings()
+
     # vars from the main section
     config.main_access_key = cfg.get('s3 main',"access_key")
     config.main_secret_key = cfg.get('s3 main',"secret_key")
@@ -213,12 +251,49 @@ def setup():
     nuke_prefixed_buckets(prefix=prefix, client=alt_client)
     nuke_prefixed_buckets(prefix=prefix, client=tenant_client)
 
+
 def teardown():
     alt_client = get_alt_client()
     tenant_client = get_tenant_client()
     nuke_prefixed_buckets(prefix=prefix)
     nuke_prefixed_buckets(prefix=prefix, client=alt_client)
     nuke_prefixed_buckets(prefix=prefix, client=tenant_client)
+    try:
+        iam_client = get_iam_client()
+        list_roles_resp = iam_client.list_roles()
+        for role in list_roles_resp['Roles']:
+            list_policies_resp = iam_client.list_role_policies(RoleName=role['RoleName'])
+            for policy in list_policies_resp['PolicyNames']:
+                del_policy_resp = iam_client.delete_role_policy(
+                                         RoleName=role['RoleName'],
+                                         PolicyName=policy
+                                        )
+            del_role_resp = iam_client.delete_role(RoleName=role['RoleName'])
+        list_oidc_resp = iam_client.list_open_id_connect_providers()
+        for oidcprovider in list_oidc_resp['OpenIDConnectProviderList']:
+            del_oidc_resp = iam_client.delete_open_id_connect_provider(
+                        OpenIDConnectProviderArn=oidcprovider['Arn']
+                    )
+    except:
+        pass
+
+def check_webidentity():
+    cfg = configparser.RawConfigParser()
+    try:
+        path = os.environ['S3TEST_CONF']
+    except KeyError:
+        raise RuntimeError(
+            'To run tests, point environment '
+            + 'variable S3TEST_CONF to a config file.',
+            )
+    cfg.read(path)
+    if not cfg.has_section("webidentity"):
+        raise RuntimeError('Your config file is missing the "webidentity" section!')
+
+    config.webidentity_thumbprint = cfg.get('webidentity', "thumbprint")
+    config.webidentity_aud = cfg.get('webidentity', "aud")
+    config.webidentity_token = cfg.get('webidentity', "token")
+    config.webidentity_realm = cfg.get('webidentity', "KC_REALM")
 
 def get_client(client_config=None):
     if client_config == None:
@@ -229,6 +304,7 @@ def get_client(client_config=None):
                         aws_secret_access_key=config.main_secret_key,
                         endpoint_url=config.default_endpoint,
                         use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
                         config=client_config)
     return client
 
@@ -238,9 +314,56 @@ def get_v2_client():
                         aws_secret_access_key=config.main_secret_key,
                         endpoint_url=config.default_endpoint,
                         use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
                         config=Config(signature_version='s3'))
     return client
 
+def get_sts_client(client_config=None):
+    if client_config == None:
+        client_config = Config(signature_version='s3v4')
+
+    client = boto3.client(service_name='sts',
+                        aws_access_key_id=config.alt_access_key,
+                        aws_secret_access_key=config.alt_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        region_name='',
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
+                        config=client_config)
+    return client
+
+def get_iam_client(client_config=None):
+    cfg = configparser.RawConfigParser()
+    try:
+        path = os.environ['S3TEST_CONF']
+    except KeyError:
+        raise RuntimeError(
+            'To run tests, point environment '
+            + 'variable S3TEST_CONF to a config file.',
+            )
+    cfg.read(path)
+    if not cfg.has_section("iam"):
+        raise RuntimeError('Your config file is missing the "iam" section!')
+
+    config.iam_access_key = cfg.get('iam',"access_key")
+    config.iam_secret_key = cfg.get('iam',"secret_key")
+    config.iam_display_name = cfg.get('iam',"display_name")
+    config.iam_user_id = cfg.get('iam',"user_id")
+    config.iam_email = cfg.get('iam',"email")    
+
+    if client_config == None:
+        client_config = Config(signature_version='s3v4')
+    
+    client = boto3.client(service_name='iam',
+                        aws_access_key_id=config.iam_access_key,
+                        aws_secret_access_key=config.iam_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        region_name='',
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
+                        config=client_config)
+    return client
+
 def get_alt_client(client_config=None):
     if client_config == None:
         client_config = Config(signature_version='s3v4')
@@ -250,6 +373,7 @@ def get_alt_client(client_config=None):
                         aws_secret_access_key=config.alt_secret_key,
                         endpoint_url=config.default_endpoint,
                         use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
                         config=client_config)
     return client
 
@@ -262,6 +386,7 @@ def get_tenant_client(client_config=None):
                         aws_secret_access_key=config.tenant_secret_key,
                         endpoint_url=config.default_endpoint,
                         use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
                         config=client_config)
     return client
 
@@ -272,6 +397,7 @@ def get_tenant_iam_client():
                           aws_access_key_id=config.tenant_access_key,
                           aws_secret_access_key=config.tenant_secret_key,
                           endpoint_url=config.default_endpoint,
+                          verify=config.default_ssl_verify,
                           use_ssl=config.default_is_secure)
     return client
 
@@ -281,6 +407,7 @@ def get_unauthenticated_client():
                         aws_secret_access_key='',
                         endpoint_url=config.default_endpoint,
                         use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
                         config=Config(signature_version=UNSIGNED))
     return client
 
@@ -290,9 +417,23 @@ def get_bad_auth_client(aws_access_key_id='badauth'):
                         aws_secret_access_key='roflmao',
                         endpoint_url=config.default_endpoint,
                         use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
                         config=Config(signature_version='s3v4'))
     return client
 
+def get_svc_client(client_config=None, svc='s3'):
+    if client_config == None:
+        client_config = Config(signature_version='s3v4')
+
+    client = boto3.client(service_name=svc,
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify,
+                        config=client_config)
+    return client
+
 bucket_counter = itertools.count(1)
 
 def get_new_bucket_name():
@@ -320,7 +461,8 @@ def get_new_bucket_resource(name=None):
                         aws_access_key_id=config.main_access_key,
                         aws_secret_access_key=config.main_secret_key,
                         endpoint_url=config.default_endpoint,
-                        use_ssl=config.default_is_secure)
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
     if name is None:
         name = get_new_bucket_name()
     bucket = s3.Bucket(name)
@@ -342,6 +484,21 @@ def get_new_bucket(client=None, name=None):
     client.create_bucket(Bucket=name)
     return name
 
+def get_parameter_name():
+    parameter_name=""
+    rand = ''.join(
+        random.choice(string.ascii_lowercase + string.digits)
+        for c in range(255)
+        )
+    while rand:
+        parameter_name = '{random}'.format(random=rand)
+        if len(parameter_name) <= 10:
+            return parameter_name
+        rand = rand[:-1]
+    return parameter_name
+
+def get_sts_user_id():
+    return config.alt_user_id
 
 def get_config_is_secure():
     return config.default_is_secure
@@ -355,6 +512,9 @@ def get_config_port():
 def get_config_endpoint():
     return config.default_endpoint
 
+def get_config_ssl_verify():
+    return config.default_ssl_verify
+
 def get_main_aws_access_key():
     return config.main_access_key
 
@@ -408,3 +568,21 @@ def get_tenant_user_id():
 
 def get_tenant_email():
     return config.tenant_email
+
+def get_thumbprint():
+    return config.webidentity_thumbprint
+
+def get_aud():
+    return config.webidentity_aud
+
+def get_token():
+    return config.webidentity_token
+
+def get_realm_name():
+    return config.webidentity_realm
+
+def get_iam_access_key():
+    return config.iam_access_key
+
+def get_iam_secret_key():
+    return config.iam_secret_key