mirror of
https://github.com/ceph/s3-tests.git
synced 2024-12-23 13:41:52 +00:00
add flexible multi-region configuration
can now create a region-specific configuration: [region foo] api_name = ... host = ... port = ... and set that region to be used for specific connection: [s3 main] region = foo Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
This commit is contained in:
Yehuda Sadeh
parent
c40b1cd348
commit
232dd35817
2 changed files with 93 additions and 47 deletions
|
|
@ -9,10 +9,18 @@ import string
|
|||
|
||||
s3 = bunch.Bunch()
|
||||
config = bunch.Bunch()
|
||||
regions = bunch.Bunch()
|
||||
targets = bunch.Bunch()
|
||||
|
||||
# this will be assigned by setup()
|
||||
prefix = None
|
||||
|
||||
calling_formats = dict(
|
||||
ordinary=boto.s3.connection.OrdinaryCallingFormat(),
|
||||
subdomain=boto.s3.connection.SubdomainCallingFormat(),
|
||||
vhost=boto.s3.connection.VHostCallingFormat(),
|
||||
)
|
||||
|
||||
def get_prefix():
|
||||
assert prefix is not None
|
||||
return prefix
|
||||
|
|
@ -71,6 +79,48 @@ def nuke_prefixed_buckets(prefix):
|
|||
print 'Done with cleanup of test buckets.'
|
||||
|
||||
|
||||
class TargetConfig:
|
||||
def __init__(self, cfg, section):
|
||||
self.port = None
|
||||
self.api_name = ''
|
||||
self.is_master = False
|
||||
self.is_secure = False
|
||||
try:
|
||||
self.api_name = cfg.get(section, 'api_name')
|
||||
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
|
||||
pass
|
||||
try:
|
||||
self.port = cfg.getint(section, 'port')
|
||||
except ConfigParser.NoOptionError:
|
||||
pass
|
||||
try:
|
||||
self.host=cfg.get(section, 'host')
|
||||
except ConfigParser.NoOptionError:
|
||||
raise RuntimeError(
|
||||
'host not specified for section {s}'.format(s=section)
|
||||
)
|
||||
try:
|
||||
self.is_secure=cfg.getboolean(section, 'is_secure')
|
||||
except ConfigParser.NoOptionError:
|
||||
pass
|
||||
|
||||
try:
|
||||
raw_calling_format = cfg.get(section, 'calling_format')
|
||||
except ConfigParser.NoOptionError:
|
||||
raw_calling_format = 'ordinary'
|
||||
|
||||
try:
|
||||
self.calling_format = calling_formats[raw_calling_format]
|
||||
except KeyError:
|
||||
raise RuntimeError(
|
||||
'calling_format unknown: %r' % raw_calling_format
|
||||
)
|
||||
|
||||
class TargetConnection:
|
||||
def __init__(self, conf, conn):
|
||||
self.conf = conf
|
||||
self.connection = conn
|
||||
|
||||
# nosetests --processes=N with N>1 is safe
|
||||
_multiprocess_can_split_ = True
|
||||
|
||||
|
|
@ -88,25 +138,28 @@ def setup():
|
|||
cfg.readfp(f)
|
||||
|
||||
global prefix
|
||||
global location
|
||||
global targets
|
||||
|
||||
try:
|
||||
template = cfg.get('fixtures', 'bucket prefix')
|
||||
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
|
||||
template = 'test-{random}-'
|
||||
prefix = choose_bucket_prefix(template=template)
|
||||
|
||||
try:
|
||||
location = cfg.get('region main', 'name')
|
||||
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
|
||||
location = ''
|
||||
|
||||
s3.clear()
|
||||
config.clear()
|
||||
calling_formats = dict(
|
||||
ordinary=boto.s3.connection.OrdinaryCallingFormat(),
|
||||
subdomain=boto.s3.connection.SubdomainCallingFormat(),
|
||||
vhost=boto.s3.connection.VHostCallingFormat(),
|
||||
)
|
||||
regions.clear()
|
||||
|
||||
for section in cfg.sections():
|
||||
try:
|
||||
(type_, name) = section.split(None, 1)
|
||||
except ValueError:
|
||||
continue
|
||||
if type_ != 'region':
|
||||
continue
|
||||
region_conf = TargetConfig(cfg, section)
|
||||
regions[name] = region_conf
|
||||
|
||||
for section in cfg.sections():
|
||||
try:
|
||||
(type_, name) = section.split(None, 1)
|
||||
|
|
@ -114,22 +167,12 @@ def setup():
|
|||
continue
|
||||
if type_ != 's3':
|
||||
continue
|
||||
try:
|
||||
port = cfg.getint(section, 'port')
|
||||
except ConfigParser.NoOptionError:
|
||||
port = None
|
||||
|
||||
try:
|
||||
raw_calling_format = cfg.get(section, 'calling_format')
|
||||
region_name = cfg.get(section, 'region')
|
||||
region_config = regions[region_name]
|
||||
except ConfigParser.NoOptionError:
|
||||
raw_calling_format = 'ordinary'
|
||||
|
||||
try:
|
||||
calling_format = calling_formats[raw_calling_format]
|
||||
except KeyError:
|
||||
raise RuntimeError(
|
||||
'calling_format unknown: %r' % raw_calling_format
|
||||
)
|
||||
region_config = TargetConfig(cfg, section)
|
||||
|
||||
config[name] = bunch.Bunch()
|
||||
for var in [
|
||||
|
|
@ -144,13 +187,14 @@ def setup():
|
|||
conn = boto.s3.connection.S3Connection(
|
||||
aws_access_key_id=cfg.get(section, 'access_key'),
|
||||
aws_secret_access_key=cfg.get(section, 'secret_key'),
|
||||
is_secure=cfg.getboolean(section, 'is_secure'),
|
||||
port=port,
|
||||
host=cfg.get(section, 'host'),
|
||||
is_secure=region_config.is_secure,
|
||||
port=region_config.port,
|
||||
host=region_config.host,
|
||||
# TODO test vhost calling format
|
||||
calling_format=calling_format,
|
||||
calling_format=region_config.calling_format,
|
||||
)
|
||||
s3[name] = conn
|
||||
targets[name] = TargetConnection(region_config, conn)
|
||||
|
||||
# WARNING! we actively delete all buckets we see with the prefix
|
||||
# we've chosen! Choose your prefix with care, and don't reuse
|
||||
|
|
@ -185,19 +229,20 @@ def get_new_bucket_name():
|
|||
return name
|
||||
|
||||
|
||||
def get_new_bucket(connection=None, name=None, headers=None):
|
||||
def get_new_bucket(target=None, name=None, headers=None):
|
||||
"""
|
||||
Get a bucket that exists and is empty.
|
||||
|
||||
Always recreates a bucket from scratch. This is useful to also
|
||||
reset ACLs and such.
|
||||
"""
|
||||
if connection is None:
|
||||
connection = s3.main
|
||||
if target is None:
|
||||
target = targets.main
|
||||
connection = target.connection
|
||||
if name is None:
|
||||
name = get_new_bucket_name()
|
||||
# the only way for this to fail with a pre-existing bucket is if
|
||||
# someone raced us between setup nuke_prefixed_buckets and here;
|
||||
# ignore that as astronomically unlikely
|
||||
bucket = connection.create_bucket(name, location=location, headers=headers)
|
||||
bucket = connection.create_bucket(name, location=target.conf.api_name, headers=headers)
|
||||
return bucket
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from . import (
|
|||
get_new_bucket,
|
||||
get_new_bucket_name,
|
||||
s3,
|
||||
targets,
|
||||
config,
|
||||
get_prefix,
|
||||
)
|
||||
|
|
@ -752,7 +753,7 @@ def test_object_write_to_nonexist_bucket():
|
|||
def test_bucket_create_delete():
|
||||
name = '{prefix}foo'.format(prefix=get_prefix())
|
||||
print 'Trying bucket {name!r}'.format(name=name)
|
||||
bucket = get_new_bucket(s3.main, name)
|
||||
bucket = get_new_bucket(targets.main, name)
|
||||
# make sure it's actually there
|
||||
s3.main.get_bucket(bucket.name)
|
||||
bucket.delete()
|
||||
|
|
@ -2311,7 +2312,7 @@ def check_bad_bucket_name(name):
|
|||
Attempt to create a bucket with a specified name, and confirm
|
||||
that the request fails because of an invalid bucket name.
|
||||
"""
|
||||
e = assert_raises(boto.exception.S3ResponseError, get_new_bucket, s3.main, name)
|
||||
e = assert_raises(boto.exception.S3ResponseError, get_new_bucket, targets.main, name)
|
||||
eq(e.status, 400)
|
||||
eq(e.reason, 'Bad Request')
|
||||
eq(e.error_code, 'InvalidBucketName')
|
||||
|
|
@ -2337,7 +2338,7 @@ def test_bucket_create_naming_bad_starts_nonalpha():
|
|||
def test_bucket_create_naming_bad_short_empty():
|
||||
# bucket creates where name is empty look like PUTs to the parent
|
||||
# resource (with slash), hence their error response is different
|
||||
e = assert_raises(boto.exception.S3ResponseError, get_new_bucket, s3.main, '')
|
||||
e = assert_raises(boto.exception.S3ResponseError, get_new_bucket, targets.main, '')
|
||||
eq(e.status, 405)
|
||||
eq(e.reason, 'Method Not Allowed')
|
||||
eq(e.error_code, 'MethodNotAllowed')
|
||||
|
|
@ -2384,7 +2385,7 @@ def check_good_bucket_name(name, _prefix=None):
|
|||
# should be very rare
|
||||
if _prefix is None:
|
||||
_prefix = get_prefix()
|
||||
get_new_bucket(s3.main, '{prefix}{name}'.format(
|
||||
get_new_bucket(targets.main, '{prefix}{name}'.format(
|
||||
prefix=_prefix,
|
||||
name=name,
|
||||
))
|
||||
|
|
@ -2398,7 +2399,7 @@ def _test_bucket_create_naming_good_long(length):
|
|||
prefix = get_prefix()
|
||||
assert len(prefix) < 255
|
||||
num = length - len(prefix)
|
||||
get_new_bucket(s3.main, '{prefix}{name}'.format(
|
||||
get_new_bucket(targets.main, '{prefix}{name}'.format(
|
||||
prefix=prefix,
|
||||
name=num*'a',
|
||||
))
|
||||
|
|
@ -2473,7 +2474,7 @@ def test_bucket_list_long_name():
|
|||
prefix = get_prefix()
|
||||
length = 251
|
||||
num = length - len(prefix)
|
||||
bucket = get_new_bucket(s3.main, '{prefix}{name}'.format(
|
||||
bucket = get_new_bucket(targets.main, '{prefix}{name}'.format(
|
||||
prefix=prefix,
|
||||
name=num*'a',
|
||||
))
|
||||
|
|
@ -2571,9 +2572,9 @@ def test_bucket_create_naming_dns_dash_dot():
|
|||
@attr(operation='re-create')
|
||||
@attr(assertion='idempotent success')
|
||||
def test_bucket_create_exists():
|
||||
bucket = get_new_bucket(s3.main)
|
||||
bucket = get_new_bucket(targets.main)
|
||||
# REST idempotency means this should be a nop
|
||||
get_new_bucket(s3.main, bucket.name)
|
||||
get_new_bucket(targets.main, bucket.name)
|
||||
|
||||
|
||||
@attr(resource='bucket')
|
||||
|
|
@ -2584,7 +2585,7 @@ def test_bucket_create_exists_nonowner():
|
|||
# Names are shared across a global namespace. As such, no two
|
||||
# users can create a bucket with that same name.
|
||||
bucket = get_new_bucket()
|
||||
e = assert_raises(boto.exception.S3CreateError, get_new_bucket, s3.alt, bucket.name)
|
||||
e = assert_raises(boto.exception.S3CreateError, get_new_bucket, targets.alt, bucket.name)
|
||||
eq(e.status, 409)
|
||||
eq(e.reason, 'Conflict')
|
||||
eq(e.error_code, 'BucketAlreadyExists')
|
||||
|
|
@ -2907,7 +2908,7 @@ def test_object_acl_canned_authenticatedread():
|
|||
@attr(operation='acl bucket-owner-read')
|
||||
@attr(assertion='read back expected values')
|
||||
def test_object_acl_canned_bucketownerread():
|
||||
bucket = get_new_bucket(s3.main)
|
||||
bucket = get_new_bucket(targets.main)
|
||||
bucket.set_acl('public-read-write')
|
||||
|
||||
key = s3.alt.get_bucket(bucket.name).new_key('foo')
|
||||
|
|
@ -2951,7 +2952,7 @@ def test_object_acl_canned_bucketownerread():
|
|||
@attr(operation='acl bucket-owner-read')
|
||||
@attr(assertion='read back expected values')
|
||||
def test_object_acl_canned_bucketownerfullcontrol():
|
||||
bucket = get_new_bucket(s3.main)
|
||||
bucket = get_new_bucket(targets.main)
|
||||
bucket.set_acl('public-read-write')
|
||||
|
||||
key = s3.alt.get_bucket(bucket.name).new_key('foo')
|
||||
|
|
@ -3460,7 +3461,7 @@ def test_object_header_acl_grants():
|
|||
@attr('fails_on_dho')
|
||||
def test_bucket_header_acl_grants():
|
||||
headers = _get_acl_header()
|
||||
bucket = get_new_bucket(s3.main, get_prefix(), headers)
|
||||
bucket = get_new_bucket(targets.main, get_prefix(), headers)
|
||||
|
||||
policy = bucket.get_acl()
|
||||
check_grants(
|
||||
|
|
@ -3595,7 +3596,7 @@ def test_bucket_acl_revoke_all():
|
|||
@attr('fails_on_rgw')
|
||||
def test_logging_toggle():
|
||||
bucket = get_new_bucket()
|
||||
log_bucket = get_new_bucket(s3.main, bucket.name + '-log')
|
||||
log_bucket = get_new_bucket(targets.main, bucket.name + '-log')
|
||||
log_bucket.set_as_logging_target()
|
||||
bucket.enable_logging(target_bucket=log_bucket, target_prefix=bucket.name)
|
||||
bucket.disable_logging()
|
||||
|
|
@ -3907,7 +3908,7 @@ def test_bucket_recreate_not_overriding():
|
|||
names = [e.name for e in list(li)]
|
||||
eq(names, key_names)
|
||||
|
||||
bucket2 = get_new_bucket(s3.main, bucket.name)
|
||||
bucket2 = get_new_bucket(targets.main, bucket.name)
|
||||
|
||||
li = bucket.list()
|
||||
|
||||
|
|
@ -4000,7 +4001,7 @@ def test_object_copy_diff_bucket():
|
|||
@attr(operation='copy from an inaccessible bucket')
|
||||
@attr(assertion='fails w/AttributeError')
|
||||
def test_object_copy_not_owned_bucket():
|
||||
buckets = [get_new_bucket(), get_new_bucket(s3.alt)]
|
||||
buckets = [get_new_bucket(), get_new_bucket(targets.alt)]
|
||||
print repr(buckets[1])
|
||||
key = buckets[0].new_key('foo123bar')
|
||||
key.set_contents_from_string('foo')
|
||||
|
|
|
|||
Loading…
Reference in a new issue