TrueCloudLab/s3-tests
7
0
Fork 1
mirror of https://github.com/ceph/s3-tests.git synced 2024-11-22 09:29:43 +00:00
Code Issues 1 Projects Releases Packages Wiki Activity

readwrite.py: adding parameters

Browse source 
Add an optional parameter to trigger deterministic
file name creation (for separate write/read tasks).
Also, change the behavior when zero writers
are specified to actually generate no data.

Signed-off-by: Joe Buck <jbbuck@gmail.com>
Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
This commit is contained in:
Joe Buck 2013-08-02 16:49:20 -07:00
parent f20c6e250e
commit 34a06133eb
1 changed files with 50 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
s3tests/readwrite.py
View file

@ -161,6 +161,10 @@ def main():
bucket_name = common.choose_bucket_prefix(config.readwrite.bucket, max_len=30) bucket_name = common.choose_bucket_prefix(config.readwrite.bucket, max_len=30)
bucket = conn.create_bucket(bucket_name) bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name) print "Created bucket: {name}".format(name=bucket.name)
# check flag for deterministic file name creation
if not config.readwrite.get('deterministic_file_names'):
print 'Creating random file names'
file_names = realistic.names( file_names = realistic.names(
mean=15, mean=15,
stddev=4, stddev=4,
@ -168,6 +172,12 @@ def main():
) )
file_names = itertools.islice(file_names, config.readwrite.files.num) file_names = itertools.islice(file_names, config.readwrite.files.num)
file_names = list(file_names) file_names = list(file_names)
else:
print 'Creating file names that are deterministic'
file_names = []
for x in xrange(config.readwrite.files.num):
file_names.append('test_file_{num}'.format(num=x))
files = realistic.files2( files = realistic.files2(
mean=1024 * config.readwrite.files.size, mean=1024 * config.readwrite.files.size,
stddev=1024 * config.readwrite.files.stddev, stddev=1024 * config.readwrite.files.stddev,
@ -175,7 +185,9 @@ def main():
) )
q = gevent.queue.Queue() q = gevent.queue.Queue()
# warmup - get initial set of files uploaded
# warmup - get initial set of files uploaded if there are any writers specified
if config.readwrite.writers > 0:
print "Uploading initial set of {num} files".format(num=config.readwrite.files.num) print "Uploading initial set of {num} files".format(num=config.readwrite.files.num)
warmup_pool = gevent.pool.Pool(size=100) warmup_pool = gevent.pool.Pool(size=100)
for file_name in file_names: for file_name in file_names:
@ -194,6 +206,9 @@ def main():
print "Spawning {w} writers and {r} readers...".format(w=config.readwrite.writers, r=config.readwrite.readers) print "Spawning {w} writers and {r} readers...".format(w=config.readwrite.writers, r=config.readwrite.readers)
group = gevent.pool.Group() group = gevent.pool.Group()
rand_writer = random.Random(seeds['writer']) rand_writer = random.Random(seeds['writer'])
# Don't create random files if deterministic_files_names is set and true
if not config.readwrite.get('deterministic_file_names'):
for x in xrange(config.readwrite.writers): for x in xrange(config.readwrite.writers):
this_rand = random.Random(rand_writer.randrange(2**32)) this_rand = random.Random(rand_writer.randrange(2**32))
group.spawn_link_exception( group.spawn_link_exception(
@ -205,6 +220,11 @@ def main():
queue=q, queue=q,
rand=this_rand, rand=this_rand,
) )
# Since the loop generating readers already uses config.readwrite.readers
# and the file names are already generated (randomly or deterministically),
# this loop needs no additional qualifiers. If zero readers are specified,
# it will behave as expected (no data is read)
rand_reader = random.Random(seeds['reader']) rand_reader = random.Random(seeds['reader'])
for x in xrange(config.readwrite.readers): for x in xrange(config.readwrite.readers):
this_rand = random.Random(rand_reader.randrange(2**32)) this_rand = random.Random(rand_reader.randrange(2**32))