TrueCloudLab/s3-tests
7
0
Fork 1
mirror of https://github.com/ceph/s3-tests.git synced 2024-11-21 23:29:47 +00:00
Code Issues 1 Projects Releases Packages Wiki Activity

readwrite.py: adding parameters

Browse source 
Add an optional parameter to trigger deterministic
file name creation (for separate write/read tasks).
Also, change the behavior when zero writers
are specified to actually generate no data.

Signed-off-by: Joe Buck <jbbuck@gmail.com>
Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
This commit is contained in:
Joe Buck 2013-08-02 16:49:20 -07:00
parent f20c6e250e
commit 34a06133eb
1 changed files with 50 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
s3tests/readwrite.py
View file

@ -161,13 +161,23 @@ def main():
bucket_name = common.choose_bucket_prefix(config.readwrite.bucket, max_len=30) bucket_name = common.choose_bucket_prefix(config.readwrite.bucket, max_len=30)
bucket = conn.create_bucket(bucket_name) bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name) print "Created bucket: {name}".format(name=bucket.name)
file_names = realistic.names(
mean=15, # check flag for deterministic file name creation
stddev=4, if not config.readwrite.get('deterministic_file_names'):
seed=seeds['names'], print 'Creating random file names'
) file_names = realistic.names(
file_names = itertools.islice(file_names, config.readwrite.files.num) mean=15,
file_names = list(file_names) stddev=4,
seed=seeds['names'],
)
file_names = itertools.islice(file_names, config.readwrite.files.num)
file_names = list(file_names)
else:
print 'Creating file names that are deterministic'
file_names = []
for x in xrange(config.readwrite.files.num):
file_names.append('test_file_{num}'.format(num=x))
files = realistic.files2( files = realistic.files2(
mean=1024 * config.readwrite.files.size, mean=1024 * config.readwrite.files.size,
stddev=1024 * config.readwrite.files.stddev, stddev=1024 * config.readwrite.files.stddev,
@ -175,18 +185,20 @@ def main():
) )
q = gevent.queue.Queue() q = gevent.queue.Queue()
# warmup - get initial set of files uploaded
print "Uploading initial set of {num} files".format(num=config.readwrite.files.num) # warmup - get initial set of files uploaded if there are any writers specified
warmup_pool = gevent.pool.Pool(size=100) if config.readwrite.writers > 0:
for file_name in file_names: print "Uploading initial set of {num} files".format(num=config.readwrite.files.num)
fp = next(files) warmup_pool = gevent.pool.Pool(size=100)
warmup_pool.spawn_link_exception( for file_name in file_names:
write_file, fp = next(files)
bucket=bucket, warmup_pool.spawn_link_exception(
file_name=file_name, write_file,
fp=fp, bucket=bucket,
) file_name=file_name,
warmup_pool.join() fp=fp,
)
warmup_pool.join()
# main work # main work
print "Starting main worker loop." print "Starting main worker loop."
@ -194,17 +206,25 @@ def main():
print "Spawning {w} writers and {r} readers...".format(w=config.readwrite.writers, r=config.readwrite.readers) print "Spawning {w} writers and {r} readers...".format(w=config.readwrite.writers, r=config.readwrite.readers)
group = gevent.pool.Group() group = gevent.pool.Group()
rand_writer = random.Random(seeds['writer']) rand_writer = random.Random(seeds['writer'])
for x in xrange(config.readwrite.writers):
this_rand = random.Random(rand_writer.randrange(2**32)) # Don't create random files if deterministic_files_names is set and true
group.spawn_link_exception( if not config.readwrite.get('deterministic_file_names'):
writer, for x in xrange(config.readwrite.writers):
bucket=bucket, this_rand = random.Random(rand_writer.randrange(2**32))
worker_id=x, group.spawn_link_exception(
file_names=file_names, writer,
files=files, bucket=bucket,
queue=q, worker_id=x,
rand=this_rand, file_names=file_names,
) files=files,
queue=q,
rand=this_rand,
)
# Since the loop generating readers already uses config.readwrite.readers
# and the file names are already generated (randomly or deterministically),
# this loop needs no additional qualifiers. If zero readers are specified,
# it will behave as expected (no data is read)
rand_reader = random.Random(seeds['reader']) rand_reader = random.Random(seeds['reader'])
for x in xrange(config.readwrite.readers): for x in xrange(config.readwrite.readers):
this_rand = random.Random(rand_reader.randrange(2**32)) this_rand = random.Random(rand_reader.randrange(2**32))