dkirillov/s3-tests
1
0
Fork 0
forked from TrueCloudLab/s3-tests
Code Pull requests Activity

always seek back to 0, remove chunk usage

Browse source 
Signed-off-by: Alfredo Deza <alfredo@deza.pe>
This commit is contained in:
Alfredo Deza 2014-12-12 13:52:43 -05:00
parent 3390afc4a5
commit b4794edbb3
1 changed files with 5 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
s3tests/readwrite.py
View file

@ -21,7 +21,7 @@ def reader(bucket, worker_id, file_names, queue, rand):
objname = rand.choice(file_names) objname = rand.choice(file_names)
key = bucket.new_key(objname) key = bucket.new_key(objname)
fp = realistic.FileVerifier() fp = realistic.FileValidator()
result = dict( result = dict(
type='r', type='r',
bucket=bucket.name, bucket=bucket.name,
@ -31,7 +31,7 @@ def reader(bucket, worker_id, file_names, queue, rand):
start = time.time() start = time.time()
try: try:
key.get_contents_to_file(fp) key.get_contents_to_file(fp._file)
except gevent.GreenletExit: except gevent.GreenletExit:
raise raise
except Exception as e: except Exception as e:
@ -50,7 +50,7 @@ def reader(bucket, worker_id, file_names, queue, rand):
end = time.time() end = time.time()
if not fp.valid(): if not fp.valid():
m='md5sum check failed start={s} ({se}) end={e} size={sz} obj={o}'.format(s=time.ctime(start), se=start, e=end, sz=fp.size, o=objname) m='md5sum check failed start={s} ({se}) end={e} size={sz} obj={o}'.format(s=time.ctime(start), se=start, e=end, sz=fp._file.tell(), o=objname)
result.update( result.update(
error=dict( error=dict(
msg=m, msg=m,
@ -63,13 +63,13 @@ def reader(bucket, worker_id, file_names, queue, rand):
result.update( result.update(
start=start, start=start,
duration=int(round(elapsed * NANOSECOND)), duration=int(round(elapsed * NANOSECOND)),
chunks=fp.chunks,
) )
queue.put(result) queue.put(result)
def writer(bucket, worker_id, file_names, files, queue, rand): def writer(bucket, worker_id, file_names, files, queue, rand):
while True: while True:
fp = next(files) fp = next(files)
fp.seek(0)
objname = rand.choice(file_names) objname = rand.choice(file_names)
key = bucket.new_key(objname) key = bucket.new_key(objname)
@ -104,7 +104,6 @@ def writer(bucket, worker_id, file_names, files, queue, rand):
result.update( result.update(
start=start, start=start,
duration=int(round(elapsed * NANOSECOND)), duration=int(round(elapsed * NANOSECOND)),
chunks=fp.last_chunks,
) )
queue.put(result) queue.put(result)
@ -189,7 +188,7 @@ def main():
) )
q = gevent.queue.Queue() q = gevent.queue.Queue()
# warmup - get initial set of files uploaded if there are any writers specified # warmup - get initial set of files uploaded if there are any writers specified
if config.readwrite.writers > 0: if config.readwrite.writers > 0:
print "Uploading initial set of {num} files".format(num=config.readwrite.files.num) print "Uploading initial set of {num} files".format(num=config.readwrite.files.num)