Compare commits

...

2 commits

Author SHA1 Message Date
Yuan Zhou
d29b2d92a9 gevent: bump to >=1.0
There are some DNS resolving issue on 0.13.6.
Bumping to >=1.0 fixes the issues for me.

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
2016-04-06 09:42:46 +02:00
Abhishek Lekshmanan
f7ef61a128 connection: add a debug option to debug boto calls
Basically setting debug level via conf file and setting a stream logger
prefixed with 'boto' for more debugging

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
2016-03-31 14:07:48 +02:00
5 changed files with 11 additions and 8 deletions

View file

@ -3,7 +3,7 @@ nose >=1.0.0
boto >=2.6.0 boto >=2.6.0
bunch >=1.0.0 bunch >=1.0.0
# 0.14 switches to libev, that means bootstrap needs to change too # 0.14 switches to libev, that means bootstrap needs to change too
gevent ==0.13.6 gevent >=1.0
isodate >=0.4.4 isodate >=0.4.4
requests ==0.14.0 requests ==0.14.0
pytz >=2011k pytz >=2011k

View file

@ -108,6 +108,9 @@ def connect(conf):
raise RuntimeError( raise RuntimeError(
'calling_format unknown: %r' % raw_calling_format 'calling_format unknown: %r' % raw_calling_format
) )
if conf.has_key('debug'):
kwargs['debug']=conf['debug']
boto.set_stream_logger('boto')
# TODO test vhost calling format # TODO test vhost calling format
conn = boto.s3.connection.S3Connection(**kwargs) conn = boto.s3.connection.S3Connection(**kwargs)
return conn return conn

View file

@ -195,7 +195,7 @@ def main():
warmup_pool = gevent.pool.Pool(size=100) warmup_pool = gevent.pool.Pool(size=100)
for file_name in file_names: for file_name in file_names:
fp = next(files) fp = next(files)
warmup_pool.spawn_link_exception( warmup_pool.spawn(
write_file, write_file,
bucket=bucket, bucket=bucket,
file_name=file_name, file_name=file_name,
@ -214,7 +214,7 @@ def main():
if not config.readwrite.get('deterministic_file_names'): if not config.readwrite.get('deterministic_file_names'):
for x in xrange(config.readwrite.writers): for x in xrange(config.readwrite.writers):
this_rand = random.Random(rand_writer.randrange(2**32)) this_rand = random.Random(rand_writer.randrange(2**32))
group.spawn_link_exception( group.spawn(
writer, writer,
bucket=bucket, bucket=bucket,
worker_id=x, worker_id=x,
@ -231,7 +231,7 @@ def main():
rand_reader = random.Random(seeds['reader']) rand_reader = random.Random(seeds['reader'])
for x in xrange(config.readwrite.readers): for x in xrange(config.readwrite.readers):
this_rand = random.Random(rand_reader.randrange(2**32)) this_rand = random.Random(rand_reader.randrange(2**32))
group.spawn_link_exception( group.spawn(
reader, reader,
bucket=bucket, bucket=bucket,
worker_id=x, worker_id=x,

View file

@ -161,7 +161,7 @@ def main():
) )
q = gevent.queue.Queue() q = gevent.queue.Queue()
logger_g = gevent.spawn_link_exception(yaml.safe_dump_all, q, stream=real_stdout) logger_g = gevent.spawn(yaml.safe_dump_all, q, stream=real_stdout)
print "Writing {num} objects with {w} workers...".format( print "Writing {num} objects with {w} workers...".format(
num=config.roundtrip.files.num, num=config.roundtrip.files.num,
@ -171,7 +171,7 @@ def main():
start = time.time() start = time.time()
for objname in objnames: for objname in objnames:
fp = next(files) fp = next(files)
pool.spawn_link_exception( pool.spawn(
writer, writer,
bucket=bucket, bucket=bucket,
objname=objname, objname=objname,
@ -195,7 +195,7 @@ def main():
pool = gevent.pool.Pool(size=config.roundtrip.readers) pool = gevent.pool.Pool(size=config.roundtrip.readers)
start = time.time() start = time.time()
for objname in objnames: for objname in objnames:
pool.spawn_link_exception( pool.spawn(
reader, reader,
bucket=bucket, bucket=bucket,
objname=objname, objname=objname,

View file

@ -16,7 +16,7 @@ setup(
'boto >=2.0b4', 'boto >=2.0b4',
'PyYAML', 'PyYAML',
'bunch >=1.0.0', 'bunch >=1.0.0',
'gevent ==0.13.6', 'gevent >=1.0',
'isodate >=0.4.4', 'isodate >=0.4.4',
], ],