forked from TrueCloudLab/s3-tests
be9935ba1a
Add fails_on_rgw to tests not passing. Some tests from the master branch do not pass on the rgw yet. Others waiting on rgw tracker issues to be resolved. Signed-off-by: Ali Maredia <amaredia@redhat.com>
117 lines
3.7 KiB
Python
117 lines
3.7 KiB
Python
from boto.s3.key import Key
|
|
from optparse import OptionParser
|
|
from . import realistic
|
|
import traceback
|
|
import random
|
|
from . import common
|
|
import sys
|
|
|
|
|
|
def parse_opts():
|
|
parser = OptionParser()
|
|
parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
|
|
parser.add_option('-b', '--bucket', dest='bucket', help='push objects to BUCKET', metavar='BUCKET')
|
|
parser.add_option('--seed', dest='seed', help='optional seed for the random number generator')
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def get_random_files(quantity, mean, stddev, seed):
|
|
"""Create file-like objects with pseudorandom contents.
|
|
IN:
|
|
number of files to create
|
|
mean file size in bytes
|
|
standard deviation from mean file size
|
|
seed for PRNG
|
|
OUT:
|
|
list of file handles
|
|
"""
|
|
file_generator = realistic.files(mean, stddev, seed)
|
|
return [next(file_generator) for _ in range(quantity)]
|
|
|
|
|
|
def upload_objects(bucket, files, seed):
|
|
"""Upload a bunch of files to an S3 bucket
|
|
IN:
|
|
boto S3 bucket object
|
|
list of file handles to upload
|
|
seed for PRNG
|
|
OUT:
|
|
list of boto S3 key objects
|
|
"""
|
|
keys = []
|
|
name_generator = realistic.names(15, 4, seed=seed)
|
|
|
|
for fp in files:
|
|
print('sending file with size %dB' % fp.size, file=sys.stderr)
|
|
key = Key(bucket)
|
|
key.key = next(name_generator)
|
|
key.set_contents_from_file(fp, rewind=True)
|
|
key.set_acl('public-read')
|
|
keys.append(key)
|
|
|
|
return keys
|
|
|
|
|
|
def _main():
|
|
'''To run the static content load test, make sure you've bootstrapped your
|
|
test environment and set up your config.yaml file, then run the following:
|
|
S3TEST_CONF=config.yaml virtualenv/bin/s3tests-generate-objects.py --seed 1234
|
|
|
|
This creates a bucket with your S3 credentials (from config.yaml) and
|
|
fills it with garbage objects as described in the
|
|
file_generation.groups section of config.yaml. It writes a list of
|
|
URLS to those objects to the file listed in file_generation.url_file
|
|
in config.yaml.
|
|
|
|
Once you have objcts in your bucket, run the siege benchmarking program:
|
|
siege --rc ./siege.conf -r 5
|
|
|
|
This tells siege to read the ./siege.conf config file which tells it to
|
|
use the urls in ./urls.txt and log to ./siege.log. It hits each url in
|
|
urls.txt 5 times (-r flag).
|
|
|
|
Results are printed to the terminal and written in CSV format to
|
|
./siege.log
|
|
'''
|
|
(options, args) = parse_opts()
|
|
|
|
#SETUP
|
|
random.seed(options.seed if options.seed else None)
|
|
conn = common.s3.main
|
|
|
|
if options.outfile:
|
|
OUTFILE = open(options.outfile, 'w')
|
|
elif common.config.file_generation.url_file:
|
|
OUTFILE = open(common.config.file_generation.url_file, 'w')
|
|
else:
|
|
OUTFILE = sys.stdout
|
|
|
|
if options.bucket:
|
|
bucket = conn.create_bucket(options.bucket)
|
|
else:
|
|
bucket = common.get_new_bucket()
|
|
|
|
bucket.set_acl('public-read')
|
|
keys = []
|
|
print('bucket: %s' % bucket.name, file=OUTFILE)
|
|
print('setup complete, generating files', file=sys.stderr)
|
|
for profile in common.config.file_generation.groups:
|
|
seed = random.random()
|
|
files = get_random_files(profile[0], profile[1], profile[2], seed)
|
|
keys += upload_objects(bucket, files, seed)
|
|
|
|
print('finished sending files. generating urls', file=sys.stderr)
|
|
for key in keys:
|
|
print(key.generate_url(0, query_auth=False), file=OUTFILE)
|
|
|
|
print('done', file=sys.stderr)
|
|
|
|
|
|
def main():
|
|
common.setup()
|
|
try:
|
|
_main()
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
common.teardown()
|