s3-tests/s3tests/generate_objects.py

from boto.s3.key import Key
from optparse import OptionParser
from . import realistic
import traceback
import random
from . import common
import sys


def parse_opts():
    parser = OptionParser()
    parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
    parser.add_option('-b', '--bucket', dest='bucket', help='push objects to BUCKET', metavar='BUCKET')
    parser.add_option('--seed', dest='seed', help='optional seed for the random number generator')

    return parser.parse_args()


def get_random_files(quantity, mean, stddev, seed):
    """Create file-like objects with pseudorandom contents.
       IN:
           number of files to create
           mean file size in bytes
           standard deviation from mean file size
           seed for PRNG
       OUT:
           list of file handles
    """
    file_generator = realistic.files(mean, stddev, seed)
    return [next(file_generator) for _ in range(quantity)]


def upload_objects(bucket, files, seed):
    """Upload a bunch of files to an S3 bucket
       IN:
         boto S3 bucket object
         list of file handles to upload
         seed for PRNG
       OUT:
         list of boto S3 key objects
    """
    keys = []
    name_generator = realistic.names(15, 4, seed=seed)

    for fp in files:
        print('sending file with size %dB' % fp.size, file=sys.stderr)
        key = Key(bucket)
        key.key = next(name_generator)
        key.set_contents_from_file(fp, rewind=True)
        key.set_acl('public-read')
        keys.append(key)

    return keys


def _main():
    '''To run the static content load test, make sure you've bootstrapped your
       test environment and set up your config.yaml file, then run the following:
          S3TEST_CONF=config.yaml virtualenv/bin/s3tests-generate-objects.py --seed 1234

        This creates a bucket with your S3 credentials (from config.yaml) and
        fills it with garbage objects as described in the
        file_generation.groups section of config.yaml.  It writes a list of
        URLS to those objects to the file listed in file_generation.url_file
        in config.yaml.

        Once you have objcts in your bucket, run the siege benchmarking program:
            siege --rc ./siege.conf -r 5

        This tells siege to read the ./siege.conf config file which tells it to
        use the urls in ./urls.txt and log to ./siege.log. It hits each url in
        urls.txt 5 times (-r flag).

        Results are printed to the terminal and written in CSV format to
        ./siege.log
    '''
    (options, args) = parse_opts()

    #SETUP
    random.seed(options.seed if options.seed else None)
    conn = common.s3.main

    if options.outfile:
        OUTFILE = open(options.outfile, 'w')
    elif common.config.file_generation.url_file:
        OUTFILE = open(common.config.file_generation.url_file, 'w')
    else:
        OUTFILE = sys.stdout

    if options.bucket:
        bucket = conn.create_bucket(options.bucket)
    else:
        bucket = common.get_new_bucket()

    bucket.set_acl('public-read')
    keys = []
    print('bucket: %s' % bucket.name, file=OUTFILE)
    print('setup complete, generating files', file=sys.stderr)
    for profile in common.config.file_generation.groups:
        seed = random.random()
        files = get_random_files(profile[0], profile[1], profile[2], seed)
        keys += upload_objects(bucket, files, seed)

    print('finished sending files. generating urls', file=sys.stderr)
    for key in keys:
        print(key.generate_url(0, query_auth=False), file=OUTFILE)

    print('done', file=sys.stderr)


def main():
    common.setup()
    try:
        _main()
    except Exception as e:
        traceback.print_exc()
        common.teardown()