Merge remote branch 'github/fuzz_headers'

request_decision_graph.yml

@@ -0,0 +1,491 @@
+start:
+    set:
+        garbage:
+            - '{random 10-3000 printable}'
+            - '{random 10-1000 binary}'
+        garbage_no_whitespace:
+            - '{random 10-3000 printable_no_whitespace}'
+            - '{random 10-1000 binary_no_whitespace}'
+        acl_header:
+            - 'private'
+            - 'public-read'
+            - 'public-read-write'
+            - 'authenticated-read'
+            - 'bucket-owner-read'
+            - 'bucket-owner-full-control'
+            - '{random 3000 letters}'
+            - '{random 100-1000 binary_no_whitespace}'
+    choices:
+        - bucket
+        - object
+
+bucket:
+    set:
+        urlpath: '/{bucket}'
+    choices:
+        - 13 bucket_get
+        - 8 bucket_put
+        - 5 bucket_delete
+        - bucket_garbage_method
+
+bucket_garbage_method:
+    set:
+        method:
+            - '{random 1-100 printable}'
+            - '{random 10-100 binary}'
+        bucket:
+            - '{bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage_no_whitespace}'
+    choices:
+        - bucket_get_simple
+        - bucket_get_filtered
+        - bucket_get_uploads
+        - bucket_put_create
+        - bucket_put_versioning
+        - bucket_put_simple
+
+bucket_delete:
+    set:
+        method: DELETE
+        bucket:
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage_no_whitespace}'
+        query:
+            - null
+            - policy
+            - website
+            - '2 {garbage_no_whitespace}'
+    choices: []
+
+bucket_get:
+    set:
+        method: GET
+        bucket:
+            - '{bucket_readable}'
+            - '{bucket_not_readable}'
+            - '2 {garbage_no_whitespace}'
+    choices:
+        - 11 bucket_get_simple
+        - bucket_get_filtered
+        - bucket_get_uploads
+
+bucket_get_simple:
+    set:
+        query:
+            - acl
+            - policy
+            - location
+            - logging
+            - notification
+            - versions
+            - requestPayment
+            - versioning
+            - website
+            - '2 {garbage_no_whitespace}'
+    choices: []
+
+bucket_get_uploads:
+    set:
+        delimiter:
+            - null
+            - '3 delimiter={garbage_no_whitespace}'
+        prefix:
+            - null
+            - '3 prefix={garbage_no_whitespace}'
+        key_marker:
+            - null
+            - 'key-marker={object_readable}'
+            - 'key-marker={object_not_readable}'
+            - 'key-marker={invalid_key}'
+            - 'key-marker={random 100-1000 printable_no_whitespace}'
+        max_uploads:
+            - null
+            - 'max-uploads={random 1-5 binary_no_whitespace}'
+            - 'max-uploads={random 1-1000 digits}'
+        upload_id_marker:
+            - null
+            - '3 upload-id-marker={random 0-1000 printable_no_whitespace}'
+        query:
+            - 'uploads'
+            - 'uploads&{delimiter}&{prefix}'
+            - 'uploads&{max_uploads}&{key_marker}&{upload_id_marker}'
+            - '2 {garbage_no_whitespace}'
+    choices: []
+
+bucket_get_filtered:
+    set:
+        delimiter:
+            - 'delimiter={garbage_no_whitespace}'
+        prefix:
+            - 'prefix={garbage_no_whitespace}'
+        marker:
+            - 'marker={object_readable}'
+            - 'marker={object_not_readable}'
+            - 'marker={invalid_key}'
+            - 'marker={random 100-1000 printable_no_whitespace}'
+        max_keys:
+            - 'max-keys={random 1-5 binary_no_whitespace}'
+            - 'max-keys={random 1-1000 digits}'
+        query:
+            - null
+            - '{delimiter}&{prefix}'
+            - '{max-keys}&{marker}'
+            - '2 {garbage_no_whitespace}'
+    choices: []
+
+bucket_put:
+    set:
+        bucket:
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage_no_whitespace}'
+        method: PUT
+    choices:
+        - bucket_put_simple
+        - bucket_put_create
+        - bucket_put_versioning
+
+bucket_put_create:
+    set:
+        body:
+            - '2 {garbage}'
+            - '<CreateBucketConfiguration><LocationConstraint>{random 2-10 binary}</LocationConstraint></CreateBucketConfiguration>'
+    headers:
+        - ['0-5', 'x-amz-acl', '{acl_header}']
+    choices: []
+
+bucket_put_versioning:
+    set:
+        body:
+            - '{garbage}'
+            - '4 <VersioningConfiguration>{versioning_status}{mfa_delete_body}</VersioningConfiguration>'
+        mfa_delete_body:
+            - null
+            - '<Status>{random 2-10 binary}</Status>'
+            - '<Status>{random 2000-3000 printable}</Status>'
+        versioning_status:
+            - null
+            - '<MfaDelete>{random 2-10 binary}</MfaDelete>'
+            - '<MfaDelete>{random 2000-3000 printable}</MfaDelete>'
+        mfa_header:
+            - '{random 10-1000 printable_no_whitespace} {random 10-1000 printable_no_whitespace}'
+    headers:
+        - ['0-1', 'x-amz-mfa', '{mfa_header}']
+    choices: []
+
+bucket_put_simple:
+    set:
+        body:
+            - '{acl_body}'
+            - '{policy_body}'
+            - '{logging_body}'
+            - '{notification_body}'
+            - '{request_payment_body}'
+            - '{website_body}'
+        acl_body:
+            - null
+            - '<AccessControlPolicy>{owner}{acl}</AccessControlPolicy>'
+        owner:
+            - null
+            - '7 <Owner>{id}{display_name}</Owner>'
+        id:
+            - null
+            - '<ID>{random 10-200 binary}</ID>'
+            - '<ID>{random 1000-3000 printable}</ID>'
+        display_name:
+            - null
+            - '2 <DisplayName>{random 10-200 binary}</DisplayName>'
+            - '2 <DisplayName>{random 1000-3000 printable}</DisplayName>'
+            - '2 <DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
+        acl:
+            - null
+            - '10 <AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
+        grantee:
+            - null
+            - '7 <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">{id}{display_name}</Grantee>'
+        permission:
+            - null
+            - '7 <Permission>{permission_value}</Permission>'
+        permission_value:
+            - '2 {garbage}'
+            - FULL_CONTROL
+            - WRITE
+            - WRITE_ACP
+            - READ
+            - READ_ACP
+        policy_body:
+            - null
+            - '2 {garbage}'
+        logging_body:
+            - null
+            - '<BucketLoggingStatus xmlns="http://doc.s3.amazonaws.com/2006-03-01" />'
+            - '<BucketLoggingStatus xmlns="http://doc.s3.amazonaws.com/2006-03-01"><LoggingEnabled>{bucket}{target_prefix}{target_grants}</LoggingEnabled></BucketLoggingStatus>'
+        target_prefix:
+            - null
+            - '<TargetPrefix>{random 10-1000 printable}</TargetPrefix>'
+            - '<TargetPrefix>{random 10-1000 binary}</TargetPrefix>'
+        target_grants:
+            - null
+            - '10 <TargetGrants><Grant>{grantee}{permission}</Grant></TargetGrants>'
+        notification_body:
+            - null
+            - '<NotificationConfiguration />'
+            - '2 <NotificationConfiguration><TopicConfiguration>{topic}{event}</TopicConfiguration></NotificationConfiguration>'
+        topic:
+            - null
+            - '2 <Topic>{garbage}</Topic>'
+        event:
+            - null
+            - '<Event>s3:ReducedRedundancyLostObject</Event>'
+            - '2 <Event>{garbage}</Event>'
+        request_payment_body:
+            - null
+            - '<RequestPaymentConfiguration xlmns="http://s3.amazonaws.com/doc/2006-03-01/"><Payer>{payer}</Payer></RequestPaymentConfiguration>'
+        payer:
+            - Requester
+            - BucketOwner
+            - '2 {garbage}'
+        website_body:
+            - null
+            - '<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IndexDocument><Suffix>{suffix}</Suffix><IndexDocument>{error_doc}<WebsiteConfiguration/>'
+        suffix:
+            - null
+            - '2 {garbage}'
+            - '{random 2-10 printable}.html'
+        error_doc:
+            - null
+            - '<ErrorDocument><Key>{suffix}</Key></ErrorDocument>'
+    choices: []
+
+object:
+    set:
+        urlpath: '/{bucket}/{object}'
+
+        range_header:
+            - null
+            - 'bytes={random 1-2 digits}-{random 1-4 digits}'
+            - 'bytes={random 1-1000 binary_no_whitespace}'
+        if_modified_since_header:
+            - null
+            - '2 {garbage_no_whitespace}'
+        if_match_header:
+            - null
+            - '2 {garbage_no_whitespace}'
+        if_none_match_header:
+            - null
+            - '2 {garbage_no_whitespace}'
+    choices:
+        - object_delete
+        - object_get
+        - object_put
+        - object_head
+        - object_garbage_method
+
+object_garbage_method:
+    set:
+        method:
+            - '{random 1-100 printable}'
+            - '{random 10-100 binary}'
+        bucket:
+            - '{bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage_no_whitespace}'
+        object:
+            - '{object_readable}'
+            - '{object_not_readable}'
+            - '{object_writable}'
+            - '{object_not_writable}'
+            - '2 {garbage_no_whitespace}'
+    choices:
+        - object_get_query
+        - object_get_head_simple
+
+object_delete:
+    set:
+        method: DELETE
+        bucket:
+            - '5 {bucket_writable}'
+            - '{bucket_not_writable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_writable}'
+            - '{object_not_writable}'
+            - '2 {garbage_no_whitespace}'
+    choices: []
+
+object_get:
+    set:
+        method: GET
+        bucket:
+            - '5 {bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_readable}'
+            - '{object_not_readable}'
+            - '{garbage_no_whitespace}'
+    choices:
+        - 5 object_get_head_simple
+        - 2 object_get_query
+
+object_get_query:
+    set:
+        query: 
+            - 'torrent'
+            - 'acl'
+    choices: []
+
+object_get_head_simple:
+    set: {}
+    headers:
+        - ['0-1', 'range', '{range_header}']
+        - ['0-1', 'if-modified-since', '{if_modified_since_header}']
+        - ['0-1', 'if-unmodified-since', '{if_modified_since_header}']
+        - ['0-1', 'if-match', '{if_match_header}']
+        - ['0-1', 'if-none-match', '{if_none_match_header}']
+    choices: []
+
+object_head:
+    set:
+        method: HEAD
+        bucket:
+            - '5 {bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_readable}'
+            - '{object_not_readable}'
+            - '{garbage_no_whitespace}'
+    choices:
+        - object_get_head_simple
+
+object_put:
+    set:
+        method: PUT
+        bucket:
+            - '5 {bucket_writable}'
+            - '{bucket_not_writable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_writable}'
+            - '{object_not_writable}'
+            - '{garbage_no_whitespace}'
+        cache_control:
+            - null
+            - '{garbage_no_whitespace}'
+            - 'no-cache'
+        content_disposition:
+            - null
+            - '{garbage_no_whitespace}'
+        content_encoding:
+            - null
+            - '{garbage_no_whitespace}'
+        content_length:
+            - '{random 1-20 digits}'
+            - '{garbage_no_whitespace}'
+        content_md5:
+            - null
+            - '{garbage_no_whitespace}'
+        content_type:
+            - null
+            - 'binary/octet-stream'
+            - '{garbage_no_whitespace}'
+        expect:
+            - null
+            - '100-continue'
+            - '{garbage_no_whitespace}'
+        expires:
+            - null
+            - '{random 1-10000000 digits}'
+            - '{garbage_no_whitespace}'
+        meta_key:
+            - null
+            - 'foo'
+            - '{garbage_no_whitespace}'
+        meta_value:
+            - null
+            - '{garbage_no_whitespace}'
+    choices:
+        - object_put_simple
+        - object_put_acl
+        - object_put_copy
+
+object_put_simple:
+    set: {}
+    headers:
+        - ['0-1', 'cache-control', '{cache_control}']
+        - ['0-1', 'content-disposition', '{content_disposition}']
+        - ['0-1', 'content-encoding', '{content_encoding}']
+        - ['0-1', 'content-length', '{content_length}']
+        - ['0-1', 'content-md5', '{content_md5}']
+        - ['0-1', 'content-type', '{content_type}']
+        - ['0-1', 'expect', '{expect}']
+        - ['0-1', 'expires', '{expires}']
+        - ['0-1', 'x-amz-acl', '{acl_header}']
+        - ['0-6', 'x-amz-meta-{meta_key}', '{meta_value}']
+    choices: []
+
+object_put_acl:
+    set:
+        query: 'acl'
+        body:
+            - null
+            - '2 {garbage}'
+            - '<AccessControlPolicy>{owner}{acl}</AccessControlPolicy>'
+        owner:
+            - null
+            - '7 <Owner>{id}{display_name}</Owner>'
+        id:
+            - null
+            - '<ID>{random 10-200 binary}</ID>'
+            - '<ID>{random 1000-3000 printable}</ID>'
+        display_name:
+            - null
+            - '2 <DisplayName>{random 10-200 binary}</DisplayName>'
+            - '2 <DisplayName>{random 1000-3000 printable}</DisplayName>'
+            - '2 <DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
+        acl:
+            - null
+            - '10 <AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
+        grantee:
+            - null
+            - '7 <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">{id}{display_name}</Grantee>'
+        permission:
+            - null
+            - '7 <Permission>{permission_value}</Permission>'
+        permission_value:
+            - '2 {garbage}'
+            - FULL_CONTROL
+            - WRITE
+            - WRITE_ACP
+            - READ
+            - READ_ACP
+    headers:
+        - ['0-1', 'cache-control', '{cache_control}']
+        - ['0-1', 'content-disposition', '{content_disposition}']
+        - ['0-1', 'content-encoding', '{content_encoding}']
+        - ['0-1', 'content-length', '{content_length}']
+        - ['0-1', 'content-md5', '{content_md5}']
+        - ['0-1', 'content-type', '{content_type}']
+        - ['0-1', 'expect', '{expect}']
+        - ['0-1', 'expires', '{expires}']
+        - ['0-1', 'x-amz-acl', '{acl_header}']
+    choices: []
+
+object_put_copy:
+    set: {}
+    headers:
+        - ['1-1', 'x-amz-copy-source', '{source_object}']
+        - ['0-1', 'x-amz-acl', '{acl_header}']
+        - ['0-1', 'x-amz-metadata-directive', '{metadata_directive}']
+        - ['0-1', 'x-amz-copy-source-if-match', '{if_match_header}']
+        - ['0-1', 'x-amz-copy-source-if-none-match', '{if_none_match_header}']
+        - ['0-1', 'x-amz-copy-source-if-modified-since', '{if_modified_since_header}']
+        - ['0-1', 'x-amz-copy-source-if-unmodified-since', '{if_modified_since_header}']
+    choices: []

s3tests/functional/test_fuzzer.py

@@ -0,0 +1,390 @@
+mport sys
+import itertools
+import nose
+import random
+import string
+import yaml
+
+from s3tests.fuzz_headers import *
+
+from nose.tools import eq_ as eq
+from nose.tools import assert_true
+from nose.plugins.attrib import attr
+
+from .utils import assert_raises
+
+_decision_graph = {}
+
+def check_access_denied(fn, *args, **kwargs):
+    e = assert_raises(boto.exception.S3ResponseError, fn, *args, **kwargs)
+    eq(e.status, 403)
+    eq(e.reason, 'Forbidden')
+    eq(e.error_code, 'AccessDenied')
+
+
+def build_graph():
+    graph = {}
+    graph['start'] = {
+        'set': {},
+        'choices': ['node2']
+    }
+    graph['leaf'] = {
+        'set': {
+            'key1': 'value1',
+            'key2': 'value2'
+        },
+        'headers': [
+            ['1-2', 'random-header-{random 5-10 printable}', '{random 20-30 punctuation}']
+        ],
+        'choices': []
+    }
+    graph['node1'] = {
+        'set': {
+            'key3': 'value3',
+            'header_val': [
+                '3 h1',
+                '2 h2',
+                'h3'
+            ]
+        },
+        'headers': [
+            ['1-1', 'my-header', '{header_val}'],
+        ],
+        'choices': ['leaf']
+    }
+    graph['node2'] = {
+        'set': {
+            'randkey': 'value-{random 10-15 printable}',
+            'path': '/{bucket_readable}',
+            'indirect_key1': '{key1}'
+        },
+        'choices': ['leaf']
+    }
+    graph['bad_node'] = {
+        'set': {
+            'key1': 'value1'
+        },
+        'choices': ['leaf']
+    }
+    graph['nonexistant_child_node'] = {
+        'set': {},
+        'choices': ['leafy_greens']
+    }
+    graph['weighted_node'] = {
+        'set': {
+            'k1': [
+                'foo',
+                '2 bar',
+                '1 baz'
+            ]
+        },
+        'choices': [
+            'foo',
+            '2 bar',
+            '1 baz'
+        ]
+    }
+    graph['null_choice_node'] = {
+        'set': {},
+        'choices': [None]
+    }
+    graph['repeated_headers_node'] = {
+        'set': {},
+        'headers': [
+            ['1-2', 'random-header-{random 5-10 printable}', '{random 20-30 punctuation}']
+        ],
+        'choices': ['leaf']
+    }
+    graph['weighted_null_choice_node'] = {
+        'set': {},
+        'choices': ['3 null']
+    }
+    return graph
+
+
+#def test_foo():
+    #graph_file = open('request_decision_graph.yml', 'r')
+    #graph = yaml.safe_load(graph_file)
+    #eq(graph['bucket_put_simple']['set']['grantee'], 0)
+
+
+def test_load_graph():
+    graph_file = open('request_decision_graph.yml', 'r')
+    graph = yaml.safe_load(graph_file)
+    graph['start']
+
+
+def test_descend_leaf_node():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'leaf', prng)
+
+    eq(decision['key1'], 'value1')
+    eq(decision['key2'], 'value2')
+    e = assert_raises(KeyError, lambda x: decision[x], 'key3')
+
+
+def test_descend_node():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'node1', prng)
+
+    eq(decision['key1'], 'value1')
+    eq(decision['key2'], 'value2')
+    eq(decision['key3'], 'value3')
+
+
+def test_descend_bad_node():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(DecisionGraphError, descend_graph, graph, 'bad_node', prng)
+
+
+def test_descend_nonexistant_child():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(KeyError, descend_graph, graph, 'nonexistant_child_node', prng)
+
+
+def test_expand_random_printable():
+    prng = random.Random(1)
+    got = expand({}, '{random 10-15 printable}', prng)
+    eq(got, '[/pNI$;92@')
+
+
+def test_expand_random_binary():
+    prng = random.Random(1)
+    got = expand({}, '{random 10-15 binary}', prng)
+    eq(got, '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
+
+
+def test_expand_random_printable_no_whitespace():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random 500 printable_no_whitespace}', prng)
+        assert_true(reduce(lambda x, y: x and y, [x not in string.whitespace and x in string.printable for x in got]))
+
+
+def test_expand_random_binary():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random 500 binary_no_whitespace}', prng)
+        assert_true(reduce(lambda x, y: x and y, [x not in string.whitespace for x in got]))
+
+
+def test_expand_random_no_args():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random}', prng)
+        assert_true(0 <= len(got) <= 1000)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in got]))
+
+
+def test_expand_random_no_charset():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random 10-30}', prng)
+        assert_true(10 <= len(got) <= 30)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in got]))
+
+
+def test_expand_random_exact_length():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random 10 digits}', prng)
+        assert_true(len(got) == 10)
+        assert_true(reduce(lambda x, y: x and y, [x in string.digits for x in got]))
+
+
+def test_expand_random_bad_charset():
+    prng = random.Random(1)
+    assert_raises(KeyError, expand, {}, '{random 10-30 foo}', prng)
+
+
+def test_expand_random_missing_length():
+    prng = random.Random(1)
+    assert_raises(ValueError, expand, {}, '{random printable}', prng)
+
+
+def test_assemble_decision():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = assemble_decision(graph, prng)
+
+    eq(decision['key1'], 'value1')
+    eq(decision['key2'], 'value2')
+    eq(decision['randkey'], 'value-{random 10-15 printable}')
+    eq(decision['indirect_key1'], '{key1}')
+    eq(decision['path'], '/{bucket_readable}')
+    assert_raises(KeyError, lambda x: decision[x], 'key3')
+
+
+def test_expand_escape():
+    prng = random.Random(1)
+    decision = dict(
+        foo='{{bar}}',
+        )
+    got = expand(decision, '{foo}', prng)
+    eq(got, '{bar}')
+
+
+def test_expand_indirect():
+    prng = random.Random(1)
+    decision = dict(
+        foo='{bar}',
+        bar='quux',
+        )
+    got = expand(decision, '{foo}', prng)
+    eq(got, 'quux')
+
+
+def test_expand_indirect_double():
+    prng = random.Random(1)
+    decision = dict(
+        foo='{bar}',
+        bar='{quux}',
+        quux='thud',
+        )
+    got = expand(decision, '{foo}', prng)
+    eq(got, 'thud')
+
+
+def test_expand_recursive():
+    prng = random.Random(1)
+    decision = dict(
+        foo='{foo}',
+        )
+    e = assert_raises(RecursionError, expand, decision, '{foo}', prng)
+    eq(str(e), "Runaway recursion in string formatting: 'foo'")
+
+
+def test_expand_recursive_mutual():
+    prng = random.Random(1)
+    decision = dict(
+        foo='{bar}',
+        bar='{foo}',
+        )
+    e = assert_raises(RecursionError, expand, decision, '{foo}', prng)
+    eq(str(e), "Runaway recursion in string formatting: 'foo'")
+
+
+def test_expand_recursive_not_too_eager():
+    prng = random.Random(1)
+    decision = dict(
+        foo='bar',
+        )
+    got = expand(decision, 100*'{foo}', prng)
+    eq(got, 100*'bar')
+
+
+def test_make_choice_unweighted_with_space():
+    prng = random.Random(1)
+    choice = make_choice(['foo bar'], prng)
+    eq(choice, 'foo bar')
+
+def test_weighted_choices():
+    graph = build_graph()
+    prng = random.Random(1)
+
+    choices_made = {}
+    for _ in xrange(1000):
+        choice = make_choice(graph['weighted_node']['choices'], prng)
+        if choices_made.has_key(choice):
+            choices_made[choice] += 1
+        else:
+            choices_made[choice] = 1
+
+    foo_percentage = choices_made['foo'] / 1000.0
+    bar_percentage = choices_made['bar'] / 1000.0
+    baz_percentage = choices_made['baz'] / 1000.0
+    nose.tools.assert_almost_equal(foo_percentage, 0.25, 1)
+    nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
+    nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
+
+
+def test_null_choices():
+    graph = build_graph()
+    prng = random.Random(1)
+    choice = make_choice(graph['null_choice_node']['choices'], prng)
+
+    eq(choice, '')
+
+
+def test_weighted_null_choices():
+    graph = build_graph()
+    prng = random.Random(1)
+    choice = make_choice(graph['weighted_null_choice_node']['choices'], prng)
+
+    eq(choice, '')
+
+
+def test_null_child():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'null_choice_node', prng)
+
+    eq(decision, {})
+
+
+def test_weighted_set():
+    graph = build_graph()
+    prng = random.Random(1)
+
+    choices_made = {}
+    for _ in xrange(1000):
+        choice = make_choice(graph['weighted_node']['set']['k1'], prng)
+        if choices_made.has_key(choice):
+            choices_made[choice] += 1
+        else:
+            choices_made[choice] = 1
+
+    foo_percentage = choices_made['foo'] / 1000.0
+    bar_percentage = choices_made['bar'] / 1000.0
+    baz_percentage = choices_made['baz'] / 1000.0
+    nose.tools.assert_almost_equal(foo_percentage, 0.25, 1)
+    nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
+    nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
+
+
+def test_header_presence():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'node1', prng)
+
+    c1 = itertools.count()
+    c2 = itertools.count()
+    for header, value in decision['headers']:
+        if header == 'my-header':
+            eq(value, '{header_val}')
+            assert_true(next(c1) < 1)
+        elif header == 'random-header-{random 5-10 printable}':
+            eq(value, '{random 20-30 punctuation}')
+            assert_true(next(c2) < 2)
+        else:
+            raise KeyError('unexpected header found: %s' % header)
+
+    assert_true(next(c1))
+    assert_true(next(c2))
+
+
+def test_duplicate_header():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(DecisionGraphError, descend_graph, graph, 'repeated_headers_node', prng)
+
+
+def test_expand_headers():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'node1', prng)
+    expanded_headers = expand_headers(decision, prng)
+
+    for header, value in expanded_headers.iteritems():
+        if header == 'my-header':
+            assert_true(value in ['h1', 'h2', 'h3'])
+        elif header.startswith('random-header-'):
+            assert_true(20 <= len(value) <= 30)
+            assert_true(string.strip(value, RepeatExpandingFormatter.charsets['punctuation']) is '')
+        else:
+            raise DecisionGraphError('unexpected header found: "%s"' % header)
+

s3tests/fuzz_headers.py

@@ -0,0 +1,376 @@
+from boto.s3.connection import S3Connection
+from boto.exception import BotoServerError
+from boto.s3.key import Key
+from httplib import BadStatusLine
+from optparse import OptionParser
+from . import common
+
+import traceback
+import itertools
+import random
+import string
+import struct
+import yaml
+import sys
+import re
+
+
+class DecisionGraphError(Exception):
+    """ Raised when a node in a graph tries to set a header or
+        key that was previously set by another node
+    """
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class RecursionError(Exception):
+    """Runaway recursion in string formatting"""
+
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return '{0.__doc__}: {0.msg!r}'.format(self)
+
+
+def assemble_decision(decision_graph, prng):
+    """ Take in a graph describing the possible decision space and a random
+        number generator and traverse the graph to build a decision
+    """
+    return descend_graph(decision_graph, 'start', prng)
+
+
+def descend_graph(decision_graph, node_name, prng):
+    """ Given a graph and a particular node in that graph, set the values in
+        the node's "set" list, pick a choice from the "choice" list, and
+        recurse.  Finally, return dictionary of values
+    """
+    node = decision_graph[node_name]
+
+    try:
+        choice = make_choice(node['choices'], prng)
+        if choice == '':
+            decision = {}
+        else:
+            decision = descend_graph(decision_graph, choice, prng)
+    except IndexError:
+        decision = {}
+
+    for key, choices in node['set'].iteritems():
+        if key in decision:
+            raise DecisionGraphError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
+        decision[key] = make_choice(choices, prng)
+
+    if 'headers' in node:
+        decision.setdefault('headers', [])
+
+        for desc in node['headers']:
+            try:
+                (repetition_range, header, value) = desc
+            except ValueError:
+                (header, value) = desc
+                repetition_range = '1'
+
+            try:
+                size_min, size_max = repetition_range.split('-', 1)
+            except ValueError:
+                size_min = size_max = repetition_range
+
+            size_min = int(size_min)
+            size_max = int(size_max)
+
+            num_reps = prng.randint(size_min, size_max)
+            if header in [h for h, v in decision['headers']]:
+                    raise DecisionGraphError("Node %s tried to add header '%s', but that header already exists!" %(node_name, header))
+            for _ in xrange(num_reps):
+                decision['headers'].append([header, value])
+
+    return decision
+
+
+def make_choice(choices, prng):
+    """ Given a list of (possibly weighted) options or just a single option!,
+        choose one of the options taking weights into account and return the
+        choice
+    """
+    if isinstance(choices, str):
+        return choices
+    weighted_choices = []
+    for option in choices:
+        if option is None:
+            weighted_choices.append('')
+            continue
+        try:
+            (weight, value) = option.split(None, 1)
+            weight = int(weight)
+        except ValueError:
+            weight = 1
+            value = option
+
+        if value == 'null' or value == 'None':
+            value = ''
+
+        for _ in xrange(weight):
+            weighted_choices.append(value)
+
+    return prng.choice(weighted_choices)
+
+
+def expand_headers(decision, prng):
+    expanded_headers = {} 
+    for header in decision['headers']:
+        h = expand(decision, header[0], prng)
+        v = expand(decision, header[1], prng)
+        expanded_headers[h] = v
+    return expanded_headers
+
+
+def expand(decision, value, prng):
+    c = itertools.count()
+    fmt = RepeatExpandingFormatter(prng)
+    new = fmt.vformat(value, [], decision)
+    return new
+
+
+class RepeatExpandingFormatter(string.Formatter):
+    charsets = {
+        'printable_no_whitespace': string.printable.translate(None, string.whitespace),
+        'printable': string.printable,
+        'punctuation': string.punctuation,
+        'whitespace': string.whitespace,
+        'digits': string.digits
+    }
+
+    def __init__(self, prng, _recursion=0):
+        super(RepeatExpandingFormatter, self).__init__()
+        # this class assumes it is always instantiated once per
+        # formatting; use that to detect runaway recursion
+        self.prng = prng
+        self._recursion = _recursion
+
+    def get_value(self, key, args, kwargs):
+        fields = key.split(None, 1)
+        fn = getattr(self, 'special_{name}'.format(name=fields[0]), None)
+        if fn is not None:
+            if len(fields) == 1:
+                fields.append('')
+            return fn(fields[1])
+
+        val = super(RepeatExpandingFormatter, self).get_value(key, args, kwargs)
+        if self._recursion > 5:
+            raise RecursionError(key)
+        fmt = self.__class__(self.prng, _recursion=self._recursion+1)
+
+        n = fmt.vformat(val, args, kwargs)
+        return n
+
+    def special_random(self, args):
+        arg_list = args.split()
+        try:
+            size_min, size_max = arg_list[0].split('-', 1)
+        except ValueError:
+            size_min = size_max = arg_list[0]
+        except IndexError:
+            size_min = '0'
+            size_max = '1000'
+
+        size_min = int(size_min)
+        size_max = int(size_max)
+        length = self.prng.randint(size_min, size_max)
+
+        try:
+            charset_arg = arg_list[1]
+        except IndexError:
+            charset_arg = 'printable'
+
+        if charset_arg == 'binary' or charset_arg == 'binary_no_whitespace':
+            num_bytes = length + 8
+            tmplist = [self.prng.getrandbits(64) for _ in xrange(num_bytes / 8)]
+            tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
+            if charset_arg == 'binary_no_whitespace':
+                tmpstring = ''.join(c for c in tmpstring if c not in string.whitespace)
+            return tmpstring[0:length]
+        else:
+            charset = self.charsets[charset_arg]
+            return ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
+
+
+def parse_options():
+    parser = OptionParser()
+    parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
+    parser.add_option('--seed', dest='seed', type='int',  help='initial seed for the random number generator')
+    parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
+    parser.add_option('-n', dest='num_requests', type='int',  help='issue NUM requests before stopping', metavar='NUM')
+    parser.add_option('-v', '--verbose', dest='verbose', action="store_true",  help='turn on verbose output')
+    parser.add_option('-d', '--debug', dest='debug', action="store_true",  help='turn on debugging (very verbose) output')
+    parser.add_option('--decision-graph', dest='graph_filename',  help='file in which to find the request decision graph')
+    parser.add_option('--no-cleanup', dest='cleanup', action="store_false", help='turn off teardown so you can peruse the state of buckets after testing')
+
+    parser.set_defaults(num_requests=5)
+    parser.set_defaults(cleanup=True)
+    parser.set_defaults(graph_filename='request_decision_graph.yml')
+    return parser.parse_args()
+
+
+def randomlist(seed=None):
+    """ Returns an infinite generator of random numbers
+    """
+    rng = random.Random(seed)
+    while True:
+        yield rng.randint(0,100000) #100,000 seeds is enough, right?
+
+
+def populate_buckets(conn, alt):
+    """ Creates buckets and keys for fuzz testing and sets appropriate
+        permissions. Returns a dictionary of the bucket and key names.
+    """
+    breadable = common.get_new_bucket(alt)
+    bwritable = common.get_new_bucket(alt)
+    bnonreadable = common.get_new_bucket(alt)
+
+    oreadable = Key(breadable)
+    owritable = Key(bwritable)
+    ononreadable = Key(breadable)
+    oreadable.set_contents_from_string('oreadable body')
+    owritable.set_contents_from_string('owritable body')
+    ononreadable.set_contents_from_string('ononreadable body')
+
+    breadable.set_acl('public-read')
+    bwritable.set_acl('public-read-write')
+    bnonreadable.set_acl('private')
+    oreadable.set_acl('public-read')
+    owritable.set_acl('public-read-write')
+    ononreadable.set_acl('private')
+
+    return dict(
+        bucket_readable=breadable.name,
+        bucket_writable=bwritable.name,
+        bucket_not_readable=bnonreadable.name,
+        bucket_not_writable=breadable.name,
+        object_readable=oreadable.key,
+        object_writable=owritable.key,
+        object_not_readable=ononreadable.key,
+        object_not_writable=oreadable.key,
+    )
+
+
+def _main():
+    """ The main script
+    """
+    (options, args) = parse_options()
+    random.seed(options.seed if options.seed else None)
+    s3_connection = common.s3.main
+    alt_connection = common.s3.alt
+
+    if options.outfile:
+        OUT = open(options.outfile, 'w')
+    else:
+        OUT = sys.stderr
+
+    VERBOSE = DEBUG = open('/dev/null', 'w')
+    if options.verbose:
+        VERBOSE = OUT
+    if options.debug:
+        DEBUG = OUT
+        VERBOSE = OUT
+
+    request_seeds = None
+    if options.seedfile:
+        FH = open(options.seedfile, 'r')
+        request_seeds = [int(line) for line in FH if line != '\n']
+        print>>OUT, 'Seedfile: %s' %options.seedfile
+        print>>OUT, 'Number of requests: %d' %len(request_seeds)
+    else:
+        if options.seed:
+            print>>OUT, 'Initial Seed: %d' %options.seed
+        print>>OUT, 'Number of requests: %d' %options.num_requests
+        random_list = randomlist(options.seed)
+        request_seeds = itertools.islice(random_list, options.num_requests)
+
+    print>>OUT, 'Decision Graph: %s' %options.graph_filename
+
+    graph_file = open(options.graph_filename, 'r')
+    decision_graph = yaml.safe_load(graph_file)
+
+    constants = populate_buckets(s3_connection, alt_connection)
+    print>>VERBOSE, "Test Buckets/Objects:"
+    for key, value in constants.iteritems():
+        print>>VERBOSE, "\t%s: %s" %(key, value)
+
+    print>>OUT, "Begin Fuzzing..."
+    print>>VERBOSE, '='*80
+    for request_seed in request_seeds:
+        print>>VERBOSE, 'Seed is: %r' %request_seed
+        prng = random.Random(request_seed)
+        decision = assemble_decision(decision_graph, prng)
+        decision.update(constants)
+
+        method = expand(decision, decision['method'], prng)
+        path = expand(decision, decision['urlpath'], prng)
+
+        try:
+            body = expand(decision, decision['body'], prng)
+        except KeyError:
+            body = ''
+
+        try:
+            headers = expand_headers(decision, prng)
+        except KeyError:
+            headers = {}
+
+        print>>VERBOSE, "%r %r" %(method[:100], path[:100])
+        for h, v in headers.iteritems():
+            print>>VERBOSE, "%r: %r" %(h[:50], v[:50])
+        print>>VERBOSE, "%r\n" % body[:100]
+
+        print>>DEBUG, 'FULL REQUEST'
+        print>>DEBUG, 'Method: %r' %method
+        print>>DEBUG, 'Path: %r' %path
+        print>>DEBUG, 'Headers:'
+        for h, v in headers.iteritems():
+            print>>DEBUG, "\t%r: %r" %(h, v)
+        print>>DEBUG, 'Body: %r\n' %body
+
+        failed = False # Let's be optimistic, shall we?
+        try:
+            response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=1)
+            body = response.read()
+        except BotoServerError, e:
+            response = e
+            body = e.body
+            failed = True
+        except BadStatusLine, e:
+            print>>OUT, 'FAILED: failed to parse response (BadStatusLine); probably a NUL byte in your request?'
+            print>>VERBOSE, '='*80
+            continue
+
+        if failed:
+            print>>OUT, 'FAILED:'
+            OLD_VERBOSE = VERBOSE
+            OLD_DEBUG = DEBUG
+            VERBOSE = DEBUG = OUT
+        print>>VERBOSE, 'Seed was: %r' %request_seed
+        print>>VERBOSE, 'Response status code: %d %s' %(response.status, response.reason)
+        print>>DEBUG, 'Body:\n%s' %body
+        print>>VERBOSE, '='*80
+        if failed:
+            VERBOSE = OLD_VERBOSE
+            DEBUG = OLD_DEBUG
+
+    print>>OUT, '...done fuzzing'
+
+    if options.cleanup:
+        common.teardown()
+
+
+def main():
+    common.setup()
+    try:
+        _main()
+    except Exception as e:
+        traceback.print_exc()
+        common.teardown()
+

setup.py

@@ -25,6 +25,7 @@ setup(
             's3tests-generate-objects = s3tests.generate_objects:main',
             's3tests-test-readwrite = s3tests.readwrite:main',
             's3tests-test-roundtrip = s3tests.roundtrip:main',
+            's3tests-fuzz-headers = s3tests.fuzz_headers:main',
             ],
         },