diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index e08155c..443b37c 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -1,14 +1,14 @@
start:
set:
garbage:
- - {random 10-3000 printable}
- - {random 10-1000 binary}
+ - '{random 10-3000 printable}'
+ - '{random 10-1000 binary}'
choices:
- bucket
bucket:
set:
- urlpath: /{bucket}
+ urlpath: '/{bucket}'
choices:
- 13 bucket_get
- 8 bucket_put
@@ -18,12 +18,12 @@ bucket:
garbage_method:
set:
method:
- - {random 1-100 printable}
- - {random 10-100 binary}
+ - '{random 1-100 printable}'
+ - '{random 10-100 binary}'
urlpath:
- - /{bucket}
- - /{bucket}/{object}
- - {random 10-1000 binary}
+ - '/{bucket}'
+ - '/{bucket}/{object}'
+ - '{random 10-1000 binary}'
choices:
- bucket_get_simple
- bucket_get_filtered
@@ -36,23 +36,23 @@ bucket_delete:
set:
method: DELETE
bucket:
- - {bucket_writable}
- - {bucket_not_writable}
- - 2 {garbage}
+ - '{bucket_writable}'
+ - '{bucket_not_writable}'
+ - '2 {garbage}'
query:
- null
- policy
- website
- - 2 {garbage}
+ - '2 {garbage}'
choices: []
bucket_get:
set:
method: GET
bucket:
- - {bucket_readable}
- - {bucket_not_readable}
- - 2 {garbage}
+ - '{bucket_readable}'
+ - '{bucket_not_readable}'
+ - '2 {garbage}'
choices:
- 11 bucket_get_simple
- bucket_get_filtered
@@ -70,17 +70,17 @@ bucket_get_simple:
- requestPayment
- versioning
- website
- - 2 {garbage}
+ - '2 {garbage}'
choices: []
bucket_get_uploads:
set:
delimiter:
- null
- - 3 'delimiter={garbage}'
+ - '3 delimiter={garbage}'
prefix:
- null
- - 3 'prefix={garbage}'
+ - '3 prefix={garbage}'
key_marker:
- null
- 'key-marker={object_readable}'
@@ -93,12 +93,12 @@ bucket_get_uploads:
- 'max-uploads={random 1-1000 digits}'
upload_id_marker:
- null
- - 3 'upload-id-marker={random}'
+ - '3 upload-id-marker={random}'
query:
- 'uploads'
- 'uploads&{delimiter}&{prefix}'
- 'uploads&{max_uploads}&{key_marker}&{upload_id_marker}'
- - 2 {garbage}
+ - '2 {garbage}'
choices: []
bucket_get_filtered:
@@ -119,15 +119,15 @@ bucket_get_filtered:
- null
- '{delimiter}&{prefix}'
- '{max-keys}&{marker}'
- - 2 {garbage}
+ - '2 {garbage}'
choices: []
bucket_put:
set:
bucket:
- - {bucket_writable}
- - {bucket_not_writable}
- - 2 {garbage}
+ - '{bucket_writable}'
+ - '{bucket_not_writable}'
+ - '2 {garbage}'
method: PUT
choices:
- bucket_put_simple
@@ -137,12 +137,12 @@ bucket_put:
bucket_put_create:
set:
body:
- - 2 {garbage}
+ - '2 {garbage}'
- '{random 2-10 binary}'
acl:
- private
- - {random 3000 letters}
- - {random 100-1000 binary}
+ - '{random 3000 letters}'
+ - '{random 100-1000 binary}'
headers:
- ['0-1', 'x-amz-acl', '{acl}']
choices: []
@@ -150,8 +150,8 @@ bucket_put_create:
bucket_put_versioning:
set:
body:
- - {garbage}
- - 4 '{versioning_status}{mfa_delete_body}'
+ - '{garbage}'
+ - '4 {versioning_status}{mfa_delete_body}'
mfa_delete_body:
- null
- '{random 2-10 binary}'
@@ -169,38 +169,38 @@ bucket_put_versioning:
bucket_put_simple:
set:
body:
- - {acl_body}
- - {policy_body}
- - {logging_body}
- - {notification_body}
- - {request_payment_body}
- - {website_body}
+ - '{acl_body}'
+ - '{policy_body}'
+ - '{logging_body}'
+ - '{notification_body}'
+ - '{request_payment_body}'
+ - '{website_body}'
acl_body:
- null
- '{owner}{acl}'
owner:
- null
- - 7 '{id}{display_name}'
+ - '7 {id}{display_name}'
id:
- null
- '{random 10-200 binary}'
- '{random 1000-3000 printable}'
display_name:
- null
- - 2 '{random 10-200 binary}'
- - 2 '{random 1000-3000 printable}'
- - 2 '{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}'
+ - '2 {random 10-200 binary}'
+ - '2 {random 1000-3000 printable}'
+ - '2 {random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}'
acl:
- null
- - 10 '{grantee}{permission}'
+ - '10 {grantee}{permission}'
grantee:
- null
- - 7 '{id}{display_name}'
+ - '7 {id}{display_name}'
permission:
- null
- - 7 '{permission_value}'
+ - '7 {permission_value}'
permission_value:
- - 2 {garbage}
+ - '2 {garbage}'
- FULL_CONTROL
- WRITE
- WRITE_ACP
@@ -208,7 +208,7 @@ bucket_put_simple:
- READ_ACP
policy_body:
- null
- - 2 {garbage}
+ - '2 {garbage}'
logging_body:
- null
- ''
@@ -219,31 +219,31 @@ bucket_put_simple:
- '{random 10-1000 binary}'
target_grants:
- null
- - 10 '{grantee}{permission}'
+ - '10 {grantee}{permission}'
notification_body:
- null
- ''
- - 2 '{topic}{event}'
+ - '2 {topic}{event}'
topic:
- null
- - 2 '{garbage}'
+ - '2 {garbage}'
event:
- null
- 's3:ReducedRedundancyLostObject'
- - 2 '{garbage}'
+ - '2 {garbage}'
request_payment_body:
- null
- '{payer}'
payer:
- Requester
- BucketOwner
- - 2 {garbage}
+ - '2 {garbage}'
website_body:
- null
- '{suffix}{error_doc}'
suffix:
- null
- - 2 {garbage}
+ - '2 {garbage}'
- '{random 2-10 printable}.html'
error_doc:
- null
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 12b19bb..eb2ab1f 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -102,6 +102,12 @@ def build_graph():
return graph
+#def test_foo():
+ #graph_file = open('request_decision_graph.yml', 'r')
+ #graph = yaml.safe_load(graph_file)
+ #eq(graph['bucket_put_simple']['set']['grantee'], 0)
+
+
def test_load_graph():
graph_file = open('request_decision_graph.yml', 'r')
graph = yaml.safe_load(graph_file)
@@ -257,6 +263,11 @@ def test_expand_recursive_not_too_eager():
eq(got, 100*'bar')
+def test_make_choice_unweighted_with_space():
+ prng = random.Random(1)
+ choice = make_choice(['foo bar'], prng)
+ eq(choice, 'foo bar')
+
def test_weighted_choices():
graph = build_graph()
prng = random.Random(1)
@@ -354,7 +365,7 @@ def test_expand_headers():
decision = descend_graph(graph, 'node1', prng)
expanded_headers = expand_headers(decision, prng)
- for header, value in expanded_headers:
+ for header, value in expanded_headers.iteritems():
if header == 'my-header':
assert_true(value in ['h1', 'h2', 'h3'])
elif header.startswith('random-header-'):
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 3796fd1..c23ebc6 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -1,4 +1,5 @@
from boto.s3.connection import S3Connection
+from boto.s3.key import Key
from optparse import OptionParser
from boto import UserAgent
from . import common
@@ -103,11 +104,11 @@ def make_choice(choices, prng):
continue
try:
(weight, value) = option.split(None, 1)
+ weight = int(weight)
except ValueError:
- weight = '1'
+ weight = 1
value = option
- weight = int(weight)
if value == 'null' or value == 'None':
value = ''
@@ -118,11 +119,11 @@ def make_choice(choices, prng):
def expand_headers(decision, prng):
- expanded_headers = []
+ expanded_headers = {}
for header in decision['headers']:
h = expand(decision, header[0], prng)
v = expand(decision, header[1], prng)
- expanded_headers.append([h, v])
+ expanded_headers[h] = v
return expanded_headers
@@ -200,6 +201,8 @@ def parse_options():
parser.add_option('--seed', dest='seed', type='int', help='initial seed for the random number generator', metavar='SEED')
parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
parser.add_option('-n', dest='num_requests', type='int', help='issue NUM requests before stopping', metavar='NUM')
+ parser.add_option('-v', '--verbose', dest='verbose', action="store_true", help='turn on verbose output')
+ parser.add_option('-d', '--debug', dest='debug', action="store_true", help='turn on debugging (very verbose) output')
parser.add_option('--decision-graph', dest='graph_filename', help='file in which to find the request decision graph', metavar='NUM')
parser.set_defaults(num_requests=5)
@@ -215,56 +218,127 @@ def randomlist(seed=None):
yield rng.random()
+def populate_buckets(conn, alt):
+ """ Creates buckets and keys for fuzz testing and sets appropriate
+ permissions. Returns a dictionary of the bucket and key names.
+ """
+ breadable = common.get_new_bucket(alt)
+ bwritable = common.get_new_bucket(alt)
+ bnonreadable = common.get_new_bucket(alt)
+
+ oreadable = Key(breadable)
+ owritable = Key(bwritable)
+ ononreadable = Key(breadable)
+ oreadable.set_contents_from_string('oreadable body')
+ owritable.set_contents_from_string('owritable body')
+ ononreadable.set_contents_from_string('ononreadable body')
+
+ breadable.set_acl('public-read')
+ bwritable.set_acl('public-read-write')
+ bnonreadable.set_acl('private')
+ oreadable.set_acl('public-read')
+ owritable.set_acl('public-read-write')
+ ononreadable.set_acl('private')
+
+ return dict(
+ bucket_readable=breadable.name,
+ bucket_writable=bwritable.name,
+ bucket_not_readable=bnonreadable.name,
+ bucket_not_writable=breadable.name,
+ object_readable=oreadable.key,
+ object_writable=owritable.key,
+ object_not_readable=ononreadable.key,
+ object_not_writable=oreadable.key,
+ )
+
+
def _main():
""" The main script
"""
(options, args) = parse_options()
random.seed(options.seed if options.seed else None)
s3_connection = common.s3.main
+ alt_connection = common.s3.alt
+
+ if options.outfile:
+ OUT = open(options.outfile, 'w')
+ else:
+ OUT = sys.stderr
+
+ VERBOSE = DEBUG = open('/dev/null', 'w')
+ if options.verbose:
+ VERBOSE = OUT
+ if options.debug:
+ DEBUG = OUT
+ VERBOSE = OUT
request_seeds = None
if options.seedfile:
FH = open(options.seedfile, 'r')
- request_seeds = FH.readlines()
+ request_seeds = [float(line) for line in FH.readlines()]
+ print>>OUT, 'Seedfile: %s' %options.seedfile
+ print>>OUT, 'Number of requests: %d' %len(request_seeds)
else:
+ if options.seed:
+ print>>OUT, 'Initial Seed: %d' %options.seed
+ print>>OUT, 'Number of requests: %d' %options.num_requests
random_list = randomlist(options.seed)
request_seeds = itertools.islice(random_list, options.num_requests)
+ print>>OUT, 'Decision Graph: %s' %options.graph_filename
graph_file = open(options.graph_filename, 'r')
decision_graph = yaml.safe_load(graph_file)
- constants = dict(
- bucket_readable='TODO-breadable',
- bucket_not_readable='TODO-bnonreadable',
- bucket_writable='TODO-bwritable',
- bucket_not_writable='TODO-bnonwritable',
- object_readable='TODO-oreadable',
- object_not_readable='TODO-ononreadable',
- object_writable='TODO-owritable',
- object_not_writable='TODO-ononwritable',
- )
+ constants = populate_buckets(s3_connection, alt_connection)
+ print>>VERBOSE, "Test Buckets/Objects:"
+ for key, value in constants.iteritems():
+ print>>VERBOSE, "\t%s: %s" %(key, value)
+ print>>OUT, "Begin Fuzzing..."
+ print>>VERBOSE, '='*80
for request_seed in request_seeds:
+ print>>OUT, request_seed
+
prng = random.Random(request_seed)
decision = assemble_decision(decision_graph, prng)
decision.update(constants)
method = expand(decision, decision['method'], prng)
- path = expand(decision, decision['path'], prng)
- body = expand(decision, decision['body'], prng)
- headers = expand_headers(decision, prng)
+ path = expand(decision, decision['urlpath'], prng)
- print "Method: %s" % method
- print "Path: %s" % path
- print "Headers: %s" % headers
- print ""
- print "Body: %s" % body
- #response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+ try:
+ body = expand(decision, decision['body'], prng)
+ except KeyError:
+ body = ''
+ try:
+ headers = expand_headers(decision, prng)
+ except KeyError:
+ headers = {}
+
+ response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+
+ print>>VERBOSE, "%s %s" %(method[:100], path[:100])
+ for h, v in headers.iteritems():
+ print>>VERBOSE, "%s: %s" %(h[:50], v[:50])
+ print>>VERBOSE, "%s\n" % body[:100]
+
+ print>>DEBUG, 'FULL REQUEST'
+ print>>DEBUG, 'Method: %r' %method
+ print>>DEBUG, 'Path: %r' %path
+ print>>DEBUG, 'Headers:'
+ for h, v in headers.iteritems():
+ print>>DEBUG, "\t%r: %r" %(h, v)
+ print>>DEBUG, 'Body: %r' %body
+
+ print>>VERBOSE, 'Response status code: %d %s' %(response.status, response.reason)
+ print>>DEBUG, 'Body:\n%s' %response.read()
if response.status == 500 or response.status == 503:
- print 'Request generated with seed %d failed:\n%s' % (request_seed, request)
- pass
+ print>>OUT, 'FAILED:\n%s' %request
+ print>>VERBOSE, '='*80
+ print>>OUT, '...done fuzzing'
+ common.teardown()
def main():