From 93cf99aed9f699eda10553bf36f6a72aa200aab1 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 3 Aug 2011 17:00:02 -0700
Subject: [PATCH 01/24] S3 Fuzzer: Add skeleton for a targeted S3 Fuzzer

---
 s3tests/fuzz_headers.py | 73 +++++++++++++++++++++++++++++++++++++++++
 setup.py                |  1 +
 2 files changed, 74 insertions(+)
 create mode 100644 s3tests/fuzz_headers.py

diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
new file mode 100644
index 0000000..c649080
--- /dev/null
+++ b/s3tests/fuzz_headers.py
@@ -0,0 +1,73 @@
+from boto.s3 import S3Connection
+from optparse import OptionParser
+from . import common
+
+import traceback
+import random
+import string
+import sys
+
+
+class FuzzyRequest(object):
+    # Initialized with a seed to be reproducible.
+    # string repr needs to look like:
+    # METHOD PATH HTTP/1.1
+    # HEADER_KEY: HEADER_VALUE[, HEADER_VALUE...]
+    # [	: HEADER_VALUE[, HEADER_VALUE...]
+    # <additional headers>
+    #
+    # BODY
+    pass
+
+def parse_options():
+    parser = OptionParser()
+    parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
+    parser.add_option('--seed', dest='seed', help='initial seed for the random number generator', metavar='SEED')
+    parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
+    parser.add_option('-n', dest='num_requests', help='issue NUM requests before stopping', metavar='NUM')
+
+    return parser.parse_args()
+
+
+def randomlist(n, seed=None):
+    """ Returns a generator function that spits out a list of random numbers n elements long.
+    """
+    rng = random.Random()
+    rng.seed(seed if seed else None)
+    for _ in xrange(n):
+        yield rng.random()
+
+
+def _main():
+    """ The main script
+    """
+    (options, args) = parse_options()
+    random.seed(options.seed if options.seed else None)
+    s3_connection = config.s3.main
+
+    request_seeds
+    if options.seedfile:
+        FH = open(options.seedfile, 'r')
+        request_seeds = FH.readlines()
+    else:
+        request_seeds = randomlist(options.num_requests, options.seed)
+
+    for i in request_seeds:
+        fuzzy = FuzzyRequest(request_seed)
+
+        http_connection = s3_connection.get_http_connection(s3_connection.host, s3_connection.is_secure)
+        http_connection.request(fuzzy.method, fuzzy.path, body=fuzzy.body, headers=fuzzy.headers)
+
+        response = http_connection.getresponse()
+        if response.status == 500 or response.status == 503:
+            print 'Request generated with seed %d failed:\n%s' % (request_seed, fuzzy)
+
+
+def main():
+    common.setup()
+    try:
+        _main()
+    except Exception as e:
+        traceback.print_exc()
+        common.teardown()
+
diff --git a/setup.py b/setup.py
index 1fc636c..edcab1a 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,7 @@ setup(
             's3tests-generate-objects = s3tests.generate_objects:main',
             's3tests-test-readwrite = s3tests.readwrite:main',
             's3tests-test-roundtrip = s3tests.roundtrip:main',
+            's3tests-fuzz-headers = s3tests.fuzz_headers:main',
             ],
         },
 

From 31adf6afa58926783f64400b0f195d330ad74a11 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Thu, 4 Aug 2011 16:59:32 -0700
Subject: [PATCH 02/24] S3 Fuzzer: Begin writing unit tests for S3 Fuzzer

---
 s3tests/fuzz_headers.py | 87 ++++++++++++++++++++++++++++++++---------
 1 file changed, 68 insertions(+), 19 deletions(-)

diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index c649080..799d918 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -1,4 +1,4 @@
-from boto.s3 import S3Connection
+from boto.s3.connection import S3Connection
 from optparse import OptionParser
 from . import common
 
@@ -9,23 +9,70 @@ import sys
 
 
 class FuzzyRequest(object):
+    """ FuzzyRequests are initialized with a random seed and generate data to
+        get sent as valid or valid-esque HTTP requests for targeted fuzz testing
+    """
     # Initialized with a seed to be reproducible.
-    # string repr needs to look like:
-    # METHOD PATH HTTP/1.1
-    # HEADER_KEY: HEADER_VALUE[, HEADER_VALUE...]
-    # [	: HEADER_VALUE[, HEADER_VALUE...]
-    # <additional headers>
-    #
-    # BODY
-    pass
+
+    def __init__(self, seed):
+        self.random = random.Random()
+        self.seed = seed
+        self.random.seed(self.seed)
+
+        self._generate_method()
+        self._generate_path()
+        self._generate_body()
+        self._generate_headers()
+
+
+    def __str__(self):
+        s = '%s %s HTTP/1.1\n' % (self.method, self.path)
+        for header, value in self.headers.iteritems():
+            s += '%s: ' %header
+            if isinstance(value, list):
+                for val in value:
+                    s += '%s ' %val
+            else:
+                s += value
+            s += '\n'
+        s += '\n' # Blank line after headers are done.
+        s += '%s\r\n\r\n' %self.body
+        return s
+
+
+    def _generate_method(self):
+        METHODS = ['GET', 'POST', 'HEAD', 'PUT']
+        self.method = self.random.choice(METHODS)
+
+
+    def _generate_path(self):
+        path_charset = string.letters + string.digits
+        path_len = self.random.randint(0,1000)
+        self.path = ''
+        for _ in xrange(path_len):
+            self.path += self.random.choice(path_charset)
+
+
+    def _generate_body(self):
+        body_charset = string.printable
+        body_len = self.random.randint(0, 1000)
+        self.body = ''
+        for _ in xrange(body_len):
+            self.body += self.random.choice(body_charset)
+
+
+    def _generate_headers(self):
+        self.headers = {'Foo': 'bar', 'baz': ['a', 'b', 'c']} #FIXME
+
 
 def parse_options():
     parser = OptionParser()
     parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
-    parser.add_option('--seed', dest='seed', help='initial seed for the random number generator', metavar='SEED')
+    parser.add_option('--seed', dest='seed', type='int',  help='initial seed for the random number generator', metavar='SEED')
     parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
-    parser.add_option('-n', dest='num_requests', help='issue NUM requests before stopping', metavar='NUM')
+    parser.add_option('-n', dest='num_requests', type='int',  help='issue NUM requests before stopping', metavar='NUM')
 
+    parser.set_defaults(num_requests=5)
     return parser.parse_args()
 
 
@@ -43,24 +90,26 @@ def _main():
     """
     (options, args) = parse_options()
     random.seed(options.seed if options.seed else None)
-    s3_connection = config.s3.main
+    s3_connection = common.s3.main
 
-    request_seeds
+    request_seeds = None
     if options.seedfile:
         FH = open(options.seedfile, 'r')
         request_seeds = FH.readlines()
     else:
         request_seeds = randomlist(options.num_requests, options.seed)
 
-    for i in request_seeds:
+    for request_seed in request_seeds:
         fuzzy = FuzzyRequest(request_seed)
 
-        http_connection = s3_connection.get_http_connection(s3_connection.host, s3_connection.is_secure)
-        http_connection.request(fuzzy.method, fuzzy.path, body=fuzzy.body, headers=fuzzy.headers)
+        print fuzzy.seed, fuzzy
+        #TODO: Authenticated requests
+        #http_connection = s3_connection.get_http_connection(s3_connection.host, s3_connection.is_secure)
+        #http_connection.request(fuzzy.method, fuzzy.path, body=fuzzy.body, headers=fuzzy.headers)
 
-        response = http_connection.getresponse()
-        if response.status == 500 or response.status == 503:
-            print 'Request generated with seed %d failed:\n%s' % (request_seed, fuzzy)
+        #response = http_connection.getresponse()
+        #if response.status == 500 or response.status == 503:
+            #print 'Request generated with seed %d failed:\n%s' % (fuzzy.seed, fuzzy)
 
 
 def main():

From 691955935da4ef7bc857010ee4c8e5ba62e6e098 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Fri, 5 Aug 2011 11:42:33 -0700
Subject: [PATCH 03/24] S3 Fuzzer: add authorization to FuzzyRequest

---
 s3tests/fuzz_headers.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 799d918..dc63a44 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -1,5 +1,6 @@
 from boto.s3.connection import S3Connection
 from optparse import OptionParser
+from boto import UserAgent
 from . import common
 
 import traceback
@@ -12,8 +13,6 @@ class FuzzyRequest(object):
     """ FuzzyRequests are initialized with a random seed and generate data to
         get sent as valid or valid-esque HTTP requests for targeted fuzz testing
     """
-    # Initialized with a seed to be reproducible.
-
     def __init__(self, seed):
         self.random = random.Random()
         self.seed = seed
@@ -47,10 +46,11 @@ class FuzzyRequest(object):
 
     def _generate_path(self):
         path_charset = string.letters + string.digits
-        path_len = self.random.randint(0,1000)
+        path_len = self.random.randint(0,100)
         self.path = ''
         for _ in xrange(path_len):
             self.path += self.random.choice(path_charset)
+        self.auth_path = self.path # Not sure how important this is for these tests
 
 
     def _generate_body(self):
@@ -65,6 +65,14 @@ class FuzzyRequest(object):
         self.headers = {'Foo': 'bar', 'baz': ['a', 'b', 'c']} #FIXME
 
 
+    def authorize(self, connection):
+        #Stolen shamelessly from boto's connection.py
+        connection._auth_handler.add_auth(self)
+        self.headers['User-Agent'] = UserAgent
+        if not self.headers.has_key('Content-Length'):
+            self.headers['Content-Length'] = str(len(self.body))
+
+
 def parse_options():
     parser = OptionParser()
     parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
@@ -101,9 +109,8 @@ def _main():
 
     for request_seed in request_seeds:
         fuzzy = FuzzyRequest(request_seed)
-
+        fuzzy.authorize(s3_connection)
         print fuzzy.seed, fuzzy
-        #TODO: Authenticated requests
         #http_connection = s3_connection.get_http_connection(s3_connection.host, s3_connection.is_secure)
         #http_connection.request(fuzzy.method, fuzzy.path, body=fuzzy.body, headers=fuzzy.headers)
 

From fc93c029639dd2fc82c1008476f09adf3fa45ffa Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Mon, 8 Aug 2011 16:51:10 -0700
Subject: [PATCH 04/24] S3 Fuzzer: Change direction towards decision tree

Fuzzer now builds requests based on a DAG that describes the request space
and attack surface.
---
 request_decision_graph.yml        |  36 +++++++++++
 s3tests/functional/test_fuzzer.py |  35 ++++++++++
 s3tests/fuzz_headers.py           | 103 ++++++++++--------------------
 3 files changed, 106 insertions(+), 68 deletions(-)
 create mode 100644 request_decision_graph.yml
 create mode 100644 s3tests/functional/test_fuzzer.py

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
new file mode 100644
index 0000000..774d391
--- /dev/null
+++ b/request_decision_graph.yml
@@ -0,0 +1,36 @@
+start:
+    set: null
+    choice:
+        - bucket
+
+bucket:
+    set:
+        urlpath: '/{bucket}'
+    choice:
+        - bucket_get
+        - bucket_put
+        - bucket_delete
+
+bucket_delete:
+    set:
+        method: 'DELETE'
+    choice:
+        - delete_bucket
+        - delete_bucket_policy
+        - delete_bucket_website
+
+delete_bucket:
+    set:
+        query: null
+    choice: null
+
+delete_bucket_policy:
+    set:
+        query: 'policy'
+    choice: null
+
+delete_bucket_website:
+    set:
+        query: 'website'
+    choice: null
+
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
new file mode 100644
index 0000000..0b28653
--- /dev/null
+++ b/s3tests/functional/test_fuzzer.py
@@ -0,0 +1,35 @@
+import nose
+import random
+import string
+import yaml
+
+from s3tests.fuzz_headers import *
+
+from nose.tools import eq_ as eq
+from nose.plugins.attrib import attr
+
+from .utils import assert_raises
+
+_decision_graph = {}
+
+def check_access_denied(fn, *args, **kwargs):
+    e = assert_raises(boto.exception.S3ResponseError, fn, *args, **kwargs)
+    eq(e.status, 403)
+    eq(e.reason, 'Forbidden')
+    eq(e.error_code, 'AccessDenied')
+
+
+def read_graph():
+    graph_file = open('request_decision_graph.yml', 'r')
+    return yaml.safe_load(graph_file)
+
+
+def test_assemble_decision():
+    graph = read_graph()
+    prng = random.Random(1)
+    decision = assemble_decision(graph, prng)
+    decision['path']
+    decision['method']
+    decision['body']
+    decision['headers']
+
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index dc63a44..9886b25 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -6,71 +6,23 @@ from . import common
 import traceback
 import random
 import string
+import yaml
 import sys
 
 
-class FuzzyRequest(object):
-    """ FuzzyRequests are initialized with a random seed and generate data to
-        get sent as valid or valid-esque HTTP requests for targeted fuzz testing
+def assemble_decision(decision_graph, prng):
+    """ Take in a graph describing the possible decision space and a random
+        number generator and traverse the graph to build a decision
     """
-    def __init__(self, seed):
-        self.random = random.Random()
-        self.seed = seed
-        self.random.seed(self.seed)
-
-        self._generate_method()
-        self._generate_path()
-        self._generate_body()
-        self._generate_headers()
+    raise NotImplementedError
 
 
-    def __str__(self):
-        s = '%s %s HTTP/1.1\n' % (self.method, self.path)
-        for header, value in self.headers.iteritems():
-            s += '%s: ' %header
-            if isinstance(value, list):
-                for val in value:
-                    s += '%s ' %val
-            else:
-                s += value
-            s += '\n'
-        s += '\n' # Blank line after headers are done.
-        s += '%s\r\n\r\n' %self.body
-        return s
-
-
-    def _generate_method(self):
-        METHODS = ['GET', 'POST', 'HEAD', 'PUT']
-        self.method = self.random.choice(METHODS)
-
-
-    def _generate_path(self):
-        path_charset = string.letters + string.digits
-        path_len = self.random.randint(0,100)
-        self.path = ''
-        for _ in xrange(path_len):
-            self.path += self.random.choice(path_charset)
-        self.auth_path = self.path # Not sure how important this is for these tests
-
-
-    def _generate_body(self):
-        body_charset = string.printable
-        body_len = self.random.randint(0, 1000)
-        self.body = ''
-        for _ in xrange(body_len):
-            self.body += self.random.choice(body_charset)
-
-
-    def _generate_headers(self):
-        self.headers = {'Foo': 'bar', 'baz': ['a', 'b', 'c']} #FIXME
-
-
-    def authorize(self, connection):
-        #Stolen shamelessly from boto's connection.py
-        connection._auth_handler.add_auth(self)
-        self.headers['User-Agent'] = UserAgent
-        if not self.headers.has_key('Content-Length'):
-            self.headers['Content-Length'] = str(len(self.body))
+def expand_decision(decision, prng):
+    """ Take in a decision and a random number generator.  Expand variables in
+        decision's values and headers until all values are fully expanded and
+        build a request out of the information
+    """
+    raise NotImplementedError
 
 
 def parse_options():
@@ -79,8 +31,10 @@ def parse_options():
     parser.add_option('--seed', dest='seed', type='int',  help='initial seed for the random number generator', metavar='SEED')
     parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
     parser.add_option('-n', dest='num_requests', type='int',  help='issue NUM requests before stopping', metavar='NUM')
+    parser.add_option('--decision-graph', dest='graph_filename',  help='file in which to find the request decision graph', metavar='NUM')
 
     parser.set_defaults(num_requests=5)
+    parser.set_defaults(graph_filename='request_decision_graph.yml')
     return parser.parse_args()
 
 
@@ -107,16 +61,29 @@ def _main():
     else:
         request_seeds = randomlist(options.num_requests, options.seed)
 
-    for request_seed in request_seeds:
-        fuzzy = FuzzyRequest(request_seed)
-        fuzzy.authorize(s3_connection)
-        print fuzzy.seed, fuzzy
-        #http_connection = s3_connection.get_http_connection(s3_connection.host, s3_connection.is_secure)
-        #http_connection.request(fuzzy.method, fuzzy.path, body=fuzzy.body, headers=fuzzy.headers)
+    graph_file = open(options.graph_filename, 'r')
+    decision_graph = yaml.safe_load(graph_file)
 
-        #response = http_connection.getresponse()
-        #if response.status == 500 or response.status == 503:
-            #print 'Request generated with seed %d failed:\n%s' % (fuzzy.seed, fuzzy)
+    constants = {
+        'bucket_readable': 'TODO',
+        'bucket_writable' : 'TODO',
+        'bucket_nonexistant' : 'TODO',
+        'object_readable' : 'TODO',
+        'object_writable' : 'TODO',
+        'object_nonexistant' : 'TODO'
+    }
+
+    for request_seed in request_seeds:
+        prng = random.Random(request_seed)
+        decision = assemble_decision(decision_graph, prng)
+        decision.update(constants)
+        request = expand_decision(decision, prng) 
+
+        response = s3_connection.make_request(request['method'], request['path'], data=request['body'], headers=request['headers'], override_num_retries=0)
+
+        if response.status == 500 or response.status == 503:
+            print 'Request generated with seed %d failed:\n%s' % (request_seed, request)
+        pass
 
 
 def main():

From a9a41a289199a6cf183be65c3176c4f07464ef05 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Tue, 9 Aug 2011 11:56:38 -0700
Subject: [PATCH 05/24] S3 Fuzzer: began writing graph descent

still missing headers and choice weights
---
 request_decision_graph.yml        |  8 ++--
 s3tests/functional/test_fuzzer.py | 61 ++++++++++++++++++++++++++-----
 s3tests/fuzz_headers.py           | 22 ++++++++++-
 3 files changed, 77 insertions(+), 14 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index 774d391..d7b0206 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -1,5 +1,5 @@
 start:
-    set: null
+    set: {}
     choice:
         - bucket
 
@@ -22,15 +22,15 @@ bucket_delete:
 delete_bucket:
     set:
         query: null
-    choice: null
+    choice: []
 
 delete_bucket_policy:
     set:
         query: 'policy'
-    choice: null
+    choice: []
 
 delete_bucket_website:
     set:
         query: 'website'
-    choice: null
+    choice: []
 
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 0b28653..69dd8fc 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -19,17 +19,60 @@ def check_access_denied(fn, *args, **kwargs):
     eq(e.error_code, 'AccessDenied')
 
 
-def read_graph():
+def build_graph():
+    graph = {}
+    graph['start'] = {
+        'set': {},
+        'choices': ['node1']
+    }
+    graph['leaf'] = {
+        'set': {
+            'key1': 'value1',
+            'key2': 'value2'
+        },
+        'choices': []
+    }
+    graph['node1'] = {
+        'set': {
+            'key3': 'value3'
+        },
+        'choices': ['leaf']
+    }
+    graph['bad_node'] = {
+        'set': {
+            'key1': 'value1'
+        },
+        'choices': ['leaf']
+    }
+    return graph
+
+
+def test_load_graph():
     graph_file = open('request_decision_graph.yml', 'r')
-    return yaml.safe_load(graph_file)
+    graph = yaml.safe_load(graph_file)
+    graph['start']
 
 
-def test_assemble_decision():
-    graph = read_graph()
+def test_descend_leaf_node():
+    graph = build_graph()
     prng = random.Random(1)
-    decision = assemble_decision(graph, prng)
-    decision['path']
-    decision['method']
-    decision['body']
-    decision['headers']
+    decision = descend_graph(graph, 'leaf', prng)
+
+    eq(decision['key1'], 'value1')
+    eq(decision['key2'], 'value2')
+    e = assert_raises(KeyError, lambda x: decision[x], 'key3')
+
+def test_descend_node():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'node1', prng)
+
+    eq(decision['key1'], 'value1')
+    eq(decision['key2'], 'value2')
+    eq(decision['key3'], 'value3')
+
+def test_descend_bad_node():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(KeyError, descend_graph, graph, 'bad_node', prng)
 
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 9886b25..5d538dd 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -14,7 +14,27 @@ def assemble_decision(decision_graph, prng):
     """ Take in a graph describing the possible decision space and a random
         number generator and traverse the graph to build a decision
     """
-    raise NotImplementedError
+    return descend_graph(decision_graph, 'start', prng)
+
+
+def descend_graph(decision_graph, node_name, prng):
+    """ Given a graph and a particular node in that graph, set the values in
+        the node's "set" list, pick a choice from the "choice" list, and
+        recurse.  Finally, return dictionary of values
+    """
+    try:
+        choice = prng.choice(decision_graph[node_name]['choices'])
+        decision = descend_graph(decision_graph, choice, prng)
+    except IndexError:
+        decision = {}
+
+    node = decision_graph[node_name]
+
+    for key in node['set']:
+        if decision.has_key(key):
+            raise KeyError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
+        decision[key] = node['set'][key]
+    return decision
 
 
 def expand_decision(decision, prng):

From 7d9ec02686d04882ef6448f481c5abd8652f4186 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Tue, 9 Aug 2011 15:44:25 -0700
Subject: [PATCH 06/24] S3 Fuzzer: Added SpecialVariables dict subclass

Helper class to catch sentinal keys and turn them into random values.  This
will be used to generate garbage data when expanding a decision.

Also add unit tests for expand_decision and assemble_decision
---
 s3tests/functional/test_fuzzer.py | 48 ++++++++++++++++++++++++++++++-
 s3tests/fuzz_headers.py           | 48 +++++++++++++++++++++++++++++--
 2 files changed, 92 insertions(+), 4 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 69dd8fc..b8393fb 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -23,7 +23,7 @@ def build_graph():
     graph = {}
     graph['start'] = {
         'set': {},
-        'choices': ['node1']
+        'choices': ['node2']
     }
     graph['leaf'] = {
         'set': {
@@ -38,6 +38,14 @@ def build_graph():
         },
         'choices': ['leaf']
     }
+    graph['node2'] = {
+        'set': {
+            'randkey': 'value-{random 10-15 printable}',
+            'path': '/{bucket_readable}',
+            'indirect_key1': '{key1}'
+        },
+        'choices': ['leaf']
+    }
     graph['bad_node'] = {
         'set': {
             'key1': 'value1'
@@ -62,6 +70,7 @@ def test_descend_leaf_node():
     eq(decision['key2'], 'value2')
     e = assert_raises(KeyError, lambda x: decision[x], 'key3')
 
+
 def test_descend_node():
     graph = build_graph()
     prng = random.Random(1)
@@ -71,8 +80,45 @@ def test_descend_node():
     eq(decision['key2'], 'value2')
     eq(decision['key3'], 'value3')
 
+
 def test_descend_bad_node():
     graph = build_graph()
     prng = random.Random(1)
     assert_raises(KeyError, descend_graph, graph, 'bad_node', prng)
 
+
+def test_SpecialVariables_dict():
+    prng = random.Random(1)
+    testdict = {'foo': 'bar'}
+    tester = SpecialVariables(testdict, prng)
+
+    eq(tester['foo'], 'bar')
+    eq(tester['random 10-15 printable'], '[/pNI$;92@') #FIXME: how should I test pseudorandom content?
+
+def test_assemble_decision():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = assemble_decision(graph, prng)
+
+    eq(decision['key1'], 'value1')
+    eq(decision['key2'], 'value2')
+    eq(decision['randkey'], 'value-{random 10-15 printable}')
+    eq(decision['indirect_key1'], '{key1}')
+    eq(decision['path'], '/{bucket_readable}')
+    assert_raises(KeyError, lambda x: decision[x], 'key3')
+
+def test_expand_decision():
+    graph = build_graph()
+    prng = random.Random(1)
+
+    decision = assemble_decision(graph, prng)
+    decision.update({'bucket_readable': 'my-readable-bucket'})
+
+    request = expand_decision(decision, prng)
+
+    eq(request['key1'], 'value1')
+    eq(request['indirect_key1'], 'value1')
+    eq(request['path'], '/my-readable-bucket')
+    eq(request['randkey'], 'value-?') #FIXME: again, how to handle the pseudorandom content?
+    assert_raises(KeyError, lambda x: decision[x], 'key3')
+
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 5d538dd..3ac6464 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -22,14 +22,16 @@ def descend_graph(decision_graph, node_name, prng):
         the node's "set" list, pick a choice from the "choice" list, and
         recurse.  Finally, return dictionary of values
     """
+    node = decision_graph[node_name]
+
     try:
-        choice = prng.choice(decision_graph[node_name]['choices'])
+        #TODO: Give weights to each choice
+        choice = prng.choice(node['choices'])
         decision = descend_graph(decision_graph, choice, prng)
     except IndexError:
         decision = {}
 
-    node = decision_graph[node_name]
-
+    #TODO: Add in headers
     for key in node['set']:
         if decision.has_key(key):
             raise KeyError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
@@ -45,6 +47,46 @@ def expand_decision(decision, prng):
     raise NotImplementedError
 
 
+class SpecialVariables(dict):
+    charsets = {
+        'printable': string.printable,
+        'punctuation': string.punctuation,
+        'whitespace': string.whitespace
+    }
+
+    def __init__(self, orig_dict, prng):
+        self.update(orig_dict)
+        self.prng = prng
+
+
+    def __getitem__(self, key):
+        fields = key.split(None, 1)
+        fn = getattr(self, 'special_{name}'.format(name=fields[0]), None)
+        if fn is None:
+            return super(SpecialVariables, self).__getitem__(key)
+
+        if len(fields) == 1:
+            fields.apppend('')
+        return fn(fields[1])
+
+
+    def special_random(self, args):
+        arg_list = args.split()
+        try:
+            size_min, size_max = [int(x) for x in arg_list[0].split('-')]
+        except IndexError:
+            size_min = 0
+            size_max = 1000
+        try:
+            charset = self.charsets[arg_list[1]]
+        except IndexError:
+            charset = self.charsets['printable']
+
+        length = self.prng.randint(size_min, size_max)
+        return ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
+
+
+
 def parse_options():
     parser = OptionParser()
     parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')

From f5bb3f9c15d1a940d484cc8d172d5d908a4bf75d Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 10 Aug 2011 11:27:06 -0700
Subject: [PATCH 07/24] S3 Fuzzer: implemented decision expansion

Decision reached by descending the graph describing the attack surface can
be expanded to fill in all the placeholders
---
 s3tests/functional/test_fuzzer.py | 29 ++++++++++++++++++++++++++++-
 s3tests/fuzz_headers.py           | 20 +++++++++++++++++++-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index b8393fb..031e59f 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -107,6 +107,33 @@ def test_assemble_decision():
     eq(decision['path'], '/{bucket_readable}')
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
+def test_expand_key():
+    prng = random.Random(1)
+    test_decision = {
+        'key1': 'value1',
+        'randkey': 'value-{random 10-15 printable}',
+        'indirect': '{key1}',
+        'dbl_indirect': '{indirect}'
+    }
+    decision = SpecialVariables(test_decision, prng)
+
+    randkey = expand_key(decision, 'randkey')
+    indirect = expand_key(decision, 'indirect')
+    dbl_indirect = expand_key(decision, 'dbl_indirect')
+
+    eq(indirect, 'value1')
+    eq(dbl_indirect, 'value1')
+    eq(randkey, 'value-[/pNI$;92@')
+
+def test_expand_loop():
+    prng = random.Random(1)
+    test_decision = {
+        'key1': '{key2}',
+        'key2': '{key1}',
+    }
+    decision = SpecialVariables(test_decision, prng)
+    assert_raises(RuntimeError, expand_key, decision, 'key1')
+
 def test_expand_decision():
     graph = build_graph()
     prng = random.Random(1)
@@ -119,6 +146,6 @@ def test_expand_decision():
     eq(request['key1'], 'value1')
     eq(request['indirect_key1'], 'value1')
     eq(request['path'], '/my-readable-bucket')
-    eq(request['randkey'], 'value-?') #FIXME: again, how to handle the pseudorandom content?
+    eq(request['randkey'], 'value-NI$;92@H/0I') #FIXME: again, how to handle the pseudorandom content?
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 3ac6464..54095b6 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -4,6 +4,7 @@ from boto import UserAgent
 from . import common
 
 import traceback
+import itertools
 import random
 import string
 import yaml
@@ -44,7 +45,24 @@ def expand_decision(decision, prng):
         decision's values and headers until all values are fully expanded and
         build a request out of the information
     """
-    raise NotImplementedError
+    special_decision = SpecialVariables(decision, prng)
+    for key in special_decision:
+        decision[key] = expand_key(special_decision, key)
+
+    return decision
+
+
+def expand_key(decision, key):
+    c = itertools.count()
+    fmt = string.Formatter()
+    old = decision[key]
+    while True:
+        new = fmt.vformat(old, [], decision)
+        if new == old:
+            return old
+        if next(c) > 5:
+            raise RuntimeError
+        old = new
 
 
 class SpecialVariables(dict):

From 195571b5551012bd8315359df3425290382c1636 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 10 Aug 2011 13:26:00 -0700
Subject: [PATCH 08/24] S3 Fuzzer: support weights for choices

---
 s3tests/functional/test_fuzzer.py | 27 +++++++++++++++++++++++++++
 s3tests/fuzz_headers.py           | 19 +++++++++++++++++--
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 031e59f..089d52b 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -52,6 +52,14 @@ def build_graph():
         },
         'choices': ['leaf']
     }
+    graph['weighted_choices'] = {
+        'set': {},
+        'choices': [
+            'foo',
+            '2 bar',
+            '1 baz'
+        ]
+    }
     return graph
 
 
@@ -149,3 +157,22 @@ def test_expand_decision():
     eq(request['randkey'], 'value-NI$;92@H/0I') #FIXME: again, how to handle the pseudorandom content?
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
+def test_weighted_choices():
+    graph = build_graph()
+    prng = random.Random(1)
+
+    choices_made = {}
+    for _ in xrange(1000):
+        choice = make_choice(graph['weighted_choices']['choices'], prng)
+        if choices_made.has_key(choice):
+            choices_made[choice] += 1
+        else:
+            choices_made[choice] = 1
+
+    foo_percentage = choices_made['foo'] / 1000.0
+    bar_percentage = choices_made['bar'] / 1000.0
+    baz_percentage = choices_made['baz'] / 1000.0
+    nose.tools.assert_almost_equal(foo_percentage, 0.25, 1)
+    nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
+    nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
+
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 54095b6..91356f8 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -26,8 +26,7 @@ def descend_graph(decision_graph, node_name, prng):
     node = decision_graph[node_name]
 
     try:
-        #TODO: Give weights to each choice
-        choice = prng.choice(node['choices'])
+        choice = make_choice(node['choices'], prng)
         decision = descend_graph(decision_graph, choice, prng)
     except IndexError:
         decision = {}
@@ -40,6 +39,22 @@ def descend_graph(decision_graph, node_name, prng):
     return decision
 
 
+def make_choice(choices, prng):
+    weighted_choices = []
+    for option in choices:
+        fields = option.split(None, 1)
+        if len(fields) == 1:
+            weight = 1
+            value = fields[0]
+        else:
+            weight = int(fields[0])
+            value = fields[1]
+        for _ in xrange(weight):
+            weighted_choices.append(value)
+
+    return prng.choice(weighted_choices)
+
+
 def expand_decision(decision, prng):
     """ Take in a decision and a random number generator.  Expand variables in
         decision's values and headers until all values are fully expanded and

From 4737652fc1b08c1442e7c7792ee71bfe5aeee441 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 10 Aug 2011 14:39:25 -0700
Subject: [PATCH 09/24] S3 Fuzzer: Added binary mode to random data generator

---
 s3tests/functional/test_fuzzer.py | 10 ++++++++--
 s3tests/fuzz_headers.py           | 10 +++++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 089d52b..dabdbee 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -101,7 +101,13 @@ def test_SpecialVariables_dict():
     tester = SpecialVariables(testdict, prng)
 
     eq(tester['foo'], 'bar')
-    eq(tester['random 10-15 printable'], '[/pNI$;92@') #FIXME: how should I test pseudorandom content?
+    eq(tester['random 10-15 printable'], '[/pNI$;92@')
+
+def test_SpecialVariables_binary():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    eq(tester['random 10-15 binary'], '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
 
 def test_assemble_decision():
     graph = build_graph()
@@ -154,7 +160,7 @@ def test_expand_decision():
     eq(request['key1'], 'value1')
     eq(request['indirect_key1'], 'value1')
     eq(request['path'], '/my-readable-bucket')
-    eq(request['randkey'], 'value-NI$;92@H/0I') #FIXME: again, how to handle the pseudorandom content?
+    eq(request['randkey'], 'value-NI$;92@H/0I')
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
 def test_weighted_choices():
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 91356f8..64245ec 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -7,6 +7,7 @@ import traceback
 import itertools
 import random
 import string
+import struct
 import yaml
 import sys
 
@@ -82,6 +83,7 @@ def expand_key(decision, key):
 
 class SpecialVariables(dict):
     charsets = {
+        'binary': 'binary',
         'printable': string.printable,
         'punctuation': string.punctuation,
         'whitespace': string.whitespace
@@ -116,7 +118,13 @@ class SpecialVariables(dict):
             charset = self.charsets['printable']
 
         length = self.prng.randint(size_min, size_max)
-        return ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
+        if charset is 'binary':
+            num_bytes = length + 8
+            tmplist = [self.prng.getrandbits(64) for _ in xrange(num_bytes / 8)]
+            tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
+            return tmpstring[0:length]
+        else:
+            return ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely; won't do binary
 
 
 

From 3f1314f7c816be193b1bf53c419334e1552220b2 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 10 Aug 2011 15:10:24 -0700
Subject: [PATCH 10/24] S3 Fuzzer: set values can be weighted lists now

---
 s3tests/functional/test_fuzzer.py | 31 ++++++++++++++++++++++++++++---
 s3tests/fuzz_headers.py           |  4 +++-
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index dabdbee..f6fb3b5 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -52,8 +52,14 @@ def build_graph():
         },
         'choices': ['leaf']
     }
-    graph['weighted_choices'] = {
-        'set': {},
+    graph['weighted_node'] = {
+        'set': {
+            'k1': [
+                'foo',
+                '2 bar',
+                '1 baz'
+            ]
+        },
         'choices': [
             'foo',
             '2 bar',
@@ -169,7 +175,26 @@ def test_weighted_choices():
 
     choices_made = {}
     for _ in xrange(1000):
-        choice = make_choice(graph['weighted_choices']['choices'], prng)
+        choice = make_choice(graph['weighted_node']['choices'], prng)
+        if choices_made.has_key(choice):
+            choices_made[choice] += 1
+        else:
+            choices_made[choice] = 1
+
+    foo_percentage = choices_made['foo'] / 1000.0
+    bar_percentage = choices_made['bar'] / 1000.0
+    baz_percentage = choices_made['baz'] / 1000.0
+    nose.tools.assert_almost_equal(foo_percentage, 0.25, 1)
+    nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
+    nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
+
+def test_weighted_set():
+    graph = build_graph()
+    prng = random.Random(1)
+
+    choices_made = {}
+    for _ in xrange(1000):
+        choice = make_choice(graph['weighted_node']['set']['k1'], prng)
         if choices_made.has_key(choice):
             choices_made[choice] += 1
         else:
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 64245ec..d01da8c 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -36,11 +36,13 @@ def descend_graph(decision_graph, node_name, prng):
     for key in node['set']:
         if decision.has_key(key):
             raise KeyError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
-        decision[key] = node['set'][key]
+        decision[key] = make_choice(node['set'][key], prng)
     return decision
 
 
 def make_choice(choices, prng):
+    if isinstance(choices, str):
+        return choices
     weighted_choices = []
     for option in choices:
         fields = option.split(None, 1)

From d7b49713f78d4a07d6d9f49dbf944a75121cf979 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Thu, 11 Aug 2011 11:32:18 -0700
Subject: [PATCH 11/24] S3 Fuzzer: Implmented headers and made random safe

Random can sometimes include } or { which will confuse the string formatter.
Formatter escapes those values when they're doubled: }}, {{ but this
required some slight hacking to the expander.
---
 s3tests/functional/test_fuzzer.py | 62 ++++++++++++++++++++++++++++---
 s3tests/fuzz_headers.py           | 47 +++++++++++++++++++----
 2 files changed, 95 insertions(+), 14 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index f6fb3b5..d16627b 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -1,3 +1,5 @@
+import sys
+import itertools
 import nose
 import random
 import string
@@ -30,12 +32,23 @@ def build_graph():
             'key1': 'value1',
             'key2': 'value2'
         },
+        'headers': [
+            ['1-2', 'random-header-{random 5-10 printable}', '{random 20-30 punctuation}']
+        ],
         'choices': []
     }
     graph['node1'] = {
         'set': {
-            'key3': 'value3'
+            'key3': 'value3',
+            'header_val': [
+                '3 h1',
+                '2 h2',
+                'h3'
+            ]
         },
+        'headers': [
+            ['1-1', 'my-header', '{header_val}'],
+        ],
         'choices': ['leaf']
     }
     graph['node2'] = {
@@ -137,9 +150,9 @@ def test_expand_key():
     }
     decision = SpecialVariables(test_decision, prng)
 
-    randkey = expand_key(decision, 'randkey')
-    indirect = expand_key(decision, 'indirect')
-    dbl_indirect = expand_key(decision, 'dbl_indirect')
+    randkey = expand_key(decision, test_decision['randkey'])
+    indirect = expand_key(decision, test_decision['indirect'])
+    dbl_indirect = expand_key(decision, test_decision['dbl_indirect'])
 
     eq(indirect, 'value1')
     eq(dbl_indirect, 'value1')
@@ -152,7 +165,7 @@ def test_expand_loop():
         'key2': '{key1}',
     }
     decision = SpecialVariables(test_decision, prng)
-    assert_raises(RuntimeError, expand_key, decision, 'key1')
+    assert_raises(RuntimeError, expand_key, decision, test_decision['key1'])
 
 def test_expand_decision():
     graph = build_graph()
@@ -166,7 +179,7 @@ def test_expand_decision():
     eq(request['key1'], 'value1')
     eq(request['indirect_key1'], 'value1')
     eq(request['path'], '/my-readable-bucket')
-    eq(request['randkey'], 'value-NI$;92@H/0I')
+    eq(request['randkey'], 'value-cx+*~G@&uW_[OW3')
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
 def test_weighted_choices():
@@ -207,3 +220,40 @@ def test_weighted_set():
     nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
     nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
 
+def test_header_presence():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'node1', prng)
+
+    c1 = itertools.count()
+    c2 = itertools.count()
+    for header, value in decision['headers']:
+        if header == 'my-header':
+            eq(value, '{header_val}')
+            nose.tools.assert_true(next(c1) < 1)
+        elif header == 'random-header-{random 5-10 printable}':
+            eq(value, '{random 20-30 punctuation}')
+            nose.tools.assert_true(next(c2) < 2)
+        else:
+            raise KeyError('unexpected header found: %s' % header)
+
+    nose.tools.assert_true(next(c1))
+    nose.tools.assert_true(next(c2))
+
+
+
+def test_header_expansion():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'node1', prng)
+    expanded_decision = expand_decision(decision, prng)
+
+    for header, value in expanded_decision['headers']:
+        if header == 'my-header':
+            nose.tools.assert_true(value in ['h1', 'h2', 'h3'])
+        elif header.startswith('random-header-'):
+            nose.tools.assert_true(20 <= len(value) <= 30)
+            nose.tools.assert_true(string.strip(value, SpecialVariables.charsets['punctuation']) is '')
+        else:
+            raise KeyError('unexpected header found: "%s"' % header)
+
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index d01da8c..fe33b7c 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -10,6 +10,7 @@ import string
 import struct
 import yaml
 import sys
+import re
 
 
 def assemble_decision(decision_graph, prng):
@@ -32,15 +33,41 @@ def descend_graph(decision_graph, node_name, prng):
     except IndexError:
         decision = {}
 
-    #TODO: Add in headers
     for key in node['set']:
         if decision.has_key(key):
             raise KeyError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
         decision[key] = make_choice(node['set'][key], prng)
+
+    if node.has_key('headers'):
+        if not decision.has_key('headers'):
+            decision['headers'] = []
+
+        for desc in node['headers']:
+            if len(desc) == 3:
+                repetition_range = desc.pop(0)
+                try:
+                    size_min, size_max = [int(x) for x in repetition_range.split('-')]
+                except IndexError:
+                    size_min = size_max = int(repetition_range)
+            else:
+                size_min = size_max = 1
+            num_reps = prng.randint(size_min, size_max)
+            for _ in xrange(num_reps):
+                header = desc[0]
+                value = desc[1]
+                if header in [h for h, v in decision['headers']]:
+                    if not re.search('{[a-zA-Z_0-9 -]+}', header):
+                        raise KeyError("Node %s tried to add header '%s', but that header already exists!" %(node_name, header))
+                decision['headers'].append([header, value])
+
     return decision
 
 
 def make_choice(choices, prng):
+    """ Given a list of (possibly weighted) options or just a single option!,
+        choose one of the options taking weights into account and return the
+        choice
+    """
     if isinstance(choices, str):
         return choices
     weighted_choices = []
@@ -65,24 +92,27 @@ def expand_decision(decision, prng):
     """
     special_decision = SpecialVariables(decision, prng)
     for key in special_decision:
-        decision[key] = expand_key(special_decision, key)
-
+        if not key == 'headers':
+            decision[key] = expand_key(special_decision, decision[key])
+        else:
+            for header in special_decision[key]:
+                header[0] = expand_key(special_decision, header[0])
+                header[1] = expand_key(special_decision, header[1])
     return decision
 
 
-def expand_key(decision, key):
+def expand_key(decision, value):
     c = itertools.count()
     fmt = string.Formatter()
-    old = decision[key]
+    old = value
     while True:
         new = fmt.vformat(old, [], decision)
-        if new == old:
+        if new == old.replace('{{', '{').replace('}}', '}'):
             return old
         if next(c) > 5:
             raise RuntimeError
         old = new
 
-
 class SpecialVariables(dict):
     charsets = {
         'binary': 'binary',
@@ -126,7 +156,8 @@ class SpecialVariables(dict):
             tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
             return tmpstring[0:length]
         else:
-            return ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely; won't do binary
+            tmpstring = ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely; won't do binary
+            return tmpstring.replace('{', '{{').replace('}', '}}')
 
 
 

From 62bd05a39024fab03cebfda972317082d3d3dc8a Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Thu, 11 Aug 2011 12:25:13 -0700
Subject: [PATCH 12/24] S3 Fuzzer: Handle null choices

Sometimes you might want to have your current node terminate the descent or
set something to the empty string.
---
 request_decision_graph.yml        | 26 ++++------------
 s3tests/functional/test_fuzzer.py | 50 ++++++++++++++++++++++++++++++-
 s3tests/fuzz_headers.py           | 10 ++++++-
 3 files changed, 64 insertions(+), 22 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index d7b0206..0d686bb 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -5,7 +5,7 @@ start:
 
 bucket:
     set:
-        urlpath: '/{bucket}'
+        urlpath: /{bucket}
     choice:
         - bucket_get
         - bucket_put
@@ -13,24 +13,10 @@ bucket:
 
 bucket_delete:
     set:
-        method: 'DELETE'
-    choice:
-        - delete_bucket
-        - delete_bucket_policy
-        - delete_bucket_website
-
-delete_bucket:
-    set:
-        query: null
-    choice: []
-
-delete_bucket_policy:
-    set:
-        query: 'policy'
-    choice: []
-
-delete_bucket_website:
-    set:
-        query: 'website'
+        method: DELETE
+        query:
+            - null
+            - policy
+            - website
     choice: []
 
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index d16627b..fae6c90 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -65,6 +65,10 @@ def build_graph():
         },
         'choices': ['leaf']
     }
+    graph['nonexistant_child_node'] = {
+        'set': {},
+        'choices': ['leafy_greens']
+    }
     graph['weighted_node'] = {
         'set': {
             'k1': [
@@ -79,6 +83,14 @@ def build_graph():
             '1 baz'
         ]
     }
+    graph['null_choice_node'] = {
+        'set': {},
+        'choices': [None]
+    }
+    graph['weighted_null_choice_node'] = {
+        'set': {},
+        'choices': ['3 null']
+    }
     return graph
 
 
@@ -114,6 +126,12 @@ def test_descend_bad_node():
     assert_raises(KeyError, descend_graph, graph, 'bad_node', prng)
 
 
+def test_descend_nonexistant_child():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(KeyError, descend_graph, graph, 'nonexistant_child_node', prng)
+
+
 def test_SpecialVariables_dict():
     prng = random.Random(1)
     testdict = {'foo': 'bar'}
@@ -128,6 +146,7 @@ def test_SpecialVariables_binary():
 
     eq(tester['random 10-15 binary'], '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
 
+
 def test_assemble_decision():
     graph = build_graph()
     prng = random.Random(1)
@@ -140,6 +159,7 @@ def test_assemble_decision():
     eq(decision['path'], '/{bucket_readable}')
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
+
 def test_expand_key():
     prng = random.Random(1)
     test_decision = {
@@ -158,6 +178,7 @@ def test_expand_key():
     eq(dbl_indirect, 'value1')
     eq(randkey, 'value-[/pNI$;92@')
 
+
 def test_expand_loop():
     prng = random.Random(1)
     test_decision = {
@@ -167,6 +188,7 @@ def test_expand_loop():
     decision = SpecialVariables(test_decision, prng)
     assert_raises(RuntimeError, expand_key, decision, test_decision['key1'])
 
+
 def test_expand_decision():
     graph = build_graph()
     prng = random.Random(1)
@@ -182,6 +204,7 @@ def test_expand_decision():
     eq(request['randkey'], 'value-cx+*~G@&uW_[OW3')
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
+
 def test_weighted_choices():
     graph = build_graph()
     prng = random.Random(1)
@@ -201,6 +224,31 @@ def test_weighted_choices():
     nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
     nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
 
+
+def test_null_choices():
+    graph = build_graph()
+    prng = random.Random(1)
+    choice = make_choice(graph['null_choice_node']['choices'], prng)
+
+    eq(choice, '')
+
+
+def test_weighted_null_choices():
+    graph = build_graph()
+    prng = random.Random(1)
+    choice = make_choice(graph['weighted_null_choice_node']['choices'], prng)
+
+    eq(choice, '')
+
+
+def test_null_child():
+    graph = build_graph()
+    prng = random.Random(1)
+    decision = descend_graph(graph, 'null_choice_node', prng)
+
+    eq(decision, {})
+
+
 def test_weighted_set():
     graph = build_graph()
     prng = random.Random(1)
@@ -220,6 +268,7 @@ def test_weighted_set():
     nose.tools.assert_almost_equal(bar_percentage, 0.50, 1)
     nose.tools.assert_almost_equal(baz_percentage, 0.25, 1)
 
+
 def test_header_presence():
     graph = build_graph()
     prng = random.Random(1)
@@ -241,7 +290,6 @@ def test_header_presence():
     nose.tools.assert_true(next(c2))
 
 
-
 def test_header_expansion():
     graph = build_graph()
     prng = random.Random(1)
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index fe33b7c..026e947 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -29,7 +29,10 @@ def descend_graph(decision_graph, node_name, prng):
 
     try:
         choice = make_choice(node['choices'], prng)
-        decision = descend_graph(decision_graph, choice, prng)
+        if choice == '':
+            decision = {}
+        else:
+            decision = descend_graph(decision_graph, choice, prng)
     except IndexError:
         decision = {}
 
@@ -72,6 +75,9 @@ def make_choice(choices, prng):
         return choices
     weighted_choices = []
     for option in choices:
+        if option is None:
+            weighted_choices.append('')
+            continue
         fields = option.split(None, 1)
         if len(fields) == 1:
             weight = 1
@@ -79,6 +85,8 @@ def make_choice(choices, prng):
         else:
             weight = int(fields[0])
             value = fields[1]
+            if value == 'null' or value == 'None':
+                value = ''
         for _ in xrange(weight):
             weighted_choices.append(value)
 

From bb7111a0d1ed6b1a0716a7a36fa9514ef9817e38 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Thu, 11 Aug 2011 15:18:27 -0700
Subject: [PATCH 13/24] S3 Fuzzer: Write Decision Graph

Start writing the decision graph describing the fuzzer's attack surface in
earnest.
---
 request_decision_graph.yml        | 123 ++++++++++++++++++++++++++++--
 s3tests/functional/test_fuzzer.py |   1 +
 s3tests/fuzz_headers.py           |  10 ++-
 3 files changed, 124 insertions(+), 10 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index 0d686bb..5e8069f 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -1,22 +1,133 @@
 start:
     set: {}
-    choice:
+    choices:
         - bucket
 
 bucket:
     set:
         urlpath: /{bucket}
-    choice:
-        - bucket_get
-        - bucket_put
-        - bucket_delete
+    choices:
+        - 13 bucket_get
+        - 8 bucket_put
+        - 5 bucket_delete
+        - bucket_garbage
+
+garbage_method:
+    set:
+        method:
+            - {random 1-100 printable}
+            - {random 10-100 binary}
+        urlpath:
+            - /{bucket}
+            - /{bucket}/{object}
+            - {random 10-1000 binary}
+    choices:
+        - bucket_get_simple
+        - bucket_get_filtered
+        - bucket_get_uploads
+        - bucket_put_create
+        - bucket_put_versioning
+        - bucket_put_simple
 
 bucket_delete:
     set:
         method: DELETE
+        bucket:
+            - {bucket_writable}
+            - {bucket_not_writable}
+            - {random 10-3000 printable}
+            - {random 10-1000 binary}
         query:
             - null
             - policy
             - website
-    choice: []
+            - {random 2-20 printable}
+            - {random 10-1000 binary}
+    choices: []
+
+bucket_get:
+    set:
+        method: GET
+        bucket:
+            - {bucket_readable}
+            - {bucket_not_readable}
+            - {random 10-3000 printable}
+            - {random 10-1000 binary}
+    choices:
+        - 11 bucket_get_simple
+        - bucket_get_filtered
+        - bucket_get_uploads
+
+bucket_get_simple:
+    set:
+        query:
+            - acl
+            - policy
+            - location
+            - logging
+            - notification
+            - versions
+            - requestPayment
+            - versioning
+            - website
+            - {random 2-20 printable}
+            - {random 10-1000 binary}
+    choices: []
+
+bucket_get_uploads:
+    set:
+        query:
+            -
+    choices: []
+
+bucket_get_filtered:
+    set:
+        delimiter:
+            - 'delimiter={random 10-50 binary}'
+            - 'delimiter={random 1000-3000 printable}'
+        prefix:
+            - 'prefix={random 10-50 binary}'
+            - 'prefix={random 100-3000 printable}'
+        marker:
+            - 'marker={object_readable}'
+            - 'marker={object_not_readable}'
+            - 'marker={invalid_key}'
+            - 'marker={random 100-1000 printable}'
+        max_keys:
+            - 'max-keys={random 1-5 binary}'
+            - 'max-keys={random 1-1000 digits}'
+        query:
+            - null
+            - '{delimiter}&{prefix}'
+            - '{max-keys}&{marker}'
+            - {random 10-1000 printable}
+            - {random 10-1000 binary}
+    choices: []
+
+bucket_put:
+    set:
+        bucket:
+            - {bucket_writable}
+            - {bucket_not_writable}
+            - {random 10-3000 printable}
+            - {random 10-1000 binary}
+        method: PUT
+    choices:
+        - bucket_put_simple
+        - bucket_put_create
+        - bucket_put_versioning
+
+bucket_put_create:
+    set:
+        body:
+            - {random 3000 printable}
+            - {random 10-3000 binary}
+            - '<CreateBucketConfiguration><LocationConstraint>{random 2-10 binary}</LocationConstraint></CreateBucketConfiguration>'
+        acl:
+            - private
+            - {random 3000 letters}
+            - {random 100-1000 binary}
+    headers:
+        - ['0-1', 'x-amz-acl', '{acl}']
+    choices: []
 
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index fae6c90..46649ef 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -140,6 +140,7 @@ def test_SpecialVariables_dict():
     eq(tester['foo'], 'bar')
     eq(tester['random 10-15 printable'], '[/pNI$;92@')
 
+
 def test_SpecialVariables_binary():
     prng = random.Random(1)
     tester = SpecialVariables({}, prng)
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 026e947..092a53d 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -126,7 +126,8 @@ class SpecialVariables(dict):
         'binary': 'binary',
         'printable': string.printable,
         'punctuation': string.punctuation,
-        'whitespace': string.whitespace
+        'whitespace': string.whitespace,
+        'digits': string.digits
     }
 
     def __init__(self, orig_dict, prng):
@@ -168,7 +169,6 @@ class SpecialVariables(dict):
             return tmpstring.replace('{', '{{').replace('}', '}}')
 
 
-
 def parse_options():
     parser = OptionParser()
     parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
@@ -210,11 +210,13 @@ def _main():
 
     constants = {
         'bucket_readable': 'TODO',
+        'bucket_not_readable': 'TODO',
         'bucket_writable' : 'TODO',
-        'bucket_nonexistant' : 'TODO',
+        'bucket_not_writable' : 'TODO',
         'object_readable' : 'TODO',
+        'object_not_readable' : 'TODO',
         'object_writable' : 'TODO',
-        'object_nonexistant' : 'TODO'
+        'object_not_writable' : 'TODO',
     }
 
     for request_seed in request_seeds:

From 14288ad2f62ffa096ff4d9fa9d5dd968c248d893 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Mon, 15 Aug 2011 14:16:40 -0700
Subject: [PATCH 14/24] S3 Fuzzer: Incorporate Tv's suggestions

Tv looked over the fuzzer and had some idiomatic and design suggestions.

Adds several tests and reworks how expansion happens in addition to idiom
changes.
---
 s3tests/functional/test_fuzzer.py | 158 +++++++++++++++++--------
 s3tests/fuzz_headers.py           | 185 ++++++++++++++++++------------
 2 files changed, 222 insertions(+), 121 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 46649ef..4db6a45 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -8,6 +8,7 @@ import yaml
 from s3tests.fuzz_headers import *
 
 from nose.tools import eq_ as eq
+from nose.tools import assert_true
 from nose.plugins.attrib import attr
 
 from .utils import assert_raises
@@ -87,6 +88,13 @@ def build_graph():
         'set': {},
         'choices': [None]
     }
+    graph['repeated_headers_node'] = {
+        'set': {},
+        'headers': [
+            ['1-2', 'random-header-{random 5-10 printable}', '{random 20-30 punctuation}']
+        ],
+        'choices': ['leaf']
+    }
     graph['weighted_null_choice_node'] = {
         'set': {},
         'choices': ['3 null']
@@ -123,7 +131,7 @@ def test_descend_node():
 def test_descend_bad_node():
     graph = build_graph()
     prng = random.Random(1)
-    assert_raises(KeyError, descend_graph, graph, 'bad_node', prng)
+    assert_raises(DecisionGraphError, descend_graph, graph, 'bad_node', prng)
 
 
 def test_descend_nonexistant_child():
@@ -148,6 +156,46 @@ def test_SpecialVariables_binary():
     eq(tester['random 10-15 binary'], '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
 
 
+def test_SpeicalVariables_random_no_args():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    for _ in xrange(1000):
+        val = tester['random']
+        val = val.replace('{{', '{').replace('}}','}')
+        assert_true(0 <= len(val) <= 1000)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in val]))
+
+
+def test_SpeicalVariables_random_no_charset():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    for _ in xrange(1000):
+        val = tester['random 10-30']
+        val = val.replace('{{', '{').replace('}}','}')
+        assert_true(10 <= len(val) <= 30)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in val]))
+
+
+def test_SpeicalVariables_random_exact_length():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    for _ in xrange(1000):
+        val = tester['random 10 digits']
+        assert_true(len(val) == 10)
+        assert_true(reduce(lambda x, y: x and y, [x in string.digits for x in val]))
+
+
+def test_SpecialVariables_random_errors():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    assert_raises(KeyError, lambda x: tester[x], 'random 10-30 foo')
+    assert_raises(ValueError, lambda x: tester[x], 'random printable')
+
+
 def test_assemble_decision():
     graph = build_graph()
     prng = random.Random(1)
@@ -161,49 +209,56 @@ def test_assemble_decision():
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
 
-def test_expand_key():
-    prng = random.Random(1)
-    test_decision = {
-        'key1': 'value1',
-        'randkey': 'value-{random 10-15 printable}',
-        'indirect': '{key1}',
-        'dbl_indirect': '{indirect}'
-    }
-    decision = SpecialVariables(test_decision, prng)
-
-    randkey = expand_key(decision, test_decision['randkey'])
-    indirect = expand_key(decision, test_decision['indirect'])
-    dbl_indirect = expand_key(decision, test_decision['dbl_indirect'])
-
-    eq(indirect, 'value1')
-    eq(dbl_indirect, 'value1')
-    eq(randkey, 'value-[/pNI$;92@')
+def test_expand_escape():
+    decision = dict(
+        foo='{{bar}}',
+        )
+    got = expand(decision, '{foo}')
+    eq(got, '{bar}')
 
 
-def test_expand_loop():
-    prng = random.Random(1)
-    test_decision = {
-        'key1': '{key2}',
-        'key2': '{key1}',
-    }
-    decision = SpecialVariables(test_decision, prng)
-    assert_raises(RuntimeError, expand_key, decision, test_decision['key1'])
+def test_expand_indirect():
+    decision = dict(
+        foo='{bar}',
+        bar='quux',
+        )
+    got = expand(decision, '{foo}')
+    eq(got, 'quux')
 
 
-def test_expand_decision():
-    graph = build_graph()
-    prng = random.Random(1)
+def test_expand_indirect_double():
+    decision = dict(
+        foo='{bar}',
+        bar='{quux}',
+        quux='thud',
+        )
+    got = expand(decision, '{foo}')
+    eq(got, 'thud')
 
-    decision = assemble_decision(graph, prng)
-    decision.update({'bucket_readable': 'my-readable-bucket'})
 
-    request = expand_decision(decision, prng)
+def test_expand_recursive():
+    decision = dict(
+        foo='{foo}',
+        )
+    e = assert_raises(RecursionError, expand, decision, '{foo}')
+    eq(str(e), "Runaway recursion in string formatting: 'foo'")
 
-    eq(request['key1'], 'value1')
-    eq(request['indirect_key1'], 'value1')
-    eq(request['path'], '/my-readable-bucket')
-    eq(request['randkey'], 'value-cx+*~G@&uW_[OW3')
-    assert_raises(KeyError, lambda x: decision[x], 'key3')
+
+def test_expand_recursive_mutual():
+    decision = dict(
+        foo='{bar}',
+        bar='{foo}',
+        )
+    e = assert_raises(RecursionError, expand, decision, '{foo}')
+    eq(str(e), "Runaway recursion in string formatting: 'foo'")
+
+
+def test_expand_recursive_not_too_eager():
+    decision = dict(
+        foo='bar',
+        )
+    got = expand(decision, 100*'{foo}')
+    eq(got, 100*'bar')
 
 
 def test_weighted_choices():
@@ -280,29 +335,36 @@ def test_header_presence():
     for header, value in decision['headers']:
         if header == 'my-header':
             eq(value, '{header_val}')
-            nose.tools.assert_true(next(c1) < 1)
+            assert_true(next(c1) < 1)
         elif header == 'random-header-{random 5-10 printable}':
             eq(value, '{random 20-30 punctuation}')
-            nose.tools.assert_true(next(c2) < 2)
+            assert_true(next(c2) < 2)
         else:
             raise KeyError('unexpected header found: %s' % header)
 
-    nose.tools.assert_true(next(c1))
-    nose.tools.assert_true(next(c2))
+    assert_true(next(c1))
+    assert_true(next(c2))
 
 
-def test_header_expansion():
+def test_duplicate_header():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(DecisionGraphError, descend_graph, graph, 'repeated_headers_node', prng)
+
+
+def test_expand_headers():
     graph = build_graph()
     prng = random.Random(1)
     decision = descend_graph(graph, 'node1', prng)
-    expanded_decision = expand_decision(decision, prng)
+    special_decision = SpecialVariables(decision, prng)
+    expanded_headers = expand_headers(special_decision)
 
-    for header, value in expanded_decision['headers']:
+    for header, value in expanded_headers:
         if header == 'my-header':
-            nose.tools.assert_true(value in ['h1', 'h2', 'h3'])
+            assert_true(value in ['h1', 'h2', 'h3'])
         elif header.startswith('random-header-'):
-            nose.tools.assert_true(20 <= len(value) <= 30)
-            nose.tools.assert_true(string.strip(value, SpecialVariables.charsets['punctuation']) is '')
+            assert_true(20 <= len(value) <= 30)
+            assert_true(string.strip(value, SpecialVariables.charsets['punctuation']) is '')
         else:
-            raise KeyError('unexpected header found: "%s"' % header)
+            raise DecisionGraphError('unexpected header found: "%s"' % header)
 
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 092a53d..1eea13a 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -13,6 +13,27 @@ import sys
 import re
 
 
+class DecisionGraphError(Exception):
+    """ Raised when a node in a graph tries to set a header or
+        key that was previously set by another node
+    """
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class RecursionError(Exception):
+    """Runaway recursion in string formatting"""
+
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return '{0.__doc__}: {0.msg!r}'.format(self)
+
+
 def assemble_decision(decision_graph, prng):
     """ Take in a graph describing the possible decision space and a random
         number generator and traverse the graph to build a decision
@@ -36,31 +57,33 @@ def descend_graph(decision_graph, node_name, prng):
     except IndexError:
         decision = {}
 
-    for key in node['set']:
-        if decision.has_key(key):
-            raise KeyError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
-        decision[key] = make_choice(node['set'][key], prng)
+    for key, choices in node['set'].iteritems():
+        if key in decision:
+            raise DecisionGraphError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
+        decision[key] = make_choice(choices, prng)
 
-    if node.has_key('headers'):
-        if not decision.has_key('headers'):
-            decision['headers'] = []
+    if 'headers' in node:
+        decision.setdefault('headers', [])
 
         for desc in node['headers']:
-            if len(desc) == 3:
-                repetition_range = desc.pop(0)
-                try:
-                    size_min, size_max = [int(x) for x in repetition_range.split('-')]
-                except IndexError:
-                    size_min = size_max = int(repetition_range)
-            else:
-                size_min = size_max = 1
+            try:
+                (repetition_range, header, value) = desc
+            except ValueError:
+                (header, value) = desc
+                repetition_range = '1'
+
+            try:
+                size_min, size_max = repetition_range.split('-', 1)
+            except ValueError:
+                size_min = size_max = repetition_range
+
+            size_min = int(size_min)
+            size_max = int(size_max)
+
             num_reps = prng.randint(size_min, size_max)
+            if header in [h for h, v in decision['headers']]:
+                    raise DecisionGraphError("Node %s tried to add header '%s', but that header already exists!" %(node_name, header))
             for _ in xrange(num_reps):
-                header = desc[0]
-                value = desc[1]
-                if header in [h for h, v in decision['headers']]:
-                    if not re.search('{[a-zA-Z_0-9 -]+}', header):
-                        raise KeyError("Node %s tried to add header '%s', but that header already exists!" %(node_name, header))
                 decision['headers'].append([header, value])
 
     return decision
@@ -78,52 +101,59 @@ def make_choice(choices, prng):
         if option is None:
             weighted_choices.append('')
             continue
-        fields = option.split(None, 1)
-        if len(fields) == 1:
-            weight = 1
-            value = fields[0]
-        else:
-            weight = int(fields[0])
-            value = fields[1]
-            if value == 'null' or value == 'None':
-                value = ''
+        try:
+            (weight, value) = option.split(None, 1)
+        except ValueError:
+            weight = '1'
+            value = option
+
+        weight = int(weight)
+        if value == 'null' or value == 'None':
+            value = ''
+
         for _ in xrange(weight):
             weighted_choices.append(value)
 
     return prng.choice(weighted_choices)
 
 
-def expand_decision(decision, prng):
-    """ Take in a decision and a random number generator.  Expand variables in
-        decision's values and headers until all values are fully expanded and
-        build a request out of the information
-    """
-    special_decision = SpecialVariables(decision, prng)
-    for key in special_decision:
-        if not key == 'headers':
-            decision[key] = expand_key(special_decision, decision[key])
-        else:
-            for header in special_decision[key]:
-                header[0] = expand_key(special_decision, header[0])
-                header[1] = expand_key(special_decision, header[1])
-    return decision
+def expand_headers(decision):
+    expanded_headers = []
+    for header in decision['headers']:
+        h = expand(decision, header[0])
+        v = expand(decision, header[1])
+        expanded_headers.append([h, v])
+    return expanded_headers
 
 
-def expand_key(decision, value):
+def expand(decision, value):
     c = itertools.count()
-    fmt = string.Formatter()
-    old = value
-    while True:
-        new = fmt.vformat(old, [], decision)
-        if new == old.replace('{{', '{').replace('}}', '}'):
-            return old
-        if next(c) > 5:
-            raise RuntimeError
-        old = new
+    fmt = RepeatExpandingFormatter()
+    new = fmt.vformat(value, [], decision)
+    return new
+
+
+class RepeatExpandingFormatter(string.Formatter):
+
+    def __init__(self, _recursion=0):
+        super(RepeatExpandingFormatter, self).__init__()
+        # this class assumes it is always instantiated once per
+        # formatting; use that to detect runaway recursion
+        self._recursion = _recursion
+
+    def get_value(self, key, args, kwargs):
+        val = super(RepeatExpandingFormatter, self).get_value(key, args, kwargs)
+        if self._recursion > 5:
+            raise RecursionError(key)
+        fmt = self.__class__(_recursion=self._recursion+1)
+        # must use vformat not **kwargs so our SpecialVariables is not
+        # downgraded to just a dict
+        n = fmt.vformat(val, args, kwargs)
+        return n
+
 
 class SpecialVariables(dict):
     charsets = {
-        'binary': 'binary',
         'printable': string.printable,
         'punctuation': string.punctuation,
         'whitespace': string.whitespace,
@@ -131,7 +161,7 @@ class SpecialVariables(dict):
     }
 
     def __init__(self, orig_dict, prng):
-        self.update(orig_dict)
+        super(SpecialVariables, self).__init__(orig_dict)
         self.prng = prng
 
 
@@ -142,31 +172,39 @@ class SpecialVariables(dict):
             return super(SpecialVariables, self).__getitem__(key)
 
         if len(fields) == 1:
-            fields.apppend('')
+            fields.append('')
         return fn(fields[1])
 
 
     def special_random(self, args):
         arg_list = args.split()
         try:
-            size_min, size_max = [int(x) for x in arg_list[0].split('-')]
+            size_min, size_max = arg_list[0].split('-', 1)
+        except ValueError:
+            size_min = size_max = arg_list[0]
         except IndexError:
-            size_min = 0
-            size_max = 1000
-        try:
-            charset = self.charsets[arg_list[1]]
-        except IndexError:
-            charset = self.charsets['printable']
+            size_min = '0'
+            size_max = '1000'
 
+        size_min = int(size_min)
+        size_max = int(size_max)
         length = self.prng.randint(size_min, size_max)
-        if charset is 'binary':
+
+        try:
+            charset_arg = arg_list[1]
+        except IndexError:
+            charset_arg = 'printable'
+
+        if charset_arg == 'binary':
             num_bytes = length + 8
             tmplist = [self.prng.getrandbits(64) for _ in xrange(num_bytes / 8)]
             tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
-            return tmpstring[0:length]
+            tmpstring = tmpstring[0:length]
         else:
-            tmpstring = ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely; won't do binary
-            return tmpstring.replace('{', '{{').replace('}', '}}')
+            charset = self.charsets[charset_arg]
+            tmpstring = ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
+
+        return tmpstring.replace('{', '{{').replace('}', '}}')
 
 
 def parse_options():
@@ -182,12 +220,11 @@ def parse_options():
     return parser.parse_args()
 
 
-def randomlist(n, seed=None):
-    """ Returns a generator function that spits out a list of random numbers n elements long.
+def randomlist(seed=None):
+    """ Returns an infinite generator of random numbers
     """
-    rng = random.Random()
-    rng.seed(seed if seed else None)
-    for _ in xrange(n):
+    rng = random.Random(seed)
+    while True:
         yield rng.random()
 
 
@@ -203,7 +240,9 @@ def _main():
         FH = open(options.seedfile, 'r')
         request_seeds = FH.readlines()
     else:
-        request_seeds = randomlist(options.num_requests, options.seed)
+        random_list = randomlist(options.seed)
+        request_seeds = itertools.islice(random_list, options.num_requests)
+
 
     graph_file = open(options.graph_filename, 'r')
     decision_graph = yaml.safe_load(graph_file)

From bfca00ac4cbf42d1fa1fd2e8f862daedc3331161 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 17 Aug 2011 17:06:53 -0700
Subject: [PATCH 15/24] S3 Fuzzer: start writing nodes for bucket PUTs

Continue working on decision graph; add PUT requests for buckets.
---
 request_decision_graph.yml | 67 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index 5e8069f..39f3a61 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -131,3 +131,70 @@ bucket_put_create:
         - ['0-1', 'x-amz-acl', '{acl}']
     choices: []
 
+bucket_put_versioning:
+    set:
+        body:
+            - {random 3000 printable}
+            - {random 10-3000 binary}
+            - 8 '<VersioningConfiguration>{versioning_status}{mfa_delete_body}</VersioningConfiguration>'
+        mfa_delete_body:
+            - null
+            - '<Status>{random 2-10 binary}</Status>'
+            - '<Status>{random 2000-3000 printable}</Status>'
+        versioning_status:
+            - null
+            - '<MfaDelete>{random 2-10 binary}</MfaDelete>'
+            - '<MfaDelete>{random 2000-3000 printable}</MfaDelete>'
+        mfa_header:
+            - {random 10-1000 printable} {random 10-1000 printable}
+    headers:
+        - ['0-1', 'x-amz-mfa', '{mfa_header}']
+    choices: []
+
+bucket_put_simple:
+    set:
+        body:
+            - {acl_body}
+            - {policy_body}
+            - {logging_body}
+            - {notification_body}
+            - {request_payment_body}
+            - {website_body}
+        acl_body:
+            - '<AccessControlPolicy>{owner}{acl}</AccessControlPolicy>'
+        owner:
+            - null
+            - 7 '<Owner>{id}{display_name}</Owner>'
+        id:
+            - null
+            - '<ID>{random 10-200 binary}</ID>'
+            - '<ID>{random 1000-3000 printable}</ID>'
+        display_name:
+            - null
+            - '<DisplayName>{random 10-200 binary}</DisplayName>'
+            - '<DisplayName>{random 1000-3000 printable}</DisplayName>'
+            - '<DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
+        acl:
+            - null
+            - 10 '<AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
+        grantee:
+            - null
+            - 7 '<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">{id}{display_name}</Grantee>'
+        permission:
+            - null
+            - 7 '<Permission>{permission_value}</Permission>'
+        permission_value:
+            - {random 10-100 binary}
+            - {random 1000-2000 printable}
+            - FULL_CONTROL
+            - WRITE
+            - WRITE_ACP
+            - READ
+            - READ_ACP
+        policy_body: TODO
+        logging_body: TODO
+        notification_body: TODO
+        request_payment_body: TODO
+        website_body: TODO
+    choices: []
+

From e12f124686e4c171cdf9e455947936ac2272bf9f Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Thu, 18 Aug 2011 12:34:56 -0700
Subject: [PATCH 16/24] S3 Fuzzer: Change how random data works

Remove SpecialVariables dict subclass in favor of RepeatExpandingFormatter
string.Formatter subclass.
---
 s3tests/functional/test_fuzzer.py | 81 +++++++++++++++----------------
 s3tests/fuzz_headers.py           | 71 ++++++++++++---------------
 2 files changed, 69 insertions(+), 83 deletions(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 4db6a45..12b19bb 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -140,60 +140,50 @@ def test_descend_nonexistant_child():
     assert_raises(KeyError, descend_graph, graph, 'nonexistant_child_node', prng)
 
 
-def test_SpecialVariables_dict():
+def test_expand_random_printable():
     prng = random.Random(1)
-    testdict = {'foo': 'bar'}
-    tester = SpecialVariables(testdict, prng)
-
-    eq(tester['foo'], 'bar')
-    eq(tester['random 10-15 printable'], '[/pNI$;92@')
+    got = expand({}, '{random 10-15 printable}', prng)
+    eq(got, '[/pNI$;92@')
 
 
-def test_SpecialVariables_binary():
+def test_expand_random_binary():
     prng = random.Random(1)
-    tester = SpecialVariables({}, prng)
-
-    eq(tester['random 10-15 binary'], '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
+    got = expand({}, '{random 10-15 binary}', prng)
+    eq(got, '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
 
 
-def test_SpeicalVariables_random_no_args():
+def test_expand_random_no_args():
     prng = random.Random(1)
-    tester = SpecialVariables({}, prng)
-
     for _ in xrange(1000):
-        val = tester['random']
-        val = val.replace('{{', '{').replace('}}','}')
-        assert_true(0 <= len(val) <= 1000)
-        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in val]))
+        got = expand({}, '{random}', prng)
+        assert_true(0 <= len(got) <= 1000)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in got]))
 
 
-def test_SpeicalVariables_random_no_charset():
+def test_expand_random_no_charset():
     prng = random.Random(1)
-    tester = SpecialVariables({}, prng)
-
     for _ in xrange(1000):
-        val = tester['random 10-30']
-        val = val.replace('{{', '{').replace('}}','}')
-        assert_true(10 <= len(val) <= 30)
-        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in val]))
+        got = expand({}, '{random 10-30}', prng)
+        assert_true(10 <= len(got) <= 30)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in got]))
 
 
-def test_SpeicalVariables_random_exact_length():
+def test_expand_random_exact_length():
     prng = random.Random(1)
-    tester = SpecialVariables({}, prng)
-
     for _ in xrange(1000):
-        val = tester['random 10 digits']
-        assert_true(len(val) == 10)
-        assert_true(reduce(lambda x, y: x and y, [x in string.digits for x in val]))
+        got = expand({}, '{random 10 digits}', prng)
+        assert_true(len(got) == 10)
+        assert_true(reduce(lambda x, y: x and y, [x in string.digits for x in got]))
 
 
-def test_SpecialVariables_random_errors():
+def test_expand_random_bad_charset():
     prng = random.Random(1)
-    tester = SpecialVariables({}, prng)
+    assert_raises(KeyError, expand, {}, '{random 10-30 foo}', prng)
 
-    assert_raises(KeyError, lambda x: tester[x], 'random 10-30 foo')
-    assert_raises(ValueError, lambda x: tester[x], 'random printable')
+
+def test_expand_random_missing_length():
+    prng = random.Random(1)
+    assert_raises(ValueError, expand, {}, '{random printable}', prng)
 
 
 def test_assemble_decision():
@@ -210,54 +200,60 @@ def test_assemble_decision():
 
 
 def test_expand_escape():
+    prng = random.Random(1)
     decision = dict(
         foo='{{bar}}',
         )
-    got = expand(decision, '{foo}')
+    got = expand(decision, '{foo}', prng)
     eq(got, '{bar}')
 
 
 def test_expand_indirect():
+    prng = random.Random(1)
     decision = dict(
         foo='{bar}',
         bar='quux',
         )
-    got = expand(decision, '{foo}')
+    got = expand(decision, '{foo}', prng)
     eq(got, 'quux')
 
 
 def test_expand_indirect_double():
+    prng = random.Random(1)
     decision = dict(
         foo='{bar}',
         bar='{quux}',
         quux='thud',
         )
-    got = expand(decision, '{foo}')
+    got = expand(decision, '{foo}', prng)
     eq(got, 'thud')
 
 
 def test_expand_recursive():
+    prng = random.Random(1)
     decision = dict(
         foo='{foo}',
         )
-    e = assert_raises(RecursionError, expand, decision, '{foo}')
+    e = assert_raises(RecursionError, expand, decision, '{foo}', prng)
     eq(str(e), "Runaway recursion in string formatting: 'foo'")
 
 
 def test_expand_recursive_mutual():
+    prng = random.Random(1)
     decision = dict(
         foo='{bar}',
         bar='{foo}',
         )
-    e = assert_raises(RecursionError, expand, decision, '{foo}')
+    e = assert_raises(RecursionError, expand, decision, '{foo}', prng)
     eq(str(e), "Runaway recursion in string formatting: 'foo'")
 
 
 def test_expand_recursive_not_too_eager():
+    prng = random.Random(1)
     decision = dict(
         foo='bar',
         )
-    got = expand(decision, 100*'{foo}')
+    got = expand(decision, 100*'{foo}', prng)
     eq(got, 100*'bar')
 
 
@@ -356,15 +352,14 @@ def test_expand_headers():
     graph = build_graph()
     prng = random.Random(1)
     decision = descend_graph(graph, 'node1', prng)
-    special_decision = SpecialVariables(decision, prng)
-    expanded_headers = expand_headers(special_decision)
+    expanded_headers = expand_headers(decision, prng)
 
     for header, value in expanded_headers:
         if header == 'my-header':
             assert_true(value in ['h1', 'h2', 'h3'])
         elif header.startswith('random-header-'):
             assert_true(20 <= len(value) <= 30)
-            assert_true(string.strip(value, SpecialVariables.charsets['punctuation']) is '')
+            assert_true(string.strip(value, RepeatExpandingFormatter.charsets['punctuation']) is '')
         else:
             raise DecisionGraphError('unexpected header found: "%s"' % header)
 
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 1eea13a..4e7a820 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -117,42 +117,23 @@ def make_choice(choices, prng):
     return prng.choice(weighted_choices)
 
 
-def expand_headers(decision):
+def expand_headers(decision, prng):
     expanded_headers = []
     for header in decision['headers']:
-        h = expand(decision, header[0])
-        v = expand(decision, header[1])
+        h = expand(decision, header[0], prng)
+        v = expand(decision, header[1], prng)
         expanded_headers.append([h, v])
     return expanded_headers
 
 
-def expand(decision, value):
+def expand(decision, value, prng):
     c = itertools.count()
-    fmt = RepeatExpandingFormatter()
+    fmt = RepeatExpandingFormatter(prng)
     new = fmt.vformat(value, [], decision)
     return new
 
 
 class RepeatExpandingFormatter(string.Formatter):
-
-    def __init__(self, _recursion=0):
-        super(RepeatExpandingFormatter, self).__init__()
-        # this class assumes it is always instantiated once per
-        # formatting; use that to detect runaway recursion
-        self._recursion = _recursion
-
-    def get_value(self, key, args, kwargs):
-        val = super(RepeatExpandingFormatter, self).get_value(key, args, kwargs)
-        if self._recursion > 5:
-            raise RecursionError(key)
-        fmt = self.__class__(_recursion=self._recursion+1)
-        # must use vformat not **kwargs so our SpecialVariables is not
-        # downgraded to just a dict
-        n = fmt.vformat(val, args, kwargs)
-        return n
-
-
-class SpecialVariables(dict):
     charsets = {
         'printable': string.printable,
         'punctuation': string.punctuation,
@@ -160,21 +141,29 @@ class SpecialVariables(dict):
         'digits': string.digits
     }
 
-    def __init__(self, orig_dict, prng):
-        super(SpecialVariables, self).__init__(orig_dict)
+    def __init__(self, prng, _recursion=0):
+        super(RepeatExpandingFormatter, self).__init__()
+        # this class assumes it is always instantiated once per
+        # formatting; use that to detect runaway recursion
         self.prng = prng
+        self._recursion = _recursion
 
-
-    def __getitem__(self, key):
+    def get_value(self, key, args, kwargs):
         fields = key.split(None, 1)
         fn = getattr(self, 'special_{name}'.format(name=fields[0]), None)
-        if fn is None:
-            return super(SpecialVariables, self).__getitem__(key)
-
-        if len(fields) == 1:
-            fields.append('')
-        return fn(fields[1])
+        if fn is not None:
+            if len(fields) == 1:
+                fields.append('')
+            return fn(fields[1])
 
+        val = super(RepeatExpandingFormatter, self).get_value(key, args, kwargs)
+        if self._recursion > 5:
+            raise RecursionError(key)
+        fmt = self.__class__(self.prng, _recursion=self._recursion+1)
+        # must use vformat not **kwargs so our SpecialVariables is not
+        # downgraded to just a dict
+        n = fmt.vformat(val, args, kwargs)
+        return n
 
     def special_random(self, args):
         arg_list = args.split()
@@ -199,12 +188,10 @@ class SpecialVariables(dict):
             num_bytes = length + 8
             tmplist = [self.prng.getrandbits(64) for _ in xrange(num_bytes / 8)]
             tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
-            tmpstring = tmpstring[0:length]
+            return tmpstring[0:length]
         else:
             charset = self.charsets[charset_arg]
-            tmpstring = ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
-
-        return tmpstring.replace('{', '{{').replace('}', '}}')
+            return ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
 
 
 def parse_options():
@@ -262,9 +249,13 @@ def _main():
         prng = random.Random(request_seed)
         decision = assemble_decision(decision_graph, prng)
         decision.update(constants)
-        request = expand_decision(decision, prng) 
 
-        response = s3_connection.make_request(request['method'], request['path'], data=request['body'], headers=request['headers'], override_num_retries=0)
+        method = expand(decision, decision['method'], prng)
+        path = expand(decision, decision['path'], prng)
+        body = expand(decision, decision['body'], prng)
+        headers = expand_headers(decision, prng)
+
+        response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
 
         if response.status == 500 or response.status == 503:
             print 'Request generated with seed %d failed:\n%s' % (request_seed, request)

From 76956d86e4d0d5325693fc0edf6f18020d0217ed Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Thu, 18 Aug 2011 14:11:54 -0700
Subject: [PATCH 17/24] S3 Fuzzer: finish describing bucket operations

Finishing touches on decision graph for bucket operations.
---
 request_decision_graph.yml | 122 ++++++++++++++++++++++++++-----------
 s3tests/fuzz_headers.py    |  27 ++++----
 2 files changed, 103 insertions(+), 46 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index 39f3a61..e08155c 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -1,5 +1,8 @@
 start:
-    set: {}
+    set:
+        garbage:
+            - {random 10-3000 printable}
+            - {random 10-1000 binary}
     choices:
         - bucket
 
@@ -10,7 +13,7 @@ bucket:
         - 13 bucket_get
         - 8 bucket_put
         - 5 bucket_delete
-        - bucket_garbage
+        - garbage_method
 
 garbage_method:
     set:
@@ -35,14 +38,12 @@ bucket_delete:
         bucket:
             - {bucket_writable}
             - {bucket_not_writable}
-            - {random 10-3000 printable}
-            - {random 10-1000 binary}
+            - 2 {garbage}
         query:
             - null
             - policy
             - website
-            - {random 2-20 printable}
-            - {random 10-1000 binary}
+            - 2 {garbage}
     choices: []
 
 bucket_get:
@@ -51,8 +52,7 @@ bucket_get:
         bucket:
             - {bucket_readable}
             - {bucket_not_readable}
-            - {random 10-3000 printable}
-            - {random 10-1000 binary}
+            - 2 {garbage}
     choices:
         - 11 bucket_get_simple
         - bucket_get_filtered
@@ -70,24 +70,43 @@ bucket_get_simple:
             - requestPayment
             - versioning
             - website
-            - {random 2-20 printable}
-            - {random 10-1000 binary}
+            - 2 {garbage}
     choices: []
 
 bucket_get_uploads:
     set:
+        delimiter:
+            - null
+            - 3 'delimiter={garbage}'
+        prefix:
+            - null
+            - 3 'prefix={garbage}'
+        key_marker:
+            - null
+            - 'key-marker={object_readable}'
+            - 'key-marker={object_not_readable}'
+            - 'key-marker={invalid_key}'
+            - 'key-marker={random 100-1000 printable}'
+        max_uploads:
+            - null
+            - 'max-uploads={random 1-5 binary}'
+            - 'max-uploads={random 1-1000 digits}'
+        upload_id_marker:
+            - null
+            - 3 'upload-id-marker={random}'
         query:
-            -
+            - 'uploads'
+            - 'uploads&{delimiter}&{prefix}'
+            - 'uploads&{max_uploads}&{key_marker}&{upload_id_marker}'
+            - 2 {garbage}
     choices: []
 
 bucket_get_filtered:
     set:
         delimiter:
-            - 'delimiter={random 10-50 binary}'
-            - 'delimiter={random 1000-3000 printable}'
+            - 'delimiter={garbage}'
         prefix:
-            - 'prefix={random 10-50 binary}'
-            - 'prefix={random 100-3000 printable}'
+            - 'prefix={garbage}'
         marker:
             - 'marker={object_readable}'
             - 'marker={object_not_readable}'
@@ -100,8 +119,7 @@ bucket_get_filtered:
             - null
             - '{delimiter}&{prefix}'
             - '{max-keys}&{marker}'
-            - {random 10-1000 printable}
-            - {random 10-1000 binary}
+            - 2 {garbage}
     choices: []
 
 bucket_put:
@@ -109,8 +127,7 @@ bucket_put:
         bucket:
             - {bucket_writable}
             - {bucket_not_writable}
-            - {random 10-3000 printable}
-            - {random 10-1000 binary}
+            - 2 {garbage}
         method: PUT
     choices:
         - bucket_put_simple
@@ -120,8 +137,7 @@ bucket_put:
 bucket_put_create:
     set:
         body:
-            - {random 3000 printable}
-            - {random 10-3000 binary}
+            - 2 {garbage}
             - '<CreateBucketConfiguration><LocationConstraint>{random 2-10 binary}</LocationConstraint></CreateBucketConfiguration>'
         acl:
             - private
@@ -134,9 +150,8 @@ bucket_put_create:
 bucket_put_versioning:
     set:
         body:
-            - {random 3000 printable}
-            - {random 10-3000 binary}
-            - 8 '<VersioningConfiguration>{versioning_status}{mfa_delete_body}</VersioningConfiguration>'
+            - {garbage}
+            - 4 '<VersioningConfiguration>{versioning_status}{mfa_delete_body}</VersioningConfiguration>'
         mfa_delete_body:
             - null
             - '<Status>{random 2-10 binary}</Status>'
@@ -146,7 +161,7 @@ bucket_put_versioning:
             - '<MfaDelete>{random 2-10 binary}</MfaDelete>'
             - '<MfaDelete>{random 2000-3000 printable}</MfaDelete>'
         mfa_header:
-            - {random 10-1000 printable} {random 10-1000 printable}
+            - '{random 10-1000 printable} {random 10-1000 printable}'
     headers:
         - ['0-1', 'x-amz-mfa', '{mfa_header}']
     choices: []
@@ -161,6 +176,7 @@ bucket_put_simple:
             - {request_payment_body}
             - {website_body}
         acl_body:
+            - null
             - '<AccessControlPolicy>{owner}{acl}</AccessControlPolicy>'
         owner:
             - null
@@ -171,9 +187,9 @@ bucket_put_simple:
             - '<ID>{random 1000-3000 printable}</ID>'
         display_name:
             - null
-            - '<DisplayName>{random 10-200 binary}</DisplayName>'
-            - '<DisplayName>{random 1000-3000 printable}</DisplayName>'
-            - '<DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
+            - 2 '<DisplayName>{random 10-200 binary}</DisplayName>'
+            - 2 '<DisplayName>{random 1000-3000 printable}</DisplayName>'
+            - 2 '<DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
         acl:
             - null
             - 10 '<AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
@@ -184,17 +200,53 @@ bucket_put_simple:
             - null
             - 7 '<Permission>{permission_value}</Permission>'
         permission_value:
-            - {random 10-100 binary}
-            - {random 1000-2000 printable}
+            - 2 {garbage}
             - FULL_CONTROL
             - WRITE
             - WRITE_ACP
             - READ
             - READ_ACP
-        policy_body: TODO
-        logging_body: TODO
-        notification_body: TODO
-        request_payment_body: TODO
-        website_body: TODO
+        policy_body:
+            - null
+            - 2 {garbage}
+        logging_body:
+            - null
+            - '<BucketLoggingStatus xmlns="http://doc.s3.amazonaws.com/2006-03-01" />'
+            - '<BucketLoggingStatus xmlns="http://doc.s3.amazonaws.com/2006-03-01"><LoggingEnabled>{bucket}{target_prefix}{target_grants}</LoggingEnabled></BucketLoggingStatus>'
+        target_prefix:
+            - null
+            - '<TargetPrefix>{random 10-1000 printable}</TargetPrefix>'
+            - '<TargetPrefix>{random 10-1000 binary}</TargetPrefix>'
+        target_grants:
+            - null
+            - 10 '<TargetGrants><Grant>{grantee}{permission}</Grant></TargetGrants>'
+        notification_body:
+            - null
+            - '<NotificationConfiguration />'
+            - 2 '<NotificationConfiguration><TopicConfiguration>{topic}{event}</TopicConfiguration}</NotificationConfiguration>'
+        topic:
+            - null
+            - 2 '<Topic>{garbage}</Topic>'
+        event:
+            - null
+            - '<Event>s3:ReducedRedundancyLostObject</Event>'
+            - 2 '<Event>{garbage}</Event>'
+        request_payment_body:
+            - null
+            - '<RequestPaymentConfiguration xlmns="http://s3.amazonaws.com/doc/2006-03-01/"><Payer>{payer}</Payer></RequestPaymentConfiguration>'
+        payer:
+            - Requester
+            - BucketOwner
+            - 2 {garbage}
+        website_body:
+            - null
+            - '<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IndexDocument><Suffix>{suffix}</Suffix><IndexDocument>{error_doc}<WebsiteConfiguration/>'
+        suffix:
+            - null
+            - 2 {garbage}
+            - '{random 2-10 printable}.html'
+        error_doc:
+            - null
+            - '<ErrorDocument><Key>{suffix}</Key></ErrorDocument>'
     choices: []
 
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 4e7a820..3796fd1 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -234,16 +234,16 @@ def _main():
     graph_file = open(options.graph_filename, 'r')
     decision_graph = yaml.safe_load(graph_file)
 
-    constants = {
-        'bucket_readable': 'TODO',
-        'bucket_not_readable': 'TODO',
-        'bucket_writable' : 'TODO',
-        'bucket_not_writable' : 'TODO',
-        'object_readable' : 'TODO',
-        'object_not_readable' : 'TODO',
-        'object_writable' : 'TODO',
-        'object_not_writable' : 'TODO',
-    }
+    constants = dict(
+        bucket_readable='TODO-breadable',
+        bucket_not_readable='TODO-bnonreadable',
+        bucket_writable='TODO-bwritable',
+        bucket_not_writable='TODO-bnonwritable',
+        object_readable='TODO-oreadable',
+        object_not_readable='TODO-ononreadable',
+        object_writable='TODO-owritable',
+        object_not_writable='TODO-ononwritable',
+    )
 
     for request_seed in request_seeds:
         prng = random.Random(request_seed)
@@ -255,7 +255,12 @@ def _main():
         body = expand(decision, decision['body'], prng)
         headers = expand_headers(decision, prng)
 
-        response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+        print "Method: %s" % method
+        print "Path: %s" % path
+        print "Headers: %s" % headers
+        print ""
+        print "Body: %s" % body
+        #response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
 
         if response.status == 500 or response.status == 503:
             print 'Request generated with seed %d failed:\n%s' % (request_seed, request)

From 23fee1476ae8809f3a4c9fa4ca52dd19c3a02d7b Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Fri, 19 Aug 2011 14:54:24 -0700
Subject: [PATCH 18/24] S3 Fuzzer: Start Fuzzing

- tweak yaml decision graph
- add test setup bucket creation, etc.
- add output with varying levels of verbosity
---
 request_decision_graph.yml        | 100 ++++++++++++------------
 s3tests/functional/test_fuzzer.py |  13 ++-
 s3tests/fuzz_headers.py           | 126 ++++++++++++++++++++++++------
 3 files changed, 162 insertions(+), 77 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index e08155c..443b37c 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -1,14 +1,14 @@
 start:
     set:
         garbage:
-            - {random 10-3000 printable}
-            - {random 10-1000 binary}
+            - '{random 10-3000 printable}'
+            - '{random 10-1000 binary}'
     choices:
         - bucket
 
 bucket:
     set:
-        urlpath: /{bucket}
+        urlpath: '/{bucket}'
     choices:
         - 13 bucket_get
         - 8 bucket_put
@@ -18,12 +18,12 @@ bucket:
 garbage_method:
     set:
         method:
-            - {random 1-100 printable}
-            - {random 10-100 binary}
+            - '{random 1-100 printable}'
+            - '{random 10-100 binary}'
         urlpath:
-            - /{bucket}
-            - /{bucket}/{object}
-            - {random 10-1000 binary}
+            - '/{bucket}'
+            - '/{bucket}/{object}'
+            - '{random 10-1000 binary}'
     choices:
         - bucket_get_simple
         - bucket_get_filtered
@@ -36,23 +36,23 @@ bucket_delete:
     set:
         method: DELETE
         bucket:
-            - {bucket_writable}
-            - {bucket_not_writable}
-            - 2 {garbage}
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage}'
         query:
             - null
             - policy
             - website
-            - 2 {garbage}
+            - '2 {garbage}'
     choices: []
 
 bucket_get:
     set:
         method: GET
         bucket:
-            - {bucket_readable}
-            - {bucket_not_readable}
-            - 2 {garbage}
+            - '{bucket_readable}'
+            - '{bucket_not_readable}'
+            - '2 {garbage}'
     choices:
         - 11 bucket_get_simple
         - bucket_get_filtered
@@ -70,17 +70,17 @@ bucket_get_simple:
             - requestPayment
             - versioning
             - website
-            - 2 {garbage}
+            - '2 {garbage}'
     choices: []
 
 bucket_get_uploads:
     set:
         delimiter:
             - null
-            - 3 'delimiter={garbage}'
+            - '3 delimiter={garbage}'
         prefix:
             - null
-            - 3 'prefix={garbage}'
+            - '3 prefix={garbage}'
         key_marker:
             - null
             - 'key-marker={object_readable}'
@@ -93,12 +93,12 @@ bucket_get_uploads:
             - 'max-uploads={random 1-1000 digits}'
         upload_id_marker:
             - null
-            - 3 'upload-id-marker={random}'
+            - '3 upload-id-marker={random}'
         query:
             - 'uploads'
             - 'uploads&{delimiter}&{prefix}'
             - 'uploads&{max_uploads}&{key_marker}&{upload_id_marker}'
-            - 2 {garbage}
+            - '2 {garbage}'
     choices: []
 
 bucket_get_filtered:
@@ -119,15 +119,15 @@ bucket_get_filtered:
             - null
             - '{delimiter}&{prefix}'
             - '{max-keys}&{marker}'
-            - 2 {garbage}
+            - '2 {garbage}'
     choices: []
 
 bucket_put:
     set:
         bucket:
-            - {bucket_writable}
-            - {bucket_not_writable}
-            - 2 {garbage}
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage}'
         method: PUT
     choices:
         - bucket_put_simple
@@ -137,12 +137,12 @@ bucket_put:
 bucket_put_create:
     set:
         body:
-            - 2 {garbage}
+            - '2 {garbage}'
             - '<CreateBucketConfiguration><LocationConstraint>{random 2-10 binary}</LocationConstraint></CreateBucketConfiguration>'
         acl:
             - private
-            - {random 3000 letters}
-            - {random 100-1000 binary}
+            - '{random 3000 letters}'
+            - '{random 100-1000 binary}'
     headers:
         - ['0-1', 'x-amz-acl', '{acl}']
     choices: []
@@ -150,8 +150,8 @@ bucket_put_create:
 bucket_put_versioning:
     set:
         body:
-            - {garbage}
-            - 4 '<VersioningConfiguration>{versioning_status}{mfa_delete_body}</VersioningConfiguration>'
+            - '{garbage}'
+            - '4 <VersioningConfiguration>{versioning_status}{mfa_delete_body}</VersioningConfiguration>'
         mfa_delete_body:
             - null
             - '<Status>{random 2-10 binary}</Status>'
@@ -169,38 +169,38 @@ bucket_put_versioning:
 bucket_put_simple:
     set:
         body:
-            - {acl_body}
-            - {policy_body}
-            - {logging_body}
-            - {notification_body}
-            - {request_payment_body}
-            - {website_body}
+            - '{acl_body}'
+            - '{policy_body}'
+            - '{logging_body}'
+            - '{notification_body}'
+            - '{request_payment_body}'
+            - '{website_body}'
         acl_body:
             - null
             - '<AccessControlPolicy>{owner}{acl}</AccessControlPolicy>'
         owner:
             - null
-            - 7 '<Owner>{id}{display_name}</Owner>'
+            - '7 <Owner>{id}{display_name}</Owner>'
         id:
             - null
             - '<ID>{random 10-200 binary}</ID>'
             - '<ID>{random 1000-3000 printable}</ID>'
         display_name:
             - null
-            - 2 '<DisplayName>{random 10-200 binary}</DisplayName>'
-            - 2 '<DisplayName>{random 1000-3000 printable}</DisplayName>'
-            - 2 '<DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
+            - '2 <DisplayName>{random 10-200 binary}</DisplayName>'
+            - '2 <DisplayName>{random 1000-3000 printable}</DisplayName>'
+            - '2 <DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
         acl:
             - null
-            - 10 '<AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
+            - '10 <AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
         grantee:
             - null
-            - 7 '<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">{id}{display_name}</Grantee>'
+            - '7 <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">{id}{display_name}</Grantee>'
         permission:
             - null
-            - 7 '<Permission>{permission_value}</Permission>'
+            - '7 <Permission>{permission_value}</Permission>'
         permission_value:
-            - 2 {garbage}
+            - '2 {garbage}'
             - FULL_CONTROL
             - WRITE
             - WRITE_ACP
@@ -208,7 +208,7 @@ bucket_put_simple:
             - READ_ACP
         policy_body:
             - null
-            - 2 {garbage}
+            - '2 {garbage}'
         logging_body:
             - null
             - '<BucketLoggingStatus xmlns="http://doc.s3.amazonaws.com/2006-03-01" />'
@@ -219,31 +219,31 @@ bucket_put_simple:
             - '<TargetPrefix>{random 10-1000 binary}</TargetPrefix>'
         target_grants:
             - null
-            - 10 '<TargetGrants><Grant>{grantee}{permission}</Grant></TargetGrants>'
+            - '10 <TargetGrants><Grant>{grantee}{permission}</Grant></TargetGrants>'
         notification_body:
             - null
             - '<NotificationConfiguration />'
-            - 2 '<NotificationConfiguration><TopicConfiguration>{topic}{event}</TopicConfiguration}</NotificationConfiguration>'
+            - '2 <NotificationConfiguration><TopicConfiguration>{topic}{event}</TopicConfiguration}</NotificationConfiguration>'
         topic:
             - null
-            - 2 '<Topic>{garbage}</Topic>'
+            - '2 <Topic>{garbage}</Topic>'
         event:
             - null
             - '<Event>s3:ReducedRedundancyLostObject</Event>'
-            - 2 '<Event>{garbage}</Event>'
+            - '2 <Event>{garbage}</Event>'
         request_payment_body:
             - null
             - '<RequestPaymentConfiguration xlmns="http://s3.amazonaws.com/doc/2006-03-01/"><Payer>{payer}</Payer></RequestPaymentConfiguration>'
         payer:
             - Requester
             - BucketOwner
-            - 2 {garbage}
+            - '2 {garbage}'
         website_body:
             - null
             - '<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IndexDocument><Suffix>{suffix}</Suffix><IndexDocument>{error_doc}<WebsiteConfiguration/>'
         suffix:
             - null
-            - 2 {garbage}
+            - '2 {garbage}'
             - '{random 2-10 printable}.html'
         error_doc:
             - null
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 12b19bb..eb2ab1f 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -102,6 +102,12 @@ def build_graph():
     return graph
 
 
+#def test_foo():
+    #graph_file = open('request_decision_graph.yml', 'r')
+    #graph = yaml.safe_load(graph_file)
+    #eq(graph['bucket_put_simple']['set']['grantee'], 0)
+
+
 def test_load_graph():
     graph_file = open('request_decision_graph.yml', 'r')
     graph = yaml.safe_load(graph_file)
@@ -257,6 +263,11 @@ def test_expand_recursive_not_too_eager():
     eq(got, 100*'bar')
 
 
+def test_make_choice_unweighted_with_space():
+    prng = random.Random(1)
+    choice = make_choice(['foo bar'], prng)
+    eq(choice, 'foo bar')
+
 def test_weighted_choices():
     graph = build_graph()
     prng = random.Random(1)
@@ -354,7 +365,7 @@ def test_expand_headers():
     decision = descend_graph(graph, 'node1', prng)
     expanded_headers = expand_headers(decision, prng)
 
-    for header, value in expanded_headers:
+    for header, value in expanded_headers.iteritems():
         if header == 'my-header':
             assert_true(value in ['h1', 'h2', 'h3'])
         elif header.startswith('random-header-'):
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 3796fd1..c23ebc6 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -1,4 +1,5 @@
 from boto.s3.connection import S3Connection
+from boto.s3.key import Key
 from optparse import OptionParser
 from boto import UserAgent
 from . import common
@@ -103,11 +104,11 @@ def make_choice(choices, prng):
             continue
         try:
             (weight, value) = option.split(None, 1)
+            weight = int(weight)
         except ValueError:
-            weight = '1'
+            weight = 1
             value = option
 
-        weight = int(weight)
         if value == 'null' or value == 'None':
             value = ''
 
@@ -118,11 +119,11 @@ def make_choice(choices, prng):
 
 
 def expand_headers(decision, prng):
-    expanded_headers = []
+    expanded_headers = {} 
     for header in decision['headers']:
         h = expand(decision, header[0], prng)
         v = expand(decision, header[1], prng)
-        expanded_headers.append([h, v])
+        expanded_headers[h] = v
     return expanded_headers
 
 
@@ -200,6 +201,8 @@ def parse_options():
     parser.add_option('--seed', dest='seed', type='int',  help='initial seed for the random number generator', metavar='SEED')
     parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
     parser.add_option('-n', dest='num_requests', type='int',  help='issue NUM requests before stopping', metavar='NUM')
+    parser.add_option('-v', '--verbose', dest='verbose', action="store_true",  help='turn on verbose output')
+    parser.add_option('-d', '--debug', dest='debug', action="store_true",  help='turn on debugging (very verbose) output')
     parser.add_option('--decision-graph', dest='graph_filename',  help='file in which to find the request decision graph', metavar='NUM')
 
     parser.set_defaults(num_requests=5)
@@ -215,56 +218,127 @@ def randomlist(seed=None):
         yield rng.random()
 
 
+def populate_buckets(conn, alt):
+    """ Creates buckets and keys for fuzz testing and sets appropriate
+        permissions. Returns a dictionary of the bucket and key names.
+    """
+    breadable = common.get_new_bucket(alt)
+    bwritable = common.get_new_bucket(alt)
+    bnonreadable = common.get_new_bucket(alt)
+
+    oreadable = Key(breadable)
+    owritable = Key(bwritable)
+    ononreadable = Key(breadable)
+    oreadable.set_contents_from_string('oreadable body')
+    owritable.set_contents_from_string('owritable body')
+    ononreadable.set_contents_from_string('ononreadable body')
+
+    breadable.set_acl('public-read')
+    bwritable.set_acl('public-read-write')
+    bnonreadable.set_acl('private')
+    oreadable.set_acl('public-read')
+    owritable.set_acl('public-read-write')
+    ononreadable.set_acl('private')
+
+    return dict(
+        bucket_readable=breadable.name,
+        bucket_writable=bwritable.name,
+        bucket_not_readable=bnonreadable.name,
+        bucket_not_writable=breadable.name,
+        object_readable=oreadable.key,
+        object_writable=owritable.key,
+        object_not_readable=ononreadable.key,
+        object_not_writable=oreadable.key,
+    )
+
+
 def _main():
     """ The main script
     """
     (options, args) = parse_options()
     random.seed(options.seed if options.seed else None)
     s3_connection = common.s3.main
+    alt_connection = common.s3.alt
+
+    if options.outfile:
+        OUT = open(options.outfile, 'w')
+    else:
+        OUT = sys.stderr
+
+    VERBOSE = DEBUG = open('/dev/null', 'w')
+    if options.verbose:
+        VERBOSE = OUT
+    if options.debug:
+        DEBUG = OUT
+        VERBOSE = OUT
 
     request_seeds = None
     if options.seedfile:
         FH = open(options.seedfile, 'r')
-        request_seeds = FH.readlines()
+        request_seeds = [float(line) for line in FH.readlines()]
+        print>>OUT, 'Seedfile: %s' %options.seedfile
+        print>>OUT, 'Number of requests: %d' %len(request_seeds)
     else:
+        if options.seed:
+            print>>OUT, 'Initial Seed: %d' %options.seed
+        print>>OUT, 'Number of requests: %d' %options.num_requests
         random_list = randomlist(options.seed)
         request_seeds = itertools.islice(random_list, options.num_requests)
 
+    print>>OUT, 'Decision Graph: %s' %options.graph_filename
 
     graph_file = open(options.graph_filename, 'r')
     decision_graph = yaml.safe_load(graph_file)
 
-    constants = dict(
-        bucket_readable='TODO-breadable',
-        bucket_not_readable='TODO-bnonreadable',
-        bucket_writable='TODO-bwritable',
-        bucket_not_writable='TODO-bnonwritable',
-        object_readable='TODO-oreadable',
-        object_not_readable='TODO-ononreadable',
-        object_writable='TODO-owritable',
-        object_not_writable='TODO-ononwritable',
-    )
+    constants = populate_buckets(s3_connection, alt_connection)
+    print>>VERBOSE, "Test Buckets/Objects:"
+    for key, value in constants.iteritems():
+        print>>VERBOSE, "\t%s: %s" %(key, value)
 
+    print>>OUT, "Begin Fuzzing..."
+    print>>VERBOSE, '='*80
     for request_seed in request_seeds:
+        print>>OUT, request_seed
+
         prng = random.Random(request_seed)
         decision = assemble_decision(decision_graph, prng)
         decision.update(constants)
 
         method = expand(decision, decision['method'], prng)
-        path = expand(decision, decision['path'], prng)
-        body = expand(decision, decision['body'], prng)
-        headers = expand_headers(decision, prng)
+        path = expand(decision, decision['urlpath'], prng)
 
-        print "Method: %s" % method
-        print "Path: %s" % path
-        print "Headers: %s" % headers
-        print ""
-        print "Body: %s" % body
-        #response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+        try:
+            body = expand(decision, decision['body'], prng)
+        except KeyError:
+            body = ''
 
+        try:
+            headers = expand_headers(decision, prng)
+        except KeyError:
+            headers = {}
+
+        response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+
+        print>>VERBOSE, "%s %s" %(method[:100], path[:100])
+        for h, v in headers.iteritems():
+            print>>VERBOSE, "%s: %s" %(h[:50], v[:50])
+        print>>VERBOSE, "%s\n" % body[:100]
+
+        print>>DEBUG, 'FULL REQUEST'
+        print>>DEBUG, 'Method: %r' %method
+        print>>DEBUG, 'Path: %r' %path
+        print>>DEBUG, 'Headers:'
+        for h, v in headers.iteritems():
+            print>>DEBUG, "\t%r: %r" %(h, v)
+        print>>DEBUG, 'Body: %r' %body
+
+        print>>VERBOSE, 'Response status code: %d %s' %(response.status, response.reason)
+        print>>DEBUG, 'Body:\n%s' %response.read()
         if response.status == 500 or response.status == 503:
-            print 'Request generated with seed %d failed:\n%s' % (request_seed, request)
-        pass
+            print>>OUT, 'FAILED:\n%s' %request
+        print>>VERBOSE, '='*80
+    print>>OUT, '...done fuzzing'
+    common.teardown()
 
 
 def main():

From d2c841d1dfa4bb5551bb38f9f794675062248f9c Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Fri, 19 Aug 2011 15:40:16 -0700
Subject: [PATCH 19/24] S3 Fuzzer: PRNG Seed and decisiong graph fixes

- Change random seed generator to always spit out ints between 0 and 100,000
- Fix seeds so randomly generated seeds are *actually* the same as those seeds
	read from a file.
- Tweaks to decision graph

Remaining Bugs:
	- Single } encountered in format string
	- _mexe complains about "BadStatusLine"
---
 request_decision_graph.yml | 14 ++++++++------
 s3tests/fuzz_headers.py    | 10 +++++-----
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index 443b37c..06dffd7 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -13,17 +13,19 @@ bucket:
         - 13 bucket_get
         - 8 bucket_put
         - 5 bucket_delete
-        - garbage_method
+        - bucket_garbage_method
 
-garbage_method:
+bucket_garbage_method:
     set:
         method:
             - '{random 1-100 printable}'
             - '{random 10-100 binary}'
-        urlpath:
-            - '/{bucket}'
-            - '/{bucket}/{object}'
-            - '{random 10-1000 binary}'
+        bucket:
+            - '{bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage}'
     choices:
         - bucket_get_simple
         - bucket_get_filtered
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index c23ebc6..facfbb0 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -215,7 +215,7 @@ def randomlist(seed=None):
     """
     rng = random.Random(seed)
     while True:
-        yield rng.random()
+        yield rng.randint(0,100000) #100,000 seeds is enough, right?
 
 
 def populate_buckets(conn, alt):
@@ -275,7 +275,7 @@ def _main():
     request_seeds = None
     if options.seedfile:
         FH = open(options.seedfile, 'r')
-        request_seeds = [float(line) for line in FH.readlines()]
+        request_seeds = [int(line) for line in FH if line != '\n']
         print>>OUT, 'Seedfile: %s' %options.seedfile
         print>>OUT, 'Number of requests: %d' %len(request_seeds)
     else:
@@ -298,7 +298,7 @@ def _main():
     print>>OUT, "Begin Fuzzing..."
     print>>VERBOSE, '='*80
     for request_seed in request_seeds:
-        print>>OUT, request_seed
+        print>>OUT, '%r' %request_seed
 
         prng = random.Random(request_seed)
         decision = assemble_decision(decision_graph, prng)
@@ -317,13 +317,13 @@ def _main():
         except KeyError:
             headers = {}
 
-        response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
-
         print>>VERBOSE, "%s %s" %(method[:100], path[:100])
         for h, v in headers.iteritems():
             print>>VERBOSE, "%s: %s" %(h[:50], v[:50])
         print>>VERBOSE, "%s\n" % body[:100]
 
+        response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+
         print>>DEBUG, 'FULL REQUEST'
         print>>DEBUG, 'Method: %r' %method
         print>>DEBUG, 'Path: %r' %path

From f45d28765dbdfd81c0c9c03578ea6dc5b9c7fcb6 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Fri, 19 Aug 2011 15:58:22 -0700
Subject: [PATCH 20/24] S3 Fuzzer: remove num-retries override

Overriding num-retries to 0 sometimes causes boto to fail when the server
doesn't respond fast enough.  Removing the override shouldn't cause any
problems with the fuzzer.
---
 s3tests/fuzz_headers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index facfbb0..79a8d47 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -322,8 +322,6 @@ def _main():
             print>>VERBOSE, "%s: %s" %(h[:50], v[:50])
         print>>VERBOSE, "%s\n" % body[:100]
 
-        response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
-
         print>>DEBUG, 'FULL REQUEST'
         print>>DEBUG, 'Method: %r' %method
         print>>DEBUG, 'Path: %r' %path
@@ -332,6 +330,9 @@ def _main():
             print>>DEBUG, "\t%r: %r" %(h, v)
         print>>DEBUG, 'Body: %r' %body
 
+        #response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
+        response = s3_connection.make_request(method, path, data=body, headers=headers)
+
         print>>VERBOSE, 'Response status code: %d %s' %(response.status, response.reason)
         print>>DEBUG, 'Body:\n%s' %response.read()
         if response.status == 500 or response.status == 503:

From 18c3fe53c225b4a558aa72dadf68b2a51d85eab0 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Tue, 23 Aug 2011 15:08:18 -0700
Subject: [PATCH 21/24] S3 Fuzzer: Output and garbage data tweaks.

- Output tweaks
- added support for printable_no_whitespace and binary_no_whitespace
---
 request_decision_graph.yml        | 50 ++++++++++++++++++-------------
 s3tests/functional/test_fuzzer.py | 14 +++++++++
 s3tests/fuzz_headers.py           | 35 +++++++++++++++-------
 3 files changed, 67 insertions(+), 32 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index 06dffd7..fa13eb5 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -3,6 +3,9 @@ start:
         garbage:
             - '{random 10-3000 printable}'
             - '{random 10-1000 binary}'
+        garbage_no_whitespace:
+            - '{random 10-3000 printable_no_whitespace}'
+            - '{random 10-1000 binary_no_whitespace}'
     choices:
         - bucket
 
@@ -25,7 +28,7 @@ bucket_garbage_method:
             - '{bucket_not_readable}'
             - '{bucket_writable}'
             - '{bucket_not_writable}'
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
     choices:
         - bucket_get_simple
         - bucket_get_filtered
@@ -40,12 +43,12 @@ bucket_delete:
         bucket:
             - '{bucket_writable}'
             - '{bucket_not_writable}'
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
         query:
             - null
             - policy
             - website
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
     choices: []
 
 bucket_get:
@@ -54,7 +57,7 @@ bucket_get:
         bucket:
             - '{bucket_readable}'
             - '{bucket_not_readable}'
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
     choices:
         - 11 bucket_get_simple
         - bucket_get_filtered
@@ -72,56 +75,56 @@ bucket_get_simple:
             - requestPayment
             - versioning
             - website
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
     choices: []
 
 bucket_get_uploads:
     set:
         delimiter:
             - null
-            - '3 delimiter={garbage}'
+            - '3 delimiter={garbage_no_whitespace}'
         prefix:
             - null
-            - '3 prefix={garbage}'
+            - '3 prefix={garbage_no_whitespace}'
         key_marker:
             - null
             - 'key-marker={object_readable}'
             - 'key-marker={object_not_readable}'
             - 'key-marker={invalid_key}'
-            - 'key-marker={random 100-1000 printable}'
+            - 'key-marker={random 100-1000 printable_no_whitespace}'
         max_uploads:
             - null
-            - 'max-uploads={random 1-5 binary}'
+            - 'max-uploads={random 1-5 binary_no_whitespace}'
             - 'max-uploads={random 1-1000 digits}'
         upload_id_marker:
             - null
-            - '3 upload-id-marker={random}'
+            - '3 upload-id-marker={random 0-1000 printable_no_whitespace}'
         query:
             - 'uploads'
             - 'uploads&{delimiter}&{prefix}'
             - 'uploads&{max_uploads}&{key_marker}&{upload_id_marker}'
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
     choices: []
 
 bucket_get_filtered:
     set:
         delimiter:
-            - 'delimiter={garbage}'
+            - 'delimiter={garbage_no_whitespace}'
         prefix:
-            - 'prefix={garbage}'
+            - 'prefix={garbage_no_whitespace}'
         marker:
             - 'marker={object_readable}'
             - 'marker={object_not_readable}'
             - 'marker={invalid_key}'
-            - 'marker={random 100-1000 printable}'
+            - 'marker={random 100-1000 printable_no_whitespace}'
         max_keys:
-            - 'max-keys={random 1-5 binary}'
+            - 'max-keys={random 1-5 binary_no_whitespace}'
             - 'max-keys={random 1-1000 digits}'
         query:
             - null
             - '{delimiter}&{prefix}'
             - '{max-keys}&{marker}'
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
     choices: []
 
 bucket_put:
@@ -129,7 +132,7 @@ bucket_put:
         bucket:
             - '{bucket_writable}'
             - '{bucket_not_writable}'
-            - '2 {garbage}'
+            - '2 {garbage_no_whitespace}'
         method: PUT
     choices:
         - bucket_put_simple
@@ -142,9 +145,14 @@ bucket_put_create:
             - '2 {garbage}'
             - '<CreateBucketConfiguration><LocationConstraint>{random 2-10 binary}</LocationConstraint></CreateBucketConfiguration>'
         acl:
-            - private
+            - 'private'
+            - 'public-read'
+            - 'public-read-write'
+            - 'authenticated-read'
+            - 'bucket-owner-read'
+            - 'bucket-owner-full-control'
             - '{random 3000 letters}'
-            - '{random 100-1000 binary}'
+            - '{random 100-1000 binary_no_whitespace}'
     headers:
         - ['0-1', 'x-amz-acl', '{acl}']
     choices: []
@@ -163,7 +171,7 @@ bucket_put_versioning:
             - '<MfaDelete>{random 2-10 binary}</MfaDelete>'
             - '<MfaDelete>{random 2000-3000 printable}</MfaDelete>'
         mfa_header:
-            - '{random 10-1000 printable} {random 10-1000 printable}'
+            - '{random 10-1000 printable_no_whitespace} {random 10-1000 printable_no_whitespace}'
     headers:
         - ['0-1', 'x-amz-mfa', '{mfa_header}']
     choices: []
@@ -225,7 +233,7 @@ bucket_put_simple:
         notification_body:
             - null
             - '<NotificationConfiguration />'
-            - '2 <NotificationConfiguration><TopicConfiguration>{topic}{event}</TopicConfiguration}</NotificationConfiguration>'
+            - '2 <NotificationConfiguration><TopicConfiguration>{topic}{event}</TopicConfiguration></NotificationConfiguration>'
         topic:
             - null
             - '2 <Topic>{garbage}</Topic>'
diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index eb2ab1f..1039b55 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -158,6 +158,20 @@ def test_expand_random_binary():
     eq(got, '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
 
 
+def test_expand_random_printable_no_whitespace():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random 500 printable_no_whitespace}', prng)
+        assert_true(reduce(lambda x, y: x and y, [x not in string.whitespace and x in string.printable for x in got]))
+
+
+def test_expand_random_binary():
+    prng = random.Random(1)
+    for _ in xrange(1000):
+        got = expand({}, '{random 500 binary_no_whitespace}', prng)
+        assert_true(reduce(lambda x, y: x and y, [x not in string.whitespace for x in got]))
+
+
 def test_expand_random_no_args():
     prng = random.Random(1)
     for _ in xrange(1000):
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 79a8d47..6e581b4 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -136,6 +136,7 @@ def expand(decision, value, prng):
 
 class RepeatExpandingFormatter(string.Formatter):
     charsets = {
+        'printable_no_whitespace': string.printable.translate(None, string.whitespace),
         'printable': string.printable,
         'punctuation': string.punctuation,
         'whitespace': string.whitespace,
@@ -161,8 +162,7 @@ class RepeatExpandingFormatter(string.Formatter):
         if self._recursion > 5:
             raise RecursionError(key)
         fmt = self.__class__(self.prng, _recursion=self._recursion+1)
-        # must use vformat not **kwargs so our SpecialVariables is not
-        # downgraded to just a dict
+
         n = fmt.vformat(val, args, kwargs)
         return n
 
@@ -185,10 +185,12 @@ class RepeatExpandingFormatter(string.Formatter):
         except IndexError:
             charset_arg = 'printable'
 
-        if charset_arg == 'binary':
+        if charset_arg == 'binary' or charset_arg == 'binary_no_whitespace':
             num_bytes = length + 8
             tmplist = [self.prng.getrandbits(64) for _ in xrange(num_bytes / 8)]
             tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
+            if charset_arg == 'binary_no_whitespace':
+                tmpstring = ''.join(c for c in tmpstring if c not in string.whitespace)
             return tmpstring[0:length]
         else:
             charset = self.charsets[charset_arg]
@@ -198,14 +200,16 @@ class RepeatExpandingFormatter(string.Formatter):
 def parse_options():
     parser = OptionParser()
     parser.add_option('-O', '--outfile', help='write output to FILE. Defaults to STDOUT', metavar='FILE')
-    parser.add_option('--seed', dest='seed', type='int',  help='initial seed for the random number generator', metavar='SEED')
+    parser.add_option('--seed', dest='seed', type='int',  help='initial seed for the random number generator')
     parser.add_option('--seed-file', dest='seedfile', help='read seeds for specific requests from FILE', metavar='FILE')
     parser.add_option('-n', dest='num_requests', type='int',  help='issue NUM requests before stopping', metavar='NUM')
     parser.add_option('-v', '--verbose', dest='verbose', action="store_true",  help='turn on verbose output')
     parser.add_option('-d', '--debug', dest='debug', action="store_true",  help='turn on debugging (very verbose) output')
-    parser.add_option('--decision-graph', dest='graph_filename',  help='file in which to find the request decision graph', metavar='NUM')
+    parser.add_option('--decision-graph', dest='graph_filename',  help='file in which to find the request decision graph')
+    parser.add_option('--no-cleanup', dest='cleanup', action="store_false", help='turn off teardown so you can peruse the state of buckets after testing')
 
     parser.set_defaults(num_requests=5)
+    parser.set_defaults(cleanup=True)
     parser.set_defaults(graph_filename='request_decision_graph.yml')
     return parser.parse_args()
 
@@ -317,10 +321,10 @@ def _main():
         except KeyError:
             headers = {}
 
-        print>>VERBOSE, "%s %s" %(method[:100], path[:100])
+        print>>VERBOSE, "%r %r" %(method[:100], path[:100])
         for h, v in headers.iteritems():
-            print>>VERBOSE, "%s: %s" %(h[:50], v[:50])
-        print>>VERBOSE, "%s\n" % body[:100]
+            print>>VERBOSE, "%r: %r" %(h[:50], v[:50])
+        print>>VERBOSE, "%r\n" % body[:100]
 
         print>>DEBUG, 'FULL REQUEST'
         print>>DEBUG, 'Method: %r' %method
@@ -333,13 +337,22 @@ def _main():
         #response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
         response = s3_connection.make_request(method, path, data=body, headers=headers)
 
+        failed = True if response.status in [500, 503] else False
+        if failed:
+            print>>OUT, 'FAILED:'
+            OLD_VERBOSE = VERBOSE
+            OLD_DEBUG = DEBUG
+            VERBOSE = DEBUG = OUT
         print>>VERBOSE, 'Response status code: %d %s' %(response.status, response.reason)
         print>>DEBUG, 'Body:\n%s' %response.read()
-        if response.status == 500 or response.status == 503:
-            print>>OUT, 'FAILED:\n%s' %request
         print>>VERBOSE, '='*80
+        if failed:
+            VERBOSE = OLD_VERBOSE
+            DEBUG = OLD_DEBUG
     print>>OUT, '...done fuzzing'
-    common.teardown()
+
+    if options.cleanup:
+        common.teardown()
 
 
 def main():

From ab8235b40c7afa36d73fd1713facac937d0df082 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Wed, 24 Aug 2011 12:39:12 -0700
Subject: [PATCH 22/24] S3 Fuzzer: changed output and failure catching.

---
 s3tests/fuzz_headers.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index 6e581b4..f5a20bc 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -1,7 +1,7 @@
 from boto.s3.connection import S3Connection
+from boto.exception import BotoServerError
 from boto.s3.key import Key
 from optparse import OptionParser
-from boto import UserAgent
 from . import common
 
 import traceback
@@ -302,8 +302,7 @@ def _main():
     print>>OUT, "Begin Fuzzing..."
     print>>VERBOSE, '='*80
     for request_seed in request_seeds:
-        print>>OUT, '%r' %request_seed
-
+        print>>VERBOSE, 'Seed is: %r' %request_seed
         prng = random.Random(request_seed)
         decision = assemble_decision(decision_graph, prng)
         decision.update(constants)
@@ -332,23 +331,30 @@ def _main():
         print>>DEBUG, 'Headers:'
         for h, v in headers.iteritems():
             print>>DEBUG, "\t%r: %r" %(h, v)
-        print>>DEBUG, 'Body: %r' %body
+        print>>DEBUG, 'Body: %r\n' %body
 
-        #response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=0)
-        response = s3_connection.make_request(method, path, data=body, headers=headers)
+        failed = False # Let's be optimistic, shall we?
+        try:
+            response = s3_connection.make_request(method, path, data=body, headers=headers, override_num_retries=1)
+            body = response.read()
+        except BotoServerError, e:
+            response = e
+            body = e.body
+            failed = True
 
-        failed = True if response.status in [500, 503] else False
         if failed:
             print>>OUT, 'FAILED:'
             OLD_VERBOSE = VERBOSE
             OLD_DEBUG = DEBUG
             VERBOSE = DEBUG = OUT
+        print>>VERBOSE, 'Seed was: %r' %request_seed
         print>>VERBOSE, 'Response status code: %d %s' %(response.status, response.reason)
-        print>>DEBUG, 'Body:\n%s' %response.read()
+        print>>DEBUG, 'Body:\n%s' %body
         print>>VERBOSE, '='*80
         if failed:
             VERBOSE = OLD_VERBOSE
             DEBUG = OLD_DEBUG
+
     print>>OUT, '...done fuzzing'
 
     if options.cleanup:

From d0a42a7a8911132797bdf7ce5a65a9cf81f279d2 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Fri, 9 Sep 2011 08:36:51 -0700
Subject: [PATCH 23/24] S3-Fuzzer: add objects to decision graph

Add objects to the fuzzer's attack surface description
---
 request_decision_graph.yml | 249 +++++++++++++++++++++++++++++++++++--
 1 file changed, 239 insertions(+), 10 deletions(-)

diff --git a/request_decision_graph.yml b/request_decision_graph.yml
index fa13eb5..6b2fab3 100644
--- a/request_decision_graph.yml
+++ b/request_decision_graph.yml
@@ -6,8 +6,18 @@ start:
         garbage_no_whitespace:
             - '{random 10-3000 printable_no_whitespace}'
             - '{random 10-1000 binary_no_whitespace}'
+        acl_header:
+            - 'private'
+            - 'public-read'
+            - 'public-read-write'
+            - 'authenticated-read'
+            - 'bucket-owner-read'
+            - 'bucket-owner-full-control'
+            - '{random 3000 letters}'
+            - '{random 100-1000 binary_no_whitespace}'
     choices:
         - bucket
+        - object
 
 bucket:
     set:
@@ -144,17 +154,8 @@ bucket_put_create:
         body:
             - '2 {garbage}'
             - '<CreateBucketConfiguration><LocationConstraint>{random 2-10 binary}</LocationConstraint></CreateBucketConfiguration>'
-        acl:
-            - 'private'
-            - 'public-read'
-            - 'public-read-write'
-            - 'authenticated-read'
-            - 'bucket-owner-read'
-            - 'bucket-owner-full-control'
-            - '{random 3000 letters}'
-            - '{random 100-1000 binary_no_whitespace}'
     headers:
-        - ['0-1', 'x-amz-acl', '{acl}']
+        - ['0-5', 'x-amz-acl', '{acl_header}']
     choices: []
 
 bucket_put_versioning:
@@ -260,3 +261,231 @@ bucket_put_simple:
             - '<ErrorDocument><Key>{suffix}</Key></ErrorDocument>'
     choices: []
 
+object:
+    set:
+        urlpath: '/{bucket}/{object}'
+
+        range_header:
+            - null
+            - 'bytes={random 1-2 digits}-{random 1-4 digits}'
+            - 'bytes={random 1-1000 binary_no_whitespace}'
+        if_modified_since_header:
+            - null
+            - '2 {garbage_no_whitespace}'
+        if_match_header:
+            - null
+            - '2 {garbage_no_whitespace}'
+        if_none_match_header:
+            - null
+            - '2 {garbage_no_whitespace}'
+    choices:
+        - object_delete
+        - object_get
+        - object_put
+        - object_head
+        - object_garbage_method
+
+object_garbage_method:
+    set:
+        method:
+            - '{random 1-100 printable}'
+            - '{random 10-100 binary}'
+        bucket:
+            - '{bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{bucket_writable}'
+            - '{bucket_not_writable}'
+            - '2 {garbage_no_whitespace}'
+        object:
+            - '{object_readable}'
+            - '{object_not_readable}'
+            - '{object_writable}'
+            - '{object_not_writable}'
+            - '2 {garbage_no_whitespace}'
+    choices:
+        - object_get_query
+        - object_get_head_simple
+
+object_delete:
+    set:
+        method: DELETE
+        bucket:
+            - '5 {bucket_writable}'
+            - '{bucket_not_writable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_writable}'
+            - '{object_not_writable}'
+            - '2 {garbage_no_whitespace}'
+    choices: []
+
+object_get:
+    set:
+        method: GET
+        bucket:
+            - '5 {bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_readable}'
+            - '{object_not_readable}'
+            - '{garbage_no_whitespace}'
+    choices:
+        - 5 object_get_head_simple
+        - 2 object_get_query
+
+object_get_query:
+    set:
+        query: 
+            - 'torrent'
+            - 'acl'
+    choices: []
+
+object_get_head_simple:
+    set: {}
+    headers:
+        - ['0-1', 'range', '{range_header}']
+        - ['0-1', 'if-modified-since', '{if_modified_since_header}']
+        - ['0-1', 'if-unmodified-since', '{if_modified_since_header}']
+        - ['0-1', 'if-match', '{if_match_header}']
+        - ['0-1', 'if-none-match', '{if_none_match_header}']
+    choices: []
+
+object_head:
+    set:
+        method: HEAD
+        bucket:
+            - '5 {bucket_readable}'
+            - '{bucket_not_readable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_readable}'
+            - '{object_not_readable}'
+            - '{garbage_no_whitespace}'
+    choices:
+        - object_get_head_simple
+
+object_put:
+    set:
+        method: PUT
+        bucket:
+            - '5 {bucket_writable}'
+            - '{bucket_not_writable}'
+            - '{garbage_no_whitespace}'
+        object:
+            - '{object_writable}'
+            - '{object_not_writable}'
+            - '{garbage_no_whitespace}'
+        cache_control:
+            - null
+            - '{garbage_no_whitespace}'
+            - 'no-cache'
+        content_disposition:
+            - null
+            - '{garbage_no_whitespace}'
+        content_encoding:
+            - null
+            - '{garbage_no_whitespace}'
+        content_length:
+            - '{random 1-20 digits}'
+            - '{garbage_no_whitespace}'
+        content_md5:
+            - null
+            - '{garbage_no_whitespace}'
+        content_type:
+            - null
+            - 'binary/octet-stream'
+            - '{garbage_no_whitespace}'
+        expect:
+            - null
+            - '100-continue'
+            - '{garbage_no_whitespace}'
+        expires:
+            - null
+            - '{random 1-10000000 digits}'
+            - '{garbage_no_whitespace}'
+        meta_key:
+            - null
+            - 'foo'
+            - '{garbage_no_whitespace}'
+        meta_value:
+            - null
+            - '{garbage_no_whitespace}'
+    choices:
+        - object_put_simple
+        - object_put_acl
+        - object_put_copy
+
+object_put_simple:
+    set: {}
+    headers:
+        - ['0-1', 'cache-control', '{cache_control}']
+        - ['0-1', 'content-disposition', '{content_disposition}']
+        - ['0-1', 'content-encoding', '{content_encoding}']
+        - ['0-1', 'content-length', '{content_length}']
+        - ['0-1', 'content-md5', '{content_md5}']
+        - ['0-1', 'content-type', '{content_type}']
+        - ['0-1', 'expect', '{expect}']
+        - ['0-1', 'expires', '{expires}']
+        - ['0-1', 'x-amz-acl', '{acl_header}']
+        - ['0-6', 'x-amz-meta-{meta_key}', '{meta_value}']
+    choices: []
+
+object_put_acl:
+    set:
+        query: 'acl'
+        body:
+            - null
+            - '2 {garbage}'
+            - '<AccessControlPolicy>{owner}{acl}</AccessControlPolicy>'
+        owner:
+            - null
+            - '7 <Owner>{id}{display_name}</Owner>'
+        id:
+            - null
+            - '<ID>{random 10-200 binary}</ID>'
+            - '<ID>{random 1000-3000 printable}</ID>'
+        display_name:
+            - null
+            - '2 <DisplayName>{random 10-200 binary}</DisplayName>'
+            - '2 <DisplayName>{random 1000-3000 printable}</DisplayName>'
+            - '2 <DisplayName>{random 10-300 letters}@{random 10-300 letters}.{random 2-4 letters}</DisplayName>'
+        acl:
+            - null
+            - '10 <AccessControlList><Grant>{grantee}{permission}</Grant></AccessControlList>'
+        grantee:
+            - null
+            - '7 <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">{id}{display_name}</Grantee>'
+        permission:
+            - null
+            - '7 <Permission>{permission_value}</Permission>'
+        permission_value:
+            - '2 {garbage}'
+            - FULL_CONTROL
+            - WRITE
+            - WRITE_ACP
+            - READ
+            - READ_ACP
+    headers:
+        - ['0-1', 'cache-control', '{cache_control}']
+        - ['0-1', 'content-disposition', '{content_disposition}']
+        - ['0-1', 'content-encoding', '{content_encoding}']
+        - ['0-1', 'content-length', '{content_length}']
+        - ['0-1', 'content-md5', '{content_md5}']
+        - ['0-1', 'content-type', '{content_type}']
+        - ['0-1', 'expect', '{expect}']
+        - ['0-1', 'expires', '{expires}']
+        - ['0-1', 'x-amz-acl', '{acl_header}']
+    choices: []
+
+object_put_copy:
+    set: {}
+    headers:
+        - ['1-1', 'x-amz-copy-source', '{source_object}']
+        - ['0-1', 'x-amz-acl', '{acl_header}']
+        - ['0-1', 'x-amz-metadata-directive', '{metadata_directive}']
+        - ['0-1', 'x-amz-copy-source-if-match', '{if_match_header}']
+        - ['0-1', 'x-amz-copy-source-if-none-match', '{if_none_match_header}']
+        - ['0-1', 'x-amz-copy-source-if-modified-since', '{if_modified_since_header}']
+        - ['0-1', 'x-amz-copy-source-if-unmodified-since', '{if_modified_since_header}']
+    choices: []

From 046b92b7ebd914543b01552c7be7639613dd2a92 Mon Sep 17 00:00:00 2001
From: Kyle Marsh <kyle.marsh@dreamhost.com>
Date: Mon, 12 Sep 2011 12:35:10 -0700
Subject: [PATCH 24/24] S3 Fuzzer: httplib error handling

Sometimes httplib will through a BadStatusLine error that the fuzzer should
catch and register as a failure.
---
 s3tests/functional/test_fuzzer.py | 2 +-
 s3tests/fuzz_headers.py           | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/s3tests/functional/test_fuzzer.py b/s3tests/functional/test_fuzzer.py
index 1039b55..717b1db 100644
--- a/s3tests/functional/test_fuzzer.py
+++ b/s3tests/functional/test_fuzzer.py
@@ -1,4 +1,4 @@
-import sys
+mport sys
 import itertools
 import nose
 import random
diff --git a/s3tests/fuzz_headers.py b/s3tests/fuzz_headers.py
index f5a20bc..e49713f 100644
--- a/s3tests/fuzz_headers.py
+++ b/s3tests/fuzz_headers.py
@@ -1,6 +1,7 @@
 from boto.s3.connection import S3Connection
 from boto.exception import BotoServerError
 from boto.s3.key import Key
+from httplib import BadStatusLine
 from optparse import OptionParser
 from . import common
 
@@ -341,6 +342,10 @@ def _main():
             response = e
             body = e.body
             failed = True
+        except BadStatusLine, e:
+            print>>OUT, 'FAILED: failed to parse response (BadStatusLine); probably a NUL byte in your request?'
+            print>>VERBOSE, '='*80
+            continue
 
         if failed:
             print>>OUT, 'FAILED:'