S3 Fuzzer: Incorporate Tv's suggestions

Tv looked over the fuzzer and had some idiomatic and design suggestions.

Adds several tests and reworks how expansion happens in addition to idiom
changes.

s3tests/functional/test_fuzzer.py

@@ -8,6 +8,7 @@ import yaml
 from s3tests.fuzz_headers import *
 
 from nose.tools import eq_ as eq
+from nose.tools import assert_true
 from nose.plugins.attrib import attr
 
 from .utils import assert_raises
@@ -87,6 +88,13 @@ def build_graph():
         'set': {},
         'choices': [None]
     }
+    graph['repeated_headers_node'] = {
+        'set': {},
+        'headers': [
+            ['1-2', 'random-header-{random 5-10 printable}', '{random 20-30 punctuation}']
+        ],
+        'choices': ['leaf']
+    }
     graph['weighted_null_choice_node'] = {
         'set': {},
         'choices': ['3 null']
@@ -123,7 +131,7 @@ def test_descend_node():
 def test_descend_bad_node():
     graph = build_graph()
     prng = random.Random(1)
-    assert_raises(KeyError, descend_graph, graph, 'bad_node', prng)
+    assert_raises(DecisionGraphError, descend_graph, graph, 'bad_node', prng)
 
 
 def test_descend_nonexistant_child():
@@ -148,6 +156,46 @@ def test_SpecialVariables_binary():
     eq(tester['random 10-15 binary'], '\xdfj\xf1\xd80>a\xcd\xc4\xbb')
 
 
+def test_SpeicalVariables_random_no_args():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    for _ in xrange(1000):
+        val = tester['random']
+        val = val.replace('{{', '{').replace('}}','}')
+        assert_true(0 <= len(val) <= 1000)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in val]))
+
+
+def test_SpeicalVariables_random_no_charset():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    for _ in xrange(1000):
+        val = tester['random 10-30']
+        val = val.replace('{{', '{').replace('}}','}')
+        assert_true(10 <= len(val) <= 30)
+        assert_true(reduce(lambda x, y: x and y, [x in string.printable for x in val]))
+
+
+def test_SpeicalVariables_random_exact_length():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    for _ in xrange(1000):
+        val = tester['random 10 digits']
+        assert_true(len(val) == 10)
+        assert_true(reduce(lambda x, y: x and y, [x in string.digits for x in val]))
+
+
+def test_SpecialVariables_random_errors():
+    prng = random.Random(1)
+    tester = SpecialVariables({}, prng)
+
+    assert_raises(KeyError, lambda x: tester[x], 'random 10-30 foo')
+    assert_raises(ValueError, lambda x: tester[x], 'random printable')
+
+
 def test_assemble_decision():
     graph = build_graph()
     prng = random.Random(1)
@@ -161,49 +209,56 @@ def test_assemble_decision():
     assert_raises(KeyError, lambda x: decision[x], 'key3')
 
 
-def test_expand_key():
+def test_expand_escape():
-    prng = random.Random(1)
+    decision = dict(
-    test_decision = {
+        foo='{{bar}}',
-        'key1': 'value1',
+        )
-        'randkey': 'value-{random 10-15 printable}',
+    got = expand(decision, '{foo}')
-        'indirect': '{key1}',
+    eq(got, '{bar}')
-        'dbl_indirect': '{indirect}'
-    }
-    decision = SpecialVariables(test_decision, prng)
-
-    randkey = expand_key(decision, test_decision['randkey'])
-    indirect = expand_key(decision, test_decision['indirect'])
-    dbl_indirect = expand_key(decision, test_decision['dbl_indirect'])
-
-    eq(indirect, 'value1')
-    eq(dbl_indirect, 'value1')
-    eq(randkey, 'value-[/pNI$;92@')
 
 
-def test_expand_loop():
+def test_expand_indirect():
-    prng = random.Random(1)
+    decision = dict(
-    test_decision = {
+        foo='{bar}',
-        'key1': '{key2}',
+        bar='quux',
-        'key2': '{key1}',
+        )
-    }
+    got = expand(decision, '{foo}')
-    decision = SpecialVariables(test_decision, prng)
+    eq(got, 'quux')
-    assert_raises(RuntimeError, expand_key, decision, test_decision['key1'])
 
 
-def test_expand_decision():
+def test_expand_indirect_double():
-    graph = build_graph()
+    decision = dict(
-    prng = random.Random(1)
+        foo='{bar}',
+        bar='{quux}',
+        quux='thud',
+        )
+    got = expand(decision, '{foo}')
+    eq(got, 'thud')
 
-    decision = assemble_decision(graph, prng)
-    decision.update({'bucket_readable': 'my-readable-bucket'})
 
-    request = expand_decision(decision, prng)
+def test_expand_recursive():
+    decision = dict(
+        foo='{foo}',
+        )
+    e = assert_raises(RecursionError, expand, decision, '{foo}')
+    eq(str(e), "Runaway recursion in string formatting: 'foo'")
 
-    eq(request['key1'], 'value1')
+
-    eq(request['indirect_key1'], 'value1')
+def test_expand_recursive_mutual():
-    eq(request['path'], '/my-readable-bucket')
+    decision = dict(
-    eq(request['randkey'], 'value-cx+*~G@&uW_[OW3')
+        foo='{bar}',
-    assert_raises(KeyError, lambda x: decision[x], 'key3')
+        bar='{foo}',
+        )
+    e = assert_raises(RecursionError, expand, decision, '{foo}')
+    eq(str(e), "Runaway recursion in string formatting: 'foo'")
+
+
+def test_expand_recursive_not_too_eager():
+    decision = dict(
+        foo='bar',
+        )
+    got = expand(decision, 100*'{foo}')
+    eq(got, 100*'bar')
 
 
 def test_weighted_choices():
@@ -280,29 +335,36 @@ def test_header_presence():
     for header, value in decision['headers']:
         if header == 'my-header':
             eq(value, '{header_val}')
-            nose.tools.assert_true(next(c1) < 1)
+            assert_true(next(c1) < 1)
         elif header == 'random-header-{random 5-10 printable}':
             eq(value, '{random 20-30 punctuation}')
-            nose.tools.assert_true(next(c2) < 2)
+            assert_true(next(c2) < 2)
         else:
             raise KeyError('unexpected header found: %s' % header)
 
-    nose.tools.assert_true(next(c1))
+    assert_true(next(c1))
-    nose.tools.assert_true(next(c2))
+    assert_true(next(c2))
 
 
-def test_header_expansion():
+def test_duplicate_header():
+    graph = build_graph()
+    prng = random.Random(1)
+    assert_raises(DecisionGraphError, descend_graph, graph, 'repeated_headers_node', prng)
+
+
+def test_expand_headers():
     graph = build_graph()
     prng = random.Random(1)
     decision = descend_graph(graph, 'node1', prng)
-    expanded_decision = expand_decision(decision, prng)
+    special_decision = SpecialVariables(decision, prng)
+    expanded_headers = expand_headers(special_decision)
 
-    for header, value in expanded_decision['headers']:
+    for header, value in expanded_headers:
         if header == 'my-header':
-            nose.tools.assert_true(value in ['h1', 'h2', 'h3'])
+            assert_true(value in ['h1', 'h2', 'h3'])
         elif header.startswith('random-header-'):
-            nose.tools.assert_true(20 <= len(value) <= 30)
+            assert_true(20 <= len(value) <= 30)
-            nose.tools.assert_true(string.strip(value, SpecialVariables.charsets['punctuation']) is '')
+            assert_true(string.strip(value, SpecialVariables.charsets['punctuation']) is '')
         else:
-            raise KeyError('unexpected header found: "%s"' % header)
+            raise DecisionGraphError('unexpected header found: "%s"' % header)
 

s3tests/fuzz_headers.py

@@ -13,6 +13,27 @@ import sys
 import re
 
 
+class DecisionGraphError(Exception):
+    """ Raised when a node in a graph tries to set a header or
+        key that was previously set by another node
+    """
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class RecursionError(Exception):
+    """Runaway recursion in string formatting"""
+
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return '{0.__doc__}: {0.msg!r}'.format(self)
+
+
 def assemble_decision(decision_graph, prng):
     """ Take in a graph describing the possible decision space and a random
         number generator and traverse the graph to build a decision
@@ -36,31 +57,33 @@ def descend_graph(decision_graph, node_name, prng):
     except IndexError:
         decision = {}
 
-    for key in node['set']:
+    for key, choices in node['set'].iteritems():
-        if decision.has_key(key):
+        if key in decision:
-            raise KeyError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
+            raise DecisionGraphError("Node %s tried to set '%s', but that key was already set by a lower node!" %(node_name, key))
-        decision[key] = make_choice(node['set'][key], prng)
+        decision[key] = make_choice(choices, prng)
 
-    if node.has_key('headers'):
+    if 'headers' in node:
-        if not decision.has_key('headers'):
+        decision.setdefault('headers', [])
-            decision['headers'] = []
 
         for desc in node['headers']:
-            if len(desc) == 3:
-                repetition_range = desc.pop(0)
             try:
-                    size_min, size_max = [int(x) for x in repetition_range.split('-')]
+                (repetition_range, header, value) = desc
-                except IndexError:
+            except ValueError:
-                    size_min = size_max = int(repetition_range)
+                (header, value) = desc
-            else:
+                repetition_range = '1'
-                size_min = size_max = 1
+
+            try:
+                size_min, size_max = repetition_range.split('-', 1)
+            except ValueError:
+                size_min = size_max = repetition_range
+
+            size_min = int(size_min)
+            size_max = int(size_max)
+
             num_reps = prng.randint(size_min, size_max)
-            for _ in xrange(num_reps):
-                header = desc[0]
-                value = desc[1]
             if header in [h for h, v in decision['headers']]:
-                    if not re.search('{[a-zA-Z_0-9 -]+}', header):
+                    raise DecisionGraphError("Node %s tried to add header '%s', but that header already exists!" %(node_name, header))
-                        raise KeyError("Node %s tried to add header '%s', but that header already exists!" %(node_name, header))
+            for _ in xrange(num_reps):
                 decision['headers'].append([header, value])
 
     return decision
@@ -78,52 +101,59 @@ def make_choice(choices, prng):
         if option is None:
             weighted_choices.append('')
             continue
-        fields = option.split(None, 1)
+        try:
-        if len(fields) == 1:
+            (weight, value) = option.split(None, 1)
-            weight = 1
+        except ValueError:
-            value = fields[0]
+            weight = '1'
-        else:
+            value = option
-            weight = int(fields[0])
+
-            value = fields[1]
+        weight = int(weight)
         if value == 'null' or value == 'None':
             value = ''
+
         for _ in xrange(weight):
             weighted_choices.append(value)
 
     return prng.choice(weighted_choices)
 
 
-def expand_decision(decision, prng):
+def expand_headers(decision):
-    """ Take in a decision and a random number generator.  Expand variables in
+    expanded_headers = []
-        decision's values and headers until all values are fully expanded and
+    for header in decision['headers']:
-        build a request out of the information
+        h = expand(decision, header[0])
-    """
+        v = expand(decision, header[1])
-    special_decision = SpecialVariables(decision, prng)
+        expanded_headers.append([h, v])
-    for key in special_decision:
+    return expanded_headers
-        if not key == 'headers':
-            decision[key] = expand_key(special_decision, decision[key])
-        else:
-            for header in special_decision[key]:
-                header[0] = expand_key(special_decision, header[0])
-                header[1] = expand_key(special_decision, header[1])
-    return decision
 
 
-def expand_key(decision, value):
+def expand(decision, value):
     c = itertools.count()
-    fmt = string.Formatter()
+    fmt = RepeatExpandingFormatter()
-    old = value
+    new = fmt.vformat(value, [], decision)
-    while True:
+    return new
-        new = fmt.vformat(old, [], decision)
+
-        if new == old.replace('{{', '{').replace('}}', '}'):
+
-            return old
+class RepeatExpandingFormatter(string.Formatter):
-        if next(c) > 5:
+
-            raise RuntimeError
+    def __init__(self, _recursion=0):
-        old = new
+        super(RepeatExpandingFormatter, self).__init__()
+        # this class assumes it is always instantiated once per
+        # formatting; use that to detect runaway recursion
+        self._recursion = _recursion
+
+    def get_value(self, key, args, kwargs):
+        val = super(RepeatExpandingFormatter, self).get_value(key, args, kwargs)
+        if self._recursion > 5:
+            raise RecursionError(key)
+        fmt = self.__class__(_recursion=self._recursion+1)
+        # must use vformat not **kwargs so our SpecialVariables is not
+        # downgraded to just a dict
+        n = fmt.vformat(val, args, kwargs)
+        return n
+
 
 class SpecialVariables(dict):
     charsets = {
-        'binary': 'binary',
         'printable': string.printable,
         'punctuation': string.punctuation,
         'whitespace': string.whitespace,
@@ -131,7 +161,7 @@ class SpecialVariables(dict):
     }
 
     def __init__(self, orig_dict, prng):
-        self.update(orig_dict)
+        super(SpecialVariables, self).__init__(orig_dict)
         self.prng = prng
 
 
@@ -142,30 +172,38 @@ class SpecialVariables(dict):
             return super(SpecialVariables, self).__getitem__(key)
 
         if len(fields) == 1:
-            fields.apppend('')
+            fields.append('')
         return fn(fields[1])
 
 
     def special_random(self, args):
         arg_list = args.split()
         try:
-            size_min, size_max = [int(x) for x in arg_list[0].split('-')]
+            size_min, size_max = arg_list[0].split('-', 1)
+        except ValueError:
+            size_min = size_max = arg_list[0]
         except IndexError:
-            size_min = 0
+            size_min = '0'
-            size_max = 1000
+            size_max = '1000'
-        try:
-            charset = self.charsets[arg_list[1]]
-        except IndexError:
-            charset = self.charsets['printable']
 
+        size_min = int(size_min)
+        size_max = int(size_max)
         length = self.prng.randint(size_min, size_max)
-        if charset is 'binary':
+
+        try:
+            charset_arg = arg_list[1]
+        except IndexError:
+            charset_arg = 'printable'
+
+        if charset_arg == 'binary':
             num_bytes = length + 8
             tmplist = [self.prng.getrandbits(64) for _ in xrange(num_bytes / 8)]
             tmpstring = struct.pack((num_bytes / 8) * 'Q', *tmplist)
-            return tmpstring[0:length]
+            tmpstring = tmpstring[0:length]
         else:
-            tmpstring = ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely; won't do binary
+            charset = self.charsets[charset_arg]
+            tmpstring = ''.join([self.prng.choice(charset) for _ in xrange(length)]) # Won't scale nicely
+
         return tmpstring.replace('{', '{{').replace('}', '}}')
 
 
@@ -182,12 +220,11 @@ def parse_options():
     return parser.parse_args()
 
 
-def randomlist(n, seed=None):
+def randomlist(seed=None):
-    """ Returns a generator function that spits out a list of random numbers n elements long.
+    """ Returns an infinite generator of random numbers
     """
-    rng = random.Random()
+    rng = random.Random(seed)
-    rng.seed(seed if seed else None)
+    while True:
-    for _ in xrange(n):
         yield rng.random()
 
 
@@ -203,7 +240,9 @@ def _main():
         FH = open(options.seedfile, 'r')
         request_seeds = FH.readlines()
     else:
-        request_seeds = randomlist(options.num_requests, options.seed)
+        random_list = randomlist(options.seed)
+        request_seeds = itertools.islice(random_list, options.num_requests)
+
 
     graph_file = open(options.graph_filename, 'r')
     decision_graph = yaml.safe_load(graph_file)