Merge pull request #142 from mdw-at-linuxbox/hammer-setupfix

Use non-broken version of setuptools, and fix bootstrap to be more po…

bootstrap

@@ -13,9 +13,18 @@ if [ -f /etc/debian_version ]; then
         echo "  sudo apt-get install $missing"
         exit 1
     fi
-fi
+elif [ -f /etc/fedora-release ]; then
-if [ -f /etc/redhat-release ]; then
+    for package in python-pip python2-virtualenv python-devel libevent-devel; do
-    for package in python-pip python-virtualenv python-devel libevent-devel; do
+        if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then
+            missing="${missing:+$missing }$package"
+        fi
+    done
+    if [ -n "$missing" ]; then
+        echo "$0: missing required RPM packages. Installing via sudo." 1>&2
+        sudo yum -y install $missing
+    fi
+elif [ -f /etc/redhat-release ]; then
+    for package in python2-pip python-virtualenv python-devel libevent-devel; do
         if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then
             missing="${missing:+$missing }$package"
         fi
@@ -32,8 +41,8 @@ virtualenv --no-site-packages --distribute virtualenv
 # avoid pip bugs
 ./virtualenv/bin/pip install --upgrade pip
 
-# work-around change in pip 1.5
+# slightly old version of setuptools; newer fails w/ requests 0.14.0
-./virtualenv/bin/pip install setuptools --no-use-wheel --upgrade
+./virtualenv/bin/pip install setuptools==32.3.1
 
 ./virtualenv/bin/pip install -r requirements.txt
 
@@ -41,5 +50,4 @@ virtualenv --no-site-packages --distribute virtualenv
 # easy_install, and we really wanted pip; next line will fail if pip
 # requirements.txt does not match setup.py requirements -- sucky but
 # good enough for now
-./virtualenv/bin/python setup.py develop \
+./virtualenv/bin/python setup.py develop
-    --allow-hosts None

s3tests/functional/__init__.py

@@ -64,22 +64,31 @@ def nuke_prefixed_buckets_on_conn(prefix, name, conn):
         print 'prefix=',prefix
         if bucket.name.startswith(prefix):
             print 'Cleaning bucket {bucket}'.format(bucket=bucket)
-            try:
+            success = False
-                # bucket.set_canned_acl('private')
+            for i in xrange(2):
-                for key in bucket.list_versions():
+                try:
-                    print 'Cleaning bucket {bucket} key {key}'.format(
+                    for key in bucket.list_versions():
-                        bucket=bucket,
+                        print 'Cleaning bucket {bucket} key {key}'.format(
-                        key=key,
+                            bucket=bucket,
-                        )
+                            key=key,
-                    # key.set_canned_acl('private')
+                            )
-                    bucket.delete_key(key.name, version_id = key.version_id)
+                        # key.set_canned_acl('private')
-                bucket.delete()
+                        bucket.delete_key(key.name, version_id = key.version_id)
-            except boto.exception.S3ResponseError as e:
+                    bucket.delete()
-                if e.error_code != 'AccessDenied':
+                    success = True
-                    print 'GOT UNWANTED ERROR', e.error_code
+                except boto.exception.S3ResponseError as e:
-                    raise
+                    if e.error_code != 'AccessDenied':
-                # seems like we're not the owner of the bucket; ignore
+                        print 'GOT UNWANTED ERROR', e.error_code
-                pass
+                        raise
+                    # seems like we don't have permissions set appropriately, we'll
+                    # modify permissions and retry
+                    pass
+
+                if success:
+                    return
+
+                bucket.set_canned_acl('private')
+
 
 def nuke_prefixed_buckets(prefix):
     # If no regions are specified, use the simple method
@@ -106,7 +115,6 @@ def nuke_prefixed_buckets(prefix):
 
     print 'Done with cleanup of test buckets.'
 
-
 class TargetConfig:
     def __init__(self, cfg, section):
         self.port = None

s3tests/functional/test_s3.py

@@ -3261,6 +3261,27 @@ def test_object_acl_canned_bucketownerfullcontrol():
     key.delete()
     bucket.delete()
 
+@attr(resource='object.acls')
+@attr(method='put')
+@attr(operation='set write-acp')
+@attr(assertion='does not modify owner')
+def test_object_acl_full_control_verify_owner():
+    bucket = get_new_bucket(targets.main.default)
+    bucket.set_acl('public-read-write')
+
+    key = bucket.new_key('foo')
+    key.set_contents_from_string('bar')
+
+    key.add_user_grant(permission='FULL_CONTROL', user_id=config.alt.user_id)
+
+    k2 = s3.alt.get_bucket(bucket.name).get_key('foo')
+
+    k2.add_user_grant(permission='READ_ACP', user_id=config.alt.user_id)
+
+    policy = k2.get_acl()
+    eq(policy.owner.id, config.main.user_id)
+
+
 @attr(resource='bucket')
 @attr(method='ACLs')
 @attr(operation='set acl private')
@@ -3532,6 +3553,12 @@ def test_bucket_acl_grant_userid_fullcontrol():
     # can write acl
     _check_bucket_acl_grant_can_writeacp(bucket)
 
+    # verify owner did not change
+    bucket2 = s3.main.get_bucket(bucket.name)
+    policy = bucket2.get_acl()
+    eq(policy.owner.id, config.main.user_id)
+    eq(policy.owner.display_name, config.main.display_name)
+
 
 @attr(resource='bucket')
 @attr(method='ACLs')
@@ -5762,7 +5789,7 @@ def test_versioning_multi_object_delete_with_marker_create():
 
         keyname = 'key'
 
-        rmkeys = { bucket.new_key(keyname) }
+        rmkeys = [ bucket.new_key(keyname) ]
 
         eq(_count_bucket_versioned_objs(bucket), 0)
 
@@ -5852,17 +5879,18 @@ def test_versioned_object_acl():
     check_grants(k.get_acl().acl.grants, default_policy)
 
 
-def _do_create_object(bucket, objname, i):
+def _do_create_object(bucket, objname, i, obj_size):
     k = bucket.new_key(objname)
-    k.set_contents_from_string('data {i}'.format(i=i))
+    s = 'x' * obj_size
+    k.set_contents_from_string(s)
 
 def _do_remove_ver(bucket, obj):
     bucket.delete_key(obj.name, version_id = obj.version_id)
 
-def _do_create_versioned_obj_concurrent(bucket, objname, num):
+def _do_create_obj_concurrent(bucket, objname, num, obj_size=0):
     t = []
     for i in range(num):
-        thr = threading.Thread(target = _do_create_object, args=(bucket, objname, i))
+        thr = threading.Thread(target = _do_create_object, args=(bucket, objname, i, obj_size))
         thr.start()
         t.append(thr)
     return t
@@ -5894,7 +5922,7 @@ def test_versioned_concurrent_object_create_concurrent_remove():
     num_objs = 5
 
     for i in xrange(5):
-        t = _do_create_versioned_obj_concurrent(bucket, keyname, num_objs)
+        t = _do_create_obj_concurrent(bucket, keyname, num_objs)
         _do_wait_completion(t)
 
         eq(_count_bucket_versioned_objs(bucket), num_objs)
@@ -5923,7 +5951,7 @@ def test_versioned_concurrent_object_create_and_remove():
     all_threads = []
 
     for i in xrange(3):
-        t = _do_create_versioned_obj_concurrent(bucket, keyname, num_objs)
+        t = _do_create_obj_concurrent(bucket, keyname, num_objs)
         all_threads.append(t)
 
         t = _do_clear_versioned_bucket_concurrent(bucket)
@@ -5938,3 +5966,36 @@ def test_versioned_concurrent_object_create_and_remove():
 
     eq(_count_bucket_versioned_objs(bucket), 0)
     eq(len(bucket.get_all_keys()), 0)
+
+@attr(resource='object')
+@attr(method='put')
+@attr(operation='concurrent creation of objects, concurrent removal')
+@attr(assertion='works')
+def test_non_versioned_concurrent_object_create_concurrent_remove():
+    bucket = get_new_bucket()
+
+    # not configuring versioning here! this test is non-versioned
+
+    keyname = 'myobj'
+
+    num_objs = 5
+    obj_size = 10 # non-zero
+
+    for i in xrange(5):
+        t = _do_create_obj_concurrent(bucket, keyname, num_objs, obj_size)
+        _do_wait_completion(t)
+
+        keys = []
+        for k in bucket.get_all_keys():
+            keys.append(k)
+        eq(len(keys), 1)
+        eq(keys[0].size, obj_size)
+
+
+        t = _do_clear_versioned_bucket_concurrent(bucket)
+        _do_wait_completion(t)
+
+        eq(_count_bucket_versioned_objs(bucket), 0)
+        eq(len(bucket.get_all_keys()), 0)
+
+