forked from TrueCloudLab/s3-tests
Now that we no longer have the need to retry, get rid of that logic.
Also fix a typo -- s/neads_first_read/needs_first_read/
This commit is contained in:
parent
c154d98b96
commit
ce6c57841a
1 changed files with 7 additions and 32 deletions
|
@ -12,7 +12,7 @@ from ..realistic import FileVerifier
|
||||||
|
|
||||||
# Make sure context has somewhere to store what we need
|
# Make sure context has somewhere to store what we need
|
||||||
context.update(bunch.Bunch(
|
context.update(bunch.Bunch(
|
||||||
neads_first_read = collections.deque(),
|
needs_first_read = collections.deque(),
|
||||||
all_keys = [],
|
all_keys = [],
|
||||||
files_iter = None,
|
files_iter = None,
|
||||||
))
|
))
|
||||||
|
@ -23,22 +23,8 @@ class SafeTransferGreenlet(gevent.Greenlet):
|
||||||
gevent.Greenlet.__init__(self)
|
gevent.Greenlet.__init__(self)
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.result = None
|
self.result = None
|
||||||
self.key = None # We store key in case we ned to retry due to gevent being a jerk
|
|
||||||
|
|
||||||
def _run(self):
|
def _run(self):
|
||||||
""" A runner loop... using gevent creates a fun little bug where if two gevents try to
|
|
||||||
do the same op (reading, for ex), it raises an AssertionError rather than just switching
|
|
||||||
contexts again. Oh joy.
|
|
||||||
|
|
||||||
To combat this, we've put the main work to do in _real_run, which handles detecting the
|
|
||||||
gevent quirk, and we'll retry as long as _real_run requests that we retry, as indicated
|
|
||||||
by _real_run returning True.
|
|
||||||
"""
|
|
||||||
while self._real_run():
|
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
def _real_run(self):
|
|
||||||
""" Return True if we need to retry, False otherwise. """
|
|
||||||
result = self.result = TransferGreenletResult(self.type)
|
result = self.result = TransferGreenletResult(self.type)
|
||||||
result.markStarted()
|
result.markStarted()
|
||||||
|
|
||||||
|
@ -46,24 +32,19 @@ class SafeTransferGreenlet(gevent.Greenlet):
|
||||||
with gevent.Timeout(self.timeout, False):
|
with gevent.Timeout(self.timeout, False):
|
||||||
result.success = self._doit()
|
result.success = self._doit()
|
||||||
except gevent.GreenletExit:
|
except gevent.GreenletExit:
|
||||||
# We don't want to retry, as it's time to exit, but we also don't want to count
|
return
|
||||||
# this as a failure.
|
|
||||||
return False
|
|
||||||
except:
|
except:
|
||||||
result.setError(show_traceback=True)
|
result.setError(show_traceback=True)
|
||||||
|
|
||||||
result.markFinished()
|
result.markFinished()
|
||||||
return False # don't retry
|
|
||||||
|
|
||||||
|
|
||||||
class ReaderGreenlet(SafeTransferGreenlet):
|
class ReaderGreenlet(SafeTransferGreenlet):
|
||||||
type = 'reader'
|
type = 'reader'
|
||||||
|
|
||||||
def _doit(self):
|
def _doit(self):
|
||||||
if self.key:
|
if context.needs_first_read:
|
||||||
key = self.key
|
key = context.needs_first_read.popleft()
|
||||||
elif context.neads_first_read:
|
|
||||||
key = context.neads_first_read.popleft()
|
|
||||||
elif context.all_keys:
|
elif context.all_keys:
|
||||||
key = random.choice(context.all_keys)
|
key = random.choice(context.all_keys)
|
||||||
else:
|
else:
|
||||||
|
@ -72,7 +53,6 @@ class ReaderGreenlet(SafeTransferGreenlet):
|
||||||
|
|
||||||
# Copynew the key object
|
# Copynew the key object
|
||||||
key = key.bucket.new_key(key.name)
|
key = key.bucket.new_key(key.name)
|
||||||
self.key = key
|
|
||||||
self.result.setKey(key)
|
self.result.setKey(key)
|
||||||
|
|
||||||
fp = FileVerifier()
|
fp = FileVerifier()
|
||||||
|
@ -94,12 +74,7 @@ class WriterGreenlet(SafeTransferGreenlet):
|
||||||
type = 'writer'
|
type = 'writer'
|
||||||
|
|
||||||
def _doit(self):
|
def _doit(self):
|
||||||
if self.key:
|
key = get_next_key(context.bucket)
|
||||||
key = self.key
|
|
||||||
else:
|
|
||||||
key = get_next_key(context.bucket)
|
|
||||||
|
|
||||||
self.key = key
|
|
||||||
self.result.setKey(key)
|
self.result.setKey(key)
|
||||||
|
|
||||||
fp = next(context.files_iter)
|
fp = next(context.files_iter)
|
||||||
|
@ -111,8 +86,8 @@ class WriterGreenlet(SafeTransferGreenlet):
|
||||||
self.result.request_start = fp.start_time
|
self.result.request_start = fp.start_time
|
||||||
self.result.chunks = fp.last_chunks
|
self.result.chunks = fp.last_chunks
|
||||||
|
|
||||||
# And at the end, add to neads_first_read and shuffle
|
# And at the end, add to needs_first_read and shuffle
|
||||||
context.neads_first_read.append(key)
|
context.needs_first_read.append(key)
|
||||||
context.all_keys.append(key)
|
context.all_keys.append(key)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
Loading…
Reference in a new issue