Now that we no longer have the need to retry, get rid of that logic.

Also fix a typo -- s/neads_first_read/needs_first_read/
This commit is contained in:
Wesley Spikes 2011-07-18 16:09:13 -07:00
parent c154d98b96
commit ce6c57841a

View file

@ -12,7 +12,7 @@ from ..realistic import FileVerifier
# Make sure context has somewhere to store what we need # Make sure context has somewhere to store what we need
context.update(bunch.Bunch( context.update(bunch.Bunch(
neads_first_read = collections.deque(), needs_first_read = collections.deque(),
all_keys = [], all_keys = [],
files_iter = None, files_iter = None,
)) ))
@ -23,22 +23,8 @@ class SafeTransferGreenlet(gevent.Greenlet):
gevent.Greenlet.__init__(self) gevent.Greenlet.__init__(self)
self.timeout = timeout self.timeout = timeout
self.result = None self.result = None
self.key = None # We store key in case we ned to retry due to gevent being a jerk
def _run(self): def _run(self):
""" A runner loop... using gevent creates a fun little bug where if two gevents try to
do the same op (reading, for ex), it raises an AssertionError rather than just switching
contexts again. Oh joy.
To combat this, we've put the main work to do in _real_run, which handles detecting the
gevent quirk, and we'll retry as long as _real_run requests that we retry, as indicated
by _real_run returning True.
"""
while self._real_run():
time.sleep(0.1)
def _real_run(self):
""" Return True if we need to retry, False otherwise. """
result = self.result = TransferGreenletResult(self.type) result = self.result = TransferGreenletResult(self.type)
result.markStarted() result.markStarted()
@ -46,24 +32,19 @@ class SafeTransferGreenlet(gevent.Greenlet):
with gevent.Timeout(self.timeout, False): with gevent.Timeout(self.timeout, False):
result.success = self._doit() result.success = self._doit()
except gevent.GreenletExit: except gevent.GreenletExit:
# We don't want to retry, as it's time to exit, but we also don't want to count return
# this as a failure.
return False
except: except:
result.setError(show_traceback=True) result.setError(show_traceback=True)
result.markFinished() result.markFinished()
return False # don't retry
class ReaderGreenlet(SafeTransferGreenlet): class ReaderGreenlet(SafeTransferGreenlet):
type = 'reader' type = 'reader'
def _doit(self): def _doit(self):
if self.key: if context.needs_first_read:
key = self.key key = context.needs_first_read.popleft()
elif context.neads_first_read:
key = context.neads_first_read.popleft()
elif context.all_keys: elif context.all_keys:
key = random.choice(context.all_keys) key = random.choice(context.all_keys)
else: else:
@ -72,7 +53,6 @@ class ReaderGreenlet(SafeTransferGreenlet):
# Copynew the key object # Copynew the key object
key = key.bucket.new_key(key.name) key = key.bucket.new_key(key.name)
self.key = key
self.result.setKey(key) self.result.setKey(key)
fp = FileVerifier() fp = FileVerifier()
@ -94,12 +74,7 @@ class WriterGreenlet(SafeTransferGreenlet):
type = 'writer' type = 'writer'
def _doit(self): def _doit(self):
if self.key: key = get_next_key(context.bucket)
key = self.key
else:
key = get_next_key(context.bucket)
self.key = key
self.result.setKey(key) self.result.setKey(key)
fp = next(context.files_iter) fp = next(context.files_iter)
@ -111,8 +86,8 @@ class WriterGreenlet(SafeTransferGreenlet):
self.result.request_start = fp.start_time self.result.request_start = fp.start_time
self.result.chunks = fp.last_chunks self.result.chunks = fp.last_chunks
# And at the end, add to neads_first_read and shuffle # And at the end, add to needs_first_read and shuffle
context.neads_first_read.append(key) context.needs_first_read.append(key)
context.all_keys.append(key) context.all_keys.append(key)
return True return True