From 440038947c6ae59be06457985771dba2b907a8ba Mon Sep 17 00:00:00 2001 From: Boden R Date: Fri, 27 May 2016 09:22:02 -0600 Subject: [PATCH 1/4] Support for retry event hook It's very common for consumers to perform some action on each attempt iteration; for example logging a message that the operation failed and a retry will be performed after some time. While this can be done today using a custom wait_func, it's inconvenient to use a partial to wrap an existing retrying sleep function just to get this behavior. This patch adds support for a new kwarg called wait_event_func that when passed should reference a function to be called before sleeping for another attempt iteration. This function looks similar to retrying wait_funcs except its first arg is the time to sleep as returned by the current 'wait' function. A handful of unit tests are also included. --- AUTHORS.rst | 3 ++- README.rst | 13 +++++++++++++ retrying.py | 5 ++++- test_retrying.py | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9cb572f..164ca2b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -30,4 +30,5 @@ Patches and Suggestions - Monty Taylor - Maxym Shalenyi - Jonathan Herriott -- Job Evers \ No newline at end of file +- Job Evers +- Boden Russell \ No newline at end of file diff --git a/README.rst b/README.rst index ab5d6bd..642fdc1 100644 --- a/README.rst +++ b/README.rst @@ -144,6 +144,19 @@ We can also use the result of the function to alter the behavior of retrying. Any combination of stop, wait, etc. is also supported to give you the freedom to mix and match. +It's also possible to provide a custom hook function that'll be called on each re-attempt +of a retry prior to the next wait. The return value of this hook is not used. + +.. code-block:: python + + def log_retry(next_wait_time, prev_attempt, time_since_first_attempt): + log.warning("Operation failed. Trying again in %s ms" % next_wait_time) + + @retry(wait_event_func=log_retry) + def logged_on_each_retry(): + my_operation() + + Contribute ---------- diff --git a/retrying.py b/retrying.py index 3ed312d..46e2a6d 100644 --- a/retrying.py +++ b/retrying.py @@ -68,7 +68,8 @@ def __init__(self, wrap_exception=False, stop_func=None, wait_func=None, - wait_jitter_max=None): + wait_jitter_max=None, + wait_event_func=None): self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay @@ -80,6 +81,7 @@ def __init__(self, self._wait_exponential_multiplier = 1 if wait_exponential_multiplier is None else wait_exponential_multiplier self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max + self._wait_event_func = wait_event_func if wait_event_func is not None else lambda next_wait, attempts, delay: None # TODO add chaining of stop behaviors # stop behavior @@ -214,6 +216,7 @@ def call(self, fn, *args, **kwargs): raise RetryError(attempt) else: sleep = self.wait(attempt_number, delay_since_first_attempt_ms) + self._wait_event_func(sleep, attempt_number, delay_since_first_attempt_ms) if self._wait_jitter_max: jitter = random.random() * self._wait_jitter_max sleep = sleep + max(0, jitter) diff --git a/test_retrying.py b/test_retrying.py index bfef02d..af1a1fb 100644 --- a/test_retrying.py +++ b/test_retrying.py @@ -434,5 +434,45 @@ def test_defaults(self): self.assertTrue(_retryable_default(NoCustomErrorAfterCount(5))) self.assertTrue(_retryable_default_f(NoCustomErrorAfterCount(5))) + +class TestRetryEvent(unittest.TestCase): + + def setUp(self): + self._event_calls = [] + + def _mark_event_call(self, sleep_time, prev_attempt, since_first): + self._event_calls.append((sleep_time, prev_attempt, since_first)) + + def test_custom_wait_hook_init(self): + retrying = Retrying(wait_event_func=self._mark_event_call) + self.assertEqual(self._mark_event_call, retrying._wait_event_func) + + def test_custom_wait_hook_call(self): + + @retry(wait_event_func=self._mark_event_call, stop_max_attempt_number=4, wait_fixed=10) + def wait_with_events(): + raise Exception() + + self.assertRaises(Exception, wait_with_events) + self.assertEqual(3, len(self._event_calls)) + for i in range(0, 3): + self.assertEqual(10, self._event_calls[i][0]) + self.assertEqual(i + 1, self._event_calls[i][1]) + + def test_no_custom_wait_hook_call(self): + + @retry(stop_max_attempt_number=4, wait_fixed=10) + def wait_with_events(): + raise Exception() + + self.assertRaises(Exception, wait_with_events) + self.assertEqual(0, len(self._event_calls)) + + def test_default_wait_hook_init(self): + retrying = Retrying() + self.assertIsNotNone(retrying._wait_event_func) + self.assertNotEqual(self._mark_event_call, retrying._wait_event_func) + + if __name__ == '__main__': unittest.main() From 23ce6743d8c2fbbcb9cac669b3e0e1374aea948c Mon Sep 17 00:00:00 2001 From: Boden R Date: Fri, 27 May 2016 09:22:02 -0600 Subject: [PATCH 2/4] Support for retry event hook It's very common for consumers to perform some action on each attempt iteration; for example logging a message that the operation failed and a retry will be performed after some time. While this can be done today using a custom wait_func, it's inconvenient to use a partial to wrap an existing retrying sleep function just to get this behavior. This patch adds support for a new kwarg called wait_event_func that when passed should reference a function to be called before sleeping for another attempt iteration. This function looks similar to retrying wait_funcs except its first arg is the time to sleep as returned by the current 'wait' function. A handful of unit tests are also included. --- AUTHORS.rst | 3 ++- README.rst | 13 +++++++++++++ retrying.py | 5 ++++- test_retrying.py | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9cb572f..164ca2b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -30,4 +30,5 @@ Patches and Suggestions - Monty Taylor - Maxym Shalenyi - Jonathan Herriott -- Job Evers \ No newline at end of file +- Job Evers +- Boden Russell \ No newline at end of file diff --git a/README.rst b/README.rst index ab5d6bd..642fdc1 100644 --- a/README.rst +++ b/README.rst @@ -144,6 +144,19 @@ We can also use the result of the function to alter the behavior of retrying. Any combination of stop, wait, etc. is also supported to give you the freedom to mix and match. +It's also possible to provide a custom hook function that'll be called on each re-attempt +of a retry prior to the next wait. The return value of this hook is not used. + +.. code-block:: python + + def log_retry(next_wait_time, prev_attempt, time_since_first_attempt): + log.warning("Operation failed. Trying again in %s ms" % next_wait_time) + + @retry(wait_event_func=log_retry) + def logged_on_each_retry(): + my_operation() + + Contribute ---------- diff --git a/retrying.py b/retrying.py index 3ed312d..46e2a6d 100644 --- a/retrying.py +++ b/retrying.py @@ -68,7 +68,8 @@ def __init__(self, wrap_exception=False, stop_func=None, wait_func=None, - wait_jitter_max=None): + wait_jitter_max=None, + wait_event_func=None): self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay @@ -80,6 +81,7 @@ def __init__(self, self._wait_exponential_multiplier = 1 if wait_exponential_multiplier is None else wait_exponential_multiplier self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max + self._wait_event_func = wait_event_func if wait_event_func is not None else lambda next_wait, attempts, delay: None # TODO add chaining of stop behaviors # stop behavior @@ -214,6 +216,7 @@ def call(self, fn, *args, **kwargs): raise RetryError(attempt) else: sleep = self.wait(attempt_number, delay_since_first_attempt_ms) + self._wait_event_func(sleep, attempt_number, delay_since_first_attempt_ms) if self._wait_jitter_max: jitter = random.random() * self._wait_jitter_max sleep = sleep + max(0, jitter) diff --git a/test_retrying.py b/test_retrying.py index bfef02d..da4ab61 100644 --- a/test_retrying.py +++ b/test_retrying.py @@ -434,5 +434,45 @@ def test_defaults(self): self.assertTrue(_retryable_default(NoCustomErrorAfterCount(5))) self.assertTrue(_retryable_default_f(NoCustomErrorAfterCount(5))) + +class TestRetryEvent(unittest.TestCase): + + def setUp(self): + self._event_calls = [] + + def _mark_event_call(self, sleep_time, prev_attempt, since_first): + self._event_calls.append((sleep_time, prev_attempt, since_first)) + + def test_custom_wait_hook_init(self): + retrying = Retrying(wait_event_func=self._mark_event_call) + self.assertEqual(self._mark_event_call, retrying._wait_event_func) + + def test_custom_wait_hook_call(self): + + @retry(wait_event_func=self._mark_event_call, stop_max_attempt_number=4, wait_fixed=10) + def wait_with_events(): + raise Exception() + + self.assertRaises(Exception, wait_with_events) + self.assertEqual(3, len(self._event_calls)) + for i in range(0, 3): + self.assertEqual(10, self._event_calls[i][0]) + self.assertEqual(i + 1, self._event_calls[i][1]) + + def test_no_custom_wait_hook_call(self): + + @retry(stop_max_attempt_number=4, wait_fixed=10) + def wait_with_events(): + raise Exception() + + self.assertRaises(Exception, wait_with_events) + self.assertEqual(0, len(self._event_calls)) + + def test_default_wait_hook_init(self): + retrying = Retrying() + self.assertTrue(retrying._wait_event_func is not None) + self.assertNotEqual(self._mark_event_call, retrying._wait_event_func) + + if __name__ == '__main__': unittest.main() From 6e1f680f1a8748ec3675abee358a0d394a275502 Mon Sep 17 00:00:00 2001 From: boden Date: Fri, 27 May 2016 11:31:32 -0600 Subject: [PATCH 3/4] fix py2.6 unit test issue No assertIsNotNone() in py26 so change to a py26 compatible assert statement. --- test_retrying.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_retrying.py b/test_retrying.py index af1a1fb..355172b 100644 --- a/test_retrying.py +++ b/test_retrying.py @@ -470,7 +470,7 @@ def wait_with_events(): def test_default_wait_hook_init(self): retrying = Retrying() - self.assertIsNotNone(retrying._wait_event_func) + self.assertTrue(retrying._wait_event_func) self.assertNotEqual(self._mark_event_call, retrying._wait_event_func) From a76f32e4c0928a53ff1ed4b067cf5abd74044e90 Mon Sep 17 00:00:00 2001 From: Boden R Date: Thu, 16 Jun 2016 10:09:25 -0600 Subject: [PATCH 4/4] Use composable wait functions --- README.rst | 12 ----------- retrying.py | 55 ++++++++++++++++++++++++++++++++++++------------ test_retrying.py | 38 --------------------------------- 3 files changed, 41 insertions(+), 64 deletions(-) diff --git a/README.rst b/README.rst index 642fdc1..bdfe4d1 100644 --- a/README.rst +++ b/README.rst @@ -144,18 +144,6 @@ We can also use the result of the function to alter the behavior of retrying. Any combination of stop, wait, etc. is also supported to give you the freedom to mix and match. -It's also possible to provide a custom hook function that'll be called on each re-attempt -of a retry prior to the next wait. The return value of this hook is not used. - -.. code-block:: python - - def log_retry(next_wait_time, prev_attempt, time_since_first_attempt): - log.warning("Operation failed. Trying again in %s ms" % next_wait_time) - - @retry(wait_event_func=log_retry) - def logged_on_each_retry(): - my_operation() - Contribute ---------- diff --git a/retrying.py b/retrying.py index 2805d96..3302aed 100644 --- a/retrying.py +++ b/retrying.py @@ -12,6 +12,7 @@ ## See the License for the specific language governing permissions and ## limitations under the License. +import inspect import random import six import sys @@ -77,8 +78,7 @@ def __init__(self, wait_func=None, wait_jitter_max=None, before_attempts=None, - after_attempts=None, - wait_event_func=None): + after_attempts=None): self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay @@ -91,7 +91,6 @@ def __init__(self, self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max self._wait_incrementing_max = MAX_WAIT if wait_incrementing_max is None else wait_incrementing_max self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max - self._wait_event_func = wait_event_func if wait_event_func is not None else lambda next_wait, attempts, delay: None self._before_attempts = before_attempts self._after_attempts = after_attempts @@ -113,29 +112,29 @@ def __init__(self, else: self.stop = getattr(self, stop) - # TODO add chaining of wait behaviors - # wait behavior - wait_funcs = [lambda *args, **kwargs: 0] + wait_funcs = CallChain(lambda *args, **kwargs: 0) if wait_fixed is not None: - wait_funcs.append(self.fixed_sleep) + wait_funcs += self.fixed_sleep if wait_random_min is not None or wait_random_max is not None: - wait_funcs.append(self.random_sleep) + wait_funcs += self.random_sleep if wait_incrementing_start is not None or wait_incrementing_increment is not None: - wait_funcs.append(self.incrementing_sleep) + wait_funcs += self.incrementing_sleep if wait_exponential_multiplier is not None or wait_exponential_max is not None: - wait_funcs.append(self.exponential_sleep) + wait_funcs += self.exponential_sleep if wait_func is not None: - self.wait = wait_func + wait_funcs += wait_func elif wait is None: - self.wait = lambda attempts, delay: max(f(attempts, delay) for f in wait_funcs) + wait_funcs += lambda attempts, delay, chain_results=None: max(chain_results) else: - self.wait = getattr(self, wait) + wait_funcs += getattr(self, wait) + + self.wait = wait_funcs # retry on exception filter if retry_on_exception is None: @@ -244,7 +243,6 @@ def call(self, fn, *args, **kwargs): raise RetryError(attempt) else: sleep = self.wait(attempt_number, delay_since_first_attempt_ms) - self._wait_event_func(sleep, attempt_number, delay_since_first_attempt_ms) if self._wait_jitter_max: jitter = random.random() * self._wait_jitter_max sleep = sleep + max(0, jitter) @@ -286,6 +284,35 @@ def __repr__(self): return "Attempts: {0}, Value: {1}".format(self.attempt_number, self.value) +class CallChain(object): + + def __init__(self, *fns): + self._chain = [] + for fn in fns: + self._assert_callable(fn) + self._chain.append(fn) + + def __add__(self, other): + self._assert_callable(object) + self._chain.append(other) + return self + + def __call__(self, *args, **kwargs): + results = [] + for fn in self._chain: + if 'chain_results' in inspect.getargspec(fn).args: + fn_kwargs = dict(kwargs) + fn_kwargs['chain_results'] = results + results.append(fn(*args, **fn_kwargs)) + else: + results.append(fn(*args, **kwargs)) + return results[-1] if results else None + + def _assert_callable(self, fn): + if not hasattr(fn, '__call__'): + raise TypeError("'%s' is not callable" % fn) + + class RetryError(Exception): """ A RetryError encapsulates the last Attempt instance right before giving up. diff --git a/test_retrying.py b/test_retrying.py index 2d14012..4ac634c 100644 --- a/test_retrying.py +++ b/test_retrying.py @@ -470,43 +470,5 @@ def _test_after(): self.assertTrue(TestBeforeAfterAttempts._attempt_number is 2) -class TestRetryEvent(unittest.TestCase): - - def setUp(self): - self._event_calls = [] - - def _mark_event_call(self, sleep_time, prev_attempt, since_first): - self._event_calls.append((sleep_time, prev_attempt, since_first)) - - def test_custom_wait_hook_init(self): - retrying = Retrying(wait_event_func=self._mark_event_call) - self.assertEqual(self._mark_event_call, retrying._wait_event_func) - - def test_custom_wait_hook_call(self): - - @retry(wait_event_func=self._mark_event_call, stop_max_attempt_number=4, wait_fixed=10) - def wait_with_events(): - raise Exception() - - self.assertRaises(Exception, wait_with_events) - self.assertEqual(3, len(self._event_calls)) - for i in range(0, 3): - self.assertEqual(10, self._event_calls[i][0]) - self.assertEqual(i + 1, self._event_calls[i][1]) - - def test_no_custom_wait_hook_call(self): - - @retry(stop_max_attempt_number=4, wait_fixed=10) - def wait_with_events(): - raise Exception() - - self.assertRaises(Exception, wait_with_events) - self.assertEqual(0, len(self._event_calls)) - - def test_default_wait_hook_init(self): - retrying = Retrying() - self.assertTrue(retrying._wait_event_func is not None) - self.assertNotEqual(self._mark_event_call, retrying._wait_event_func) - if __name__ == '__main__': unittest.main()