Commit de0ea66b authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Add generic retry loop function

There are quite a few retry loops with timeouts in Ganeti's
code. Duplicating code is not good, so this patch introduces
a new function named “utils.Retry” to remedy this situation.
Signed-off-by: default avatarMichael Hanselmann <>
Reviewed-by: default avatarIustin Pop <>
parent 25231ec5
......@@ -2006,6 +2006,143 @@ def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
return value
class RetryTimeout(Exception):
"""Retry loop timed out.
class RetryAgain(Exception):
"""Retry again.
class _RetryDelayCalculator(object):
"""Calculator for increasing delays.
__slots__ = [
def __init__(self, start, factor, limit):
"""Initializes this class.
@type start: float
@param start: Initial delay
@type factor: float
@param factor: Factor for delay increase
@type limit: float or None
@param limit: Upper limit for delay or None for no limit
assert start > 0.0
assert factor >= 1.0
assert limit is None or limit >= 0.0
self._start = start
self._factor = factor
self._limit = limit
self._next = start
def __call__(self):
"""Returns current delay and calculates the next one.
current = self._next
# Update for next run
if self._limit is None or self._next < self._limit:
self._next = max(self._limit, self._next * self._factor)
return current
#: Special delay to specify whole remaining timeout
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
"""Call a function repeatedly until it succeeds.
The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
anymore. Between calls a delay, specified by C{delay}, is inserted. After a
total of C{timeout} seconds, this function throws L{RetryTimeout}.
C{delay} can be one of the following:
- callable returning the delay length as a float
- Tuple of (start, factor, limit)
- L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
useful when overriding L{wait_fn} to wait for an external event)
- A static delay as a number (int or float)
@type fn: callable
@param fn: Function to be called
@param delay: Either a callable (returning the delay), a tuple of (start,
factor, limit) (see L{_RetryDelayCalculator}),
L{RETRY_REMAINING_TIME} or a number (int or float)
@type timeout: float
@param timeout: Total timeout
@type wait_fn: callable
@param wait_fn: Waiting function
@return: Return value of function
assert callable(fn)
assert callable(wait_fn)
assert callable(_time_fn)
if args is None:
args = []
end_time = _time_fn() + timeout
if callable(delay):
# External function to calculate delay
calc_delay = delay
elif isinstance(delay, (tuple, list)):
# Increasing delay with optional upper boundary
(start, factor, limit) = delay
calc_delay = _RetryDelayCalculator(start, factor, limit)
# Always use the remaining time
calc_delay = None
# Static delay
calc_delay = lambda: delay
assert calc_delay is None or callable(calc_delay)
while True:
return fn(*args)
except RetryAgain:
remaining_time = end_time - _time_fn()
if remaining_time < 0.0:
raise RetryTimeout()
assert remaining_time >= 0.0
if calc_delay is None:
current_delay = calc_delay()
if current_delay > 0.0:
class FileLock(object):
"""Utility class for file locks.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment