Add generic retry loop function

There are quite a few retry loops with timeouts in Ganeti's
code. Duplicating code is not good, so this patch introduces
a new function named “utils.Retry” to remedy this situation.

Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>

lib/utils.py

@@ -2006,6 +2006,143 @@ def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
   return value
 
 
+class RetryTimeout(Exception):
+  """Retry loop timed out.
+
+  """
+
+
+class RetryAgain(Exception):
+  """Retry again.
+
+  """
+
+
+class _RetryDelayCalculator(object):
+  """Calculator for increasing delays.
+
+  """
+  __slots__ = [
+    "_factor",
+    "_limit",
+    "_next",
+    "_start",
+    ]
+
+  def __init__(self, start, factor, limit):
+    """Initializes this class.
+
+    @type start: float
+    @param start: Initial delay
+    @type factor: float
+    @param factor: Factor for delay increase
+    @type limit: float or None
+    @param limit: Upper limit for delay or None for no limit
+
+    """
+    assert start > 0.0
+    assert factor >= 1.0
+    assert limit is None or limit >= 0.0
+
+    self._start = start
+    self._factor = factor
+    self._limit = limit
+
+    self._next = start
+
+  def __call__(self):
+    """Returns current delay and calculates the next one.
+
+    """
+    current = self._next
+
+    # Update for next run
+    if self._limit is None or self._next < self._limit:
+      self._next = max(self._limit, self._next * self._factor)
+
+    return current
+
+
+#: Special delay to specify whole remaining timeout
+RETRY_REMAINING_TIME = object()
+
+
+def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
+          _time_fn=time.time):
+  """Call a function repeatedly until it succeeds.
+
+  The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
+  anymore. Between calls a delay, specified by C{delay}, is inserted. After a
+  total of C{timeout} seconds, this function throws L{RetryTimeout}.
+
+  C{delay} can be one of the following:
+    - callable returning the delay length as a float
+    - Tuple of (start, factor, limit)
+    - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
+      useful when overriding L{wait_fn} to wait for an external event)
+    - A static delay as a number (int or float)
+
+  @type fn: callable
+  @param fn: Function to be called
+  @param delay: Either a callable (returning the delay), a tuple of (start,
+                factor, limit) (see L{_RetryDelayCalculator}),
+                L{RETRY_REMAINING_TIME} or a number (int or float)
+  @type timeout: float
+  @param timeout: Total timeout
+  @type wait_fn: callable
+  @param wait_fn: Waiting function
+  @return: Return value of function
+
+  """
+  assert callable(fn)
+  assert callable(wait_fn)
+  assert callable(_time_fn)
+
+  if args is None:
+    args = []
+
+  end_time = _time_fn() + timeout
+
+  if callable(delay):
+    # External function to calculate delay
+    calc_delay = delay
+
+  elif isinstance(delay, (tuple, list)):
+    # Increasing delay with optional upper boundary
+    (start, factor, limit) = delay
+    calc_delay = _RetryDelayCalculator(start, factor, limit)
+
+  elif delay is RETRY_REMAINING_TIME:
+    # Always use the remaining time
+    calc_delay = None
+
+  else:
+    # Static delay
+    calc_delay = lambda: delay
+
+  assert calc_delay is None or callable(calc_delay)
+
+  while True:
+    try:
+      return fn(*args)
+    except RetryAgain:
+      pass
+
+    remaining_time = end_time - _time_fn()
+
+    if remaining_time < 0.0:
+      raise RetryTimeout()
+
+    assert remaining_time >= 0.0
+
+    if calc_delay is None:
+      wait_fn(remaining_time)
+    else:
+      current_delay = calc_delay()
+      if current_delay > 0.0:
+        wait_fn(current_delay)
+
+
 class FileLock(object):
   """Utility class for file locks.