Commit 79d22269 authored by Michael Hanselmann's avatar Michael Hanselmann

utils: Split Retry & co. into separate file

Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent 4fd029cf
......@@ -213,7 +213,8 @@ server_PYTHON = \
utils_PYTHON = \
lib/utils/__init__.py \
lib/utils/algo.py
lib/utils/algo.py \
lib/utils/retry.py
docrst = \
doc/admin.rst \
......@@ -480,6 +481,7 @@ python_tests = \
test/ganeti.ssh_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils.algo_unittest.py \
test/ganeti.utils.retry_unittest.py \
test/ganeti.utils_mlockall_unittest.py \
test/ganeti.utils_unittest.py \
test/ganeti.workerpool_unittest.py \
......
......@@ -63,6 +63,7 @@ from ganeti import constants
from ganeti import compat
from ganeti.utils.algo import * # pylint: disable-msg=W0401
from ganeti.utils.retry import * # pylint: disable-msg=W0401
_locksheld = []
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
......@@ -3353,162 +3354,6 @@ def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
return value
class RetryTimeout(Exception):
"""Retry loop timed out.
Any arguments which was passed by the retried function to RetryAgain will be
preserved in RetryTimeout, if it is raised. If such argument was an exception
the RaiseInner helper method will reraise it.
"""
def RaiseInner(self):
if self.args and isinstance(self.args[0], Exception):
raise self.args[0]
else:
raise RetryTimeout(*self.args)
class RetryAgain(Exception):
"""Retry again.
Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
of the RetryTimeout() method can be used to reraise it.
"""
class _RetryDelayCalculator(object):
"""Calculator for increasing delays.
"""
__slots__ = [
"_factor",
"_limit",
"_next",
"_start",
]
def __init__(self, start, factor, limit):
"""Initializes this class.
@type start: float
@param start: Initial delay
@type factor: float
@param factor: Factor for delay increase
@type limit: float or None
@param limit: Upper limit for delay or None for no limit
"""
assert start > 0.0
assert factor >= 1.0
assert limit is None or limit >= 0.0
self._start = start
self._factor = factor
self._limit = limit
self._next = start
def __call__(self):
"""Returns current delay and calculates the next one.
"""
current = self._next
# Update for next run
if self._limit is None or self._next < self._limit:
self._next = min(self._limit, self._next * self._factor)
return current
#: Special delay to specify whole remaining timeout
RETRY_REMAINING_TIME = object()
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
_time_fn=time.time):
"""Call a function repeatedly until it succeeds.
The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
anymore. Between calls a delay, specified by C{delay}, is inserted. After a
total of C{timeout} seconds, this function throws L{RetryTimeout}.
C{delay} can be one of the following:
- callable returning the delay length as a float
- Tuple of (start, factor, limit)
- L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
useful when overriding L{wait_fn} to wait for an external event)
- A static delay as a number (int or float)
@type fn: callable
@param fn: Function to be called
@param delay: Either a callable (returning the delay), a tuple of (start,
factor, limit) (see L{_RetryDelayCalculator}),
L{RETRY_REMAINING_TIME} or a number (int or float)
@type timeout: float
@param timeout: Total timeout
@type wait_fn: callable
@param wait_fn: Waiting function
@return: Return value of function
"""
assert callable(fn)
assert callable(wait_fn)
assert callable(_time_fn)
if args is None:
args = []
end_time = _time_fn() + timeout
if callable(delay):
# External function to calculate delay
calc_delay = delay
elif isinstance(delay, (tuple, list)):
# Increasing delay with optional upper boundary
(start, factor, limit) = delay
calc_delay = _RetryDelayCalculator(start, factor, limit)
elif delay is RETRY_REMAINING_TIME:
# Always use the remaining time
calc_delay = None
else:
# Static delay
calc_delay = lambda: delay
assert calc_delay is None or callable(calc_delay)
while True:
retry_args = []
try:
# pylint: disable-msg=W0142
return fn(*args)
except RetryAgain, err:
retry_args = err.args
except RetryTimeout:
raise errors.ProgrammerError("Nested retry loop detected that didn't"
" handle RetryTimeout")
remaining_time = end_time - _time_fn()
if remaining_time < 0.0:
# pylint: disable-msg=W0142
raise RetryTimeout(*retry_args)
assert remaining_time >= 0.0
if calc_delay is None:
wait_fn(remaining_time)
else:
current_delay = calc_delay()
if current_delay > 0.0:
wait_fn(current_delay)
def GetClosedTempfile(*args, **kwargs):
"""Creates a temporary file and returns its path.
......
#
#
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Utility functions for retrying function calls with a timeout.
"""
import time
from ganeti import errors
#: Special delay to specify whole remaining timeout
RETRY_REMAINING_TIME = object()
class RetryTimeout(Exception):
"""Retry loop timed out.
Any arguments which was passed by the retried function to RetryAgain will be
preserved in RetryTimeout, if it is raised. If such argument was an exception
the RaiseInner helper method will reraise it.
"""
def RaiseInner(self):
if self.args and isinstance(self.args[0], Exception):
raise self.args[0]
else:
raise RetryTimeout(*self.args)
class RetryAgain(Exception):
"""Retry again.
Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
of the RetryTimeout() method can be used to reraise it.
"""
class _RetryDelayCalculator(object):
"""Calculator for increasing delays.
"""
__slots__ = [
"_factor",
"_limit",
"_next",
"_start",
]
def __init__(self, start, factor, limit):
"""Initializes this class.
@type start: float
@param start: Initial delay
@type factor: float
@param factor: Factor for delay increase
@type limit: float or None
@param limit: Upper limit for delay or None for no limit
"""
assert start > 0.0
assert factor >= 1.0
assert limit is None or limit >= 0.0
self._start = start
self._factor = factor
self._limit = limit
self._next = start
def __call__(self):
"""Returns current delay and calculates the next one.
"""
current = self._next
# Update for next run
if self._limit is None or self._next < self._limit:
self._next = min(self._limit, self._next * self._factor)
return current
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
_time_fn=time.time):
"""Call a function repeatedly until it succeeds.
The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
anymore. Between calls a delay, specified by C{delay}, is inserted. After a
total of C{timeout} seconds, this function throws L{RetryTimeout}.
C{delay} can be one of the following:
- callable returning the delay length as a float
- Tuple of (start, factor, limit)
- L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
useful when overriding L{wait_fn} to wait for an external event)
- A static delay as a number (int or float)
@type fn: callable
@param fn: Function to be called
@param delay: Either a callable (returning the delay), a tuple of (start,
factor, limit) (see L{_RetryDelayCalculator}),
L{RETRY_REMAINING_TIME} or a number (int or float)
@type timeout: float
@param timeout: Total timeout
@type wait_fn: callable
@param wait_fn: Waiting function
@return: Return value of function
"""
assert callable(fn)
assert callable(wait_fn)
assert callable(_time_fn)
if args is None:
args = []
end_time = _time_fn() + timeout
if callable(delay):
# External function to calculate delay
calc_delay = delay
elif isinstance(delay, (tuple, list)):
# Increasing delay with optional upper boundary
(start, factor, limit) = delay
calc_delay = _RetryDelayCalculator(start, factor, limit)
elif delay is RETRY_REMAINING_TIME:
# Always use the remaining time
calc_delay = None
else:
# Static delay
calc_delay = lambda: delay
assert calc_delay is None or callable(calc_delay)
while True:
retry_args = []
try:
# pylint: disable-msg=W0142
return fn(*args)
except RetryAgain, err:
retry_args = err.args
except RetryTimeout:
raise errors.ProgrammerError("Nested retry loop detected that didn't"
" handle RetryTimeout")
remaining_time = end_time - _time_fn()
if remaining_time < 0.0:
# pylint: disable-msg=W0142
raise RetryTimeout(*retry_args)
assert remaining_time >= 0.0
if calc_delay is None:
wait_fn(remaining_time)
else:
current_delay = calc_delay()
if current_delay > 0.0:
wait_fn(current_delay)
#!/usr/bin/python
#
# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Script for testing ganeti.utils.retry"""
import unittest
from ganeti import constants
from ganeti import errors
from ganeti import utils
import testutils
class TestRetry(testutils.GanetiTestCase):
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.retries = 0
@staticmethod
def _RaiseRetryAgain():
raise utils.RetryAgain()
@staticmethod
def _RaiseRetryAgainWithArg(args):
raise utils.RetryAgain(*args)
def _WrongNestedLoop(self):
return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)
def _RetryAndSucceed(self, retries):
if self.retries < retries:
self.retries += 1
raise utils.RetryAgain()
else:
return True
def testRaiseTimeout(self):
self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
self._RaiseRetryAgain, 0.01, 0.02)
self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
self._RetryAndSucceed, 0.01, 0, args=[1])
self.failUnlessEqual(self.retries, 1)
def testComplete(self):
self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)
self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),
True)
self.failUnlessEqual(self.retries, 2)
def testNestedLoop(self):
try:
self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
self._WrongNestedLoop, 0, 1)
except utils.RetryTimeout:
self.fail("Didn't detect inner loop's exception")
def testTimeoutArgument(self):
retry_arg="my_important_debugging_message"
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])
except utils.RetryTimeout, err:
self.failUnlessEqual(err.args, (retry_arg, ))
else:
self.fail("Expected timeout didn't happen")
def testRaiseInnerWithExc(self):
retry_arg="my_important_debugging_message"
try:
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
args=[[errors.GenericError(retry_arg, retry_arg)]])
except utils.RetryTimeout, err:
err.RaiseInner()
else:
self.fail("Expected timeout didn't happen")
except errors.GenericError, err:
self.failUnlessEqual(err.args, (retry_arg, retry_arg))
else:
self.fail("Expected GenericError didn't happen")
def testRaiseInnerWithMsg(self):
retry_arg="my_important_debugging_message"
try:
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
args=[[retry_arg, retry_arg]])
except utils.RetryTimeout, err:
err.RaiseInner()
else:
self.fail("Expected timeout didn't happen")
except utils.RetryTimeout, err:
self.failUnlessEqual(err.args, (retry_arg, retry_arg))
else:
self.fail("Expected RetryTimeout didn't happen")
if __name__ == "__main__":
testutils.GanetiTestProgram()
......@@ -1995,89 +1995,6 @@ class TestMakedirs(unittest.TestCase):
self.assert_(os.path.isdir(path))
class TestRetry(testutils.GanetiTestCase):
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.retries = 0
@staticmethod
def _RaiseRetryAgain():
raise utils.RetryAgain()
@staticmethod
def _RaiseRetryAgainWithArg(args):
raise utils.RetryAgain(*args)
def _WrongNestedLoop(self):
return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)
def _RetryAndSucceed(self, retries):
if self.retries < retries:
self.retries += 1
raise utils.RetryAgain()
else:
return True
def testRaiseTimeout(self):
self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
self._RaiseRetryAgain, 0.01, 0.02)
self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
self._RetryAndSucceed, 0.01, 0, args=[1])
self.failUnlessEqual(self.retries, 1)
def testComplete(self):
self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)
self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),
True)
self.failUnlessEqual(self.retries, 2)
def testNestedLoop(self):
try:
self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
self._WrongNestedLoop, 0, 1)
except utils.RetryTimeout:
self.fail("Didn't detect inner loop's exception")
def testTimeoutArgument(self):
retry_arg="my_important_debugging_message"
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])
except utils.RetryTimeout, err:
self.failUnlessEqual(err.args, (retry_arg, ))
else:
self.fail("Expected timeout didn't happen")
def testRaiseInnerWithExc(self):
retry_arg="my_important_debugging_message"
try:
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
args=[[errors.GenericError(retry_arg, retry_arg)]])
except utils.RetryTimeout, err:
err.RaiseInner()
else:
self.fail("Expected timeout didn't happen")
except errors.GenericError, err:
self.failUnlessEqual(err.args, (retry_arg, retry_arg))
else:
self.fail("Expected GenericError didn't happen")
def testRaiseInnerWithMsg(self):
retry_arg="my_important_debugging_message"
try:
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
args=[[retry_arg, retry_arg]])
except utils.RetryTimeout, err:
err.RaiseInner()
else:
self.fail("Expected timeout didn't happen")
except utils.RetryTimeout, err:
self.failUnlessEqual(err.args, (retry_arg, retry_arg))
else:
self.fail("Expected RetryTimeout didn't happen")
class TestLineSplitter(unittest.TestCase):
def test(self):
lines = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment