Commit 42e85303 authored by Klaus Aehlig's avatar Klaus Aehlig
Browse files

Merge branch 'stable-2.9' into master



* stable-2.9
  Including missing RST files in packaging
  Update supported lint tools version numbers
  Fix some wrong indentations in the code
  Disable parenthesis indentation check
  Fix an improperly escaped string
  Make QaConfig a proper container
  Make QmpMessage a proper container
  Prevent static field checking for dynamic classes
  Fix metaclasses support in recent pylint versions

* stable-2.8
  Add additional tests for utils.Retry
  Make retry tests independent of actual time
  Fix corner-case in handling of remaining retry time
  Perform proper cleanup on termination of Haskell daemons

Conflicts:
	Makefile.am
	doc/devnotes.rst
In doc/devnotes.rst, take version updates from
stable-2.9 while keeping the addition of mock
from master. In Makefile.am take both additions.
Signed-off-by: default avatarKlaus Aehlig <aehlig@google.com>
Reviewed-by: default avatarThomas Thrainer <thomasth@google.com>
parents f89ec0e0 08d58f4c
......@@ -509,6 +509,7 @@ docinput = \
doc/design-device-uuid-name.rst \
doc/design-draft.rst \
doc/design-hotplug.rst \
doc/design-glusterfs-ganeti-support.rst \
doc/design-htools-2.3.rst \
doc/design-http-server.rst \
doc/design-impexp2.rst \
......@@ -519,6 +520,7 @@ docinput = \
doc/design-network.rst \
doc/design-node-add.rst \
doc/design-oob.rst \
doc/design-openvswitch.rst \
doc/design-ovf-support.rst \
doc/design-opportunistic-locking.rst \
doc/design-optables.rst \
......@@ -2086,6 +2088,7 @@ hs-check: hs-tests hs-shell
# (since our indent level is not 4)
# E125: continuation line does not distinguish itself from next logical line
# (since our indent level is not 4)
# E123: closing bracket does not match indentation of opening bracket's line
# E127: continuation line over-indented for visual indent
# (since our indent level is not 4)
# note: do NOT add E128 here; it's a valid style error in most cases!
......@@ -2094,7 +2097,7 @@ hs-check: hs-tests hs-shell
# instead of silencing it
# E261: at least two spaces before inline comment
# E501: line too long (80 characters)
PEP8_IGNORE = E111,E121,E125,E127,E261,E501
PEP8_IGNORE = E111,E121,E123,E125,E127,E261,E501
# For excluding pep8 expects filenames only, not whole paths
PEP8_EXCLUDE = $(subst $(space),$(comma),$(strip $(notdir $(BUILT_PYTHON_SOURCES))))
......
......@@ -147,15 +147,15 @@ in_chroot -- \
in_chroot -- \
easy_install \
logilab-astng==0.20.1 \
logilab-common==0.50.3 \
logilab-astng==0.24.1 \
logilab-common==0.58.3 \
mock==1.0.1 \
pylint==0.25.1
pylint==0.26.0
in_chroot -- \
easy_install \
sphinx==1.1.3 \
pep8==1.2 \
pep8==1.3.3 \
coverage==3.4 \
bitarray==0.8.0
......@@ -183,7 +183,7 @@ in_chroot -- \
cabal install --global \
hunit==1.2.5.2 \
happy==1.18.10 \
hlint==1.8.34 \
hlint==1.8.43 \
hscolour==1.20.3 \
temporary==1.1.2.3 \
test-framework==0.6.1 \
......@@ -225,7 +225,7 @@ in_chroot -- \
#Python development tools
in_chroot -- \
$APT_INSTALL pandoc python-epydoc
$APT_INSTALL python-epydoc
#Tools for creating debian packages
in_chroot -- \
......
......@@ -32,15 +32,15 @@ Note that for pylint, at the current moment the following versions
must be used::
$ pylint --version
pylint 0.25.1,
astng 0.23.1, common 0.58.0
pylint 0.26.0,
astng 0.24.1, common 0.58.3
The same with pep8, other versions may give you errors::
$ pep8 --version
1.2
1.3.3
Both these versions are the ones shipped with Debian Wheezy.
Both these versions are the ones shipped with Ubuntu 13.04.
To generate unittest coverage reports (``make coverage``), `coverage
<http://pypi.python.org/pypi/coverage>`_ needs to be installed.
......@@ -52,10 +52,10 @@ Installation of all dependencies listed here::
$ apt-get install python-yaml
$ cd / && sudo easy_install \
sphinx \
logilab-astng==0.23.1 \
logilab-common==0.58.0 \
pylint==0.25.1 \
pep8==1.2 \
logilab-astng==0.24.1 \
logilab-common==0.58.3 \
pylint==0.26.0 \
pep8==1.3.3 \
mock==1.0.1 \
coverage
......@@ -68,7 +68,7 @@ document, plus:
used for documentation (it's source-code pretty-printing)
- `hlint <http://community.haskell.org/~ndm/hlint/>`_, a source code
linter (equivalent to pylint for Python), recommended version 1.8 or
above (tested with 1.8.15)
above (tested with 1.8.43)
- the `QuickCheck <http://hackage.haskell.org/package/QuickCheck>`_
library, version 2.x
- the `HUnit <http://hunit.sourceforge.net/>`_ library (tested with
......
......@@ -343,8 +343,9 @@ commands = {
"Lists all available fields for node groups"),
"modify": (
SetGroupParams, ARGS_ONE_GROUP,
[DRY_RUN_OPT] + SUBMIT_OPTS + [ALLOC_POLICY_OPT, NODE_PARAMS_OPT,
HV_STATE_OPT, DISK_STATE_OPT, DISK_PARAMS_OPT, PRIORITY_OPT]
[DRY_RUN_OPT] + SUBMIT_OPTS +
[ALLOC_POLICY_OPT, NODE_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT,
DISK_PARAMS_OPT, PRIORITY_OPT]
+ INSTANCE_POLICY_OPTS,
"<group_name>", "Alters the parameters of a node group"),
"remove": (
......
......@@ -717,7 +717,7 @@ class LUNetworkDisconnect(LogicalUnit):
_NetworkConflictCheck(
self, lambda nic: nic.network == self.network_uuid, "disconnect from",
[instance_info for (_, instance_info) in
self.cfg.GetMultiInstanceInfoByName(owned_instances)])
self.cfg.GetMultiInstanceInfoByName(owned_instances)])
def Exec(self, feedback_fn):
# Disconnect the network and update the group only if network is connected
......
......@@ -194,6 +194,18 @@ class QmpMessage:
"""
self.data[field_name] = field_value
def __len__(self):
"""Return the number of fields stored in this QmpMessage.
"""
return len(self.data)
def __delitem__(self, key):
"""Delete the specified element from the QmpMessage.
"""
del(self.data[key])
@staticmethod
def BuildFromJsonString(json_string):
"""Build a QmpMessage from a JSON encoded string.
......
......@@ -1945,16 +1945,16 @@ def _GetInstanceDiskFields():
fields.extend([
(_MakeField("disk.size/%s" % i, "Disk/%s" % i, QFT_UNIT,
"Disk size of %s disk" % numtext),
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDiskSize)),
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDiskSize)),
(_MakeField("disk.spindles/%s" % i, "DiskSpindles/%s" % i, QFT_NUMBER,
"Spindles of %s disk" % numtext),
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDiskSpindles)),
(_MakeField("disk.name/%s" % i, "DiskName/%s" % i, QFT_TEXT,
"Name of %s disk" % numtext),
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDeviceName)),
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDeviceName)),
(_MakeField("disk.uuid/%s" % i, "DiskUUID/%s" % i, QFT_TEXT,
"UUID of %s disk" % numtext),
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDeviceUUID))])
IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDeviceUUID))])
return fields
......@@ -2794,7 +2794,7 @@ def _BuildNetworkFields():
# Add fields for usage statistics
fields.extend([
(_MakeField(name, title, kind, doc), NETQ_STATS, 0,
compat.partial(_GetNetworkStatsField, name, kind))
compat.partial(_GetNetworkStatsField, name, kind))
for (name, (title, kind, _, doc)) in _NETWORK_STATS_FIELDS.items()])
# Add timestamps
......
......@@ -170,11 +170,11 @@ def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
remaining_time = end_time - _time_fn()
if remaining_time < 0.0:
if remaining_time <= 0.0:
# pylint: disable=W0142
raise RetryTimeout(*retry_args)
assert remaining_time >= 0.0
assert remaining_time > 0.0
if calc_delay is None:
wait_fn(remaining_time)
......
......@@ -45,6 +45,7 @@ bad-functions = xrange
ignore-mixin-members = yes
zope = no
acquired-members =
ignored-classes = sha1,md5,Popen,ChildProcess
[VARIABLES]
init-import = no
......@@ -54,6 +55,7 @@ additional-builtins =
[CLASSES]
ignore-iface-methods =
defining-attr-methods = __init__,__new__,setUp
valid-classmethod-first-arg = cls,mcs
[DESIGN]
max-args = 15
......
......@@ -506,12 +506,12 @@ def TestClusterModifyFileBasedStorageDir(
for fail, cmd in [
(False, ["gnt-cluster", "modify",
"--enabled-disk-templates=%s" % file_disk_template,
"--ipolicy-disk-templates=%s" % file_disk_template]),
"--enabled-disk-templates=%s" % file_disk_template,
"--ipolicy-disk-templates=%s" % file_disk_template]),
(False, ["gnt-cluster", "modify",
"--%s=%s" % (option_name, file_storage_dir)]),
"--%s=%s" % (option_name, file_storage_dir)]),
(False, ["gnt-cluster", "modify",
"--%s=%s" % (option_name, invalid_file_storage_dir)]),
"--%s=%s" % (option_name, invalid_file_storage_dir)]),
# file storage dir is set to an inacceptable path, thus verify
# should fail
(True, ["gnt-cluster", "verify"]),
......@@ -520,24 +520,24 @@ def TestClusterModifyFileBasedStorageDir(
(True, ["gnt-cluster", "modify",
"--%s=" % option_name]),
(False, ["gnt-cluster", "modify",
"--%s=%s" % (option_name, file_storage_dir)]),
"--%s=%s" % (option_name, file_storage_dir)]),
(False, ["gnt-cluster", "modify",
"--enabled-disk-templates=%s" % other_disk_template,
"--ipolicy-disk-templates=%s" % other_disk_template]),
"--enabled-disk-templates=%s" % other_disk_template,
"--ipolicy-disk-templates=%s" % other_disk_template]),
(False, ["gnt-cluster", "modify",
"--%s=%s" % (option_name, invalid_file_storage_dir)]),
"--%s=%s" % (option_name, invalid_file_storage_dir)]),
# file storage is set to an inacceptable path, but file storage
# is disabled, thus verify should not fail
(False, ["gnt-cluster", "verify"]),
# unsetting the file storage dir while file storage is not enabled
# should be fine
(False, ["gnt-cluster", "modify",
"--%s=" % option_name]),
"--%s=" % option_name]),
# resetting everything to sane values
(False, ["gnt-cluster", "modify",
"--%s=%s" % (option_name, file_storage_dir),
"--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
"--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
"--%s=%s" % (option_name, file_storage_dir),
"--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
"--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
]:
AssertCommand(cmd, fail=fail)
......
......@@ -324,6 +324,24 @@ class _QaConfig(object):
"""
return self._data[name]
def __setitem__(self, key, value):
"""Sets a configuration value.
"""
self._data[key] = value
def __delitem__(self, key):
"""Deletes a value from the configuration.
"""
del(self._data[key])
def __len__(self):
"""Return the number of configuration items.
"""
return len(self._data)
def get(self, name, default=None):
"""Returns configuration value.
......
......@@ -625,8 +625,8 @@ def RemoveFromEtcHosts(hostnames):
sed_data = " ".join(hostnames)
try:
AssertCommand(("sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
" && mv %s %s") %
AssertCommand((r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
r" && mv %s %s") %
(sed_data, utils.ShellQuote(pathutils.ETC_HOSTS),
quoted_tmp_hosts, quoted_tmp_hosts,
utils.ShellQuote(pathutils.ETC_HOSTS)))
......
......@@ -45,6 +45,7 @@ module Ganeti.Daemon
, genericMain
) where
import Control.Concurrent
import Control.Exception
import Control.Monad
import Data.Maybe (fromMaybe, listToMaybe)
......@@ -53,6 +54,7 @@ import GHC.IO.Handle (hDuplicateTo)
import Network.BSD (getHostName)
import qualified Network.Socket as Socket
import System.Console.GetOpt
import System.Directory
import System.Exit
import System.Environment
import System.IO
......@@ -235,6 +237,19 @@ setupDaemonEnv cwd umask = do
_ <- createSession
return ()
-- | Cleanup function, performing all the operations that need to be done prior
-- to shutting down a daemon.
finalCleanup :: FilePath -> IO ()
finalCleanup = removeFile
-- | Signal handler for the termination signal.
handleSigTerm :: ThreadId -> IO ()
handleSigTerm mainTID =
-- Throw termination exception to the main thread, so that the daemon is
-- actually stopped in the proper way, executing all the functions waiting on
-- "finally" statement.
Control.Exception.throwTo mainTID ExitSuccess
-- | Signal handler for reopening log files.
handleSigHup :: FilePath -> IO ()
handleSigHup path = do
......@@ -404,7 +419,7 @@ fullPrep :: GanetiDaemon -- ^ The daemon we're running
-> SyslogUsage -- ^ Syslog mode
-> a -- ^ Check results
-> PrepFn a b -- ^ Prepare function
-> IO b
-> IO (FilePath, b)
fullPrep daemon opts syslog check_result prep_fn = do
logfile <- if optDaemonize opts
then return Nothing
......@@ -415,7 +430,10 @@ fullPrep daemon opts syslog check_result prep_fn = do
_ <- describeError "writing PID file; already locked?"
Nothing (Just pidfile) $ writePidFile pidfile
logNotice $ dname ++ " daemon startup"
prep_fn opts check_result
prep_res <- prep_fn opts check_result
tid <- myThreadId
_ <- installHandler sigTERM (Catch $ handleSigTerm tid) Nothing
return (pidfile, prep_res)
-- | Inner daemon function.
--
......@@ -429,11 +447,11 @@ innerMain :: GanetiDaemon -- ^ The daemon we're running
-> Maybe Fd -- ^ Error reporting function
-> IO ()
innerMain daemon opts syslog check_result prep_fn exec_fn fd = do
prep_result <- fullPrep daemon opts syslog check_result prep_fn
(pidFile, prep_result) <- fullPrep daemon opts syslog check_result prep_fn
`Control.Exception.catch` handlePrepErr True fd
-- no error reported, we should now close the fd
maybeCloseFd fd
exec_fn opts check_result prep_result
finally (exec_fn opts check_result prep_result) (finalCleanup pidFile)
-- | Daemon prepare error handling function.
handlePrepErr :: Bool -> Maybe Fd -> IOError -> IO a
......
......@@ -135,6 +135,20 @@ class TestQmpMessage(testutils.GanetiTestCase):
rebuilt_message = hv_kvm.QmpMessage.BuildFromJsonString(serialized)
self.assertEqual(rebuilt_message, message)
self.assertEqual(len(rebuilt_message), len(test_data))
def testDelete(self):
toDelete = "execute"
test_data = {
toDelete: "command",
"arguments": ["a", "b", "c"],
}
message = hv_kvm.QmpMessage(test_data)
oldLen = len(message)
del(message[toDelete])
newLen = len(message)
self.assertEqual(oldLen - 1, newLen)
class TestQmp(testutils.GanetiTestCase):
......
......@@ -35,6 +35,16 @@ class TestRetry(testutils.GanetiTestCase):
testutils.GanetiTestCase.setUp(self)
self.retries = 0
self.called = 0
self.time = 1379601882.0
self.time_for_time_fn = 0
self.time_for_retry_and_succeed = 0
def _time_fn(self):
self.time += self.time_for_time_fn
return self.time
def _wait_fn(self, delay):
self.time += delay
@staticmethod
def _RaiseRetryAgain():
......@@ -48,6 +58,7 @@ class TestRetry(testutils.GanetiTestCase):
return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)
def _RetryAndSucceed(self, retries):
self.time += self.time_for_retry_and_succeed
if self.retries < retries:
self.retries += 1
raise utils.RetryAgain()
......@@ -64,39 +75,77 @@ class TestRetry(testutils.GanetiTestCase):
def testRaiseTimeout(self):
self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
self._RaiseRetryAgain, 0.01, 0.02)
self._RaiseRetryAgain, 0.01, 0.02,
wait_fn = self._wait_fn, _time_fn = self._time_fn)
self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
self._RetryAndSucceed, 0.01, 0, args=[1])
self._RetryAndSucceed, 0.01, 0, args=[1],
wait_fn = self._wait_fn, _time_fn = self._time_fn)
self.failUnlessEqual(self.retries, 1)
def testComplete(self):
self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)
self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),
self.failUnlessEqual(utils.Retry(lambda: True, 0, 1,
wait_fn = self._wait_fn,
_time_fn = self._time_fn),
True)
self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2],
wait_fn = self._wait_fn,
_time_fn = self._time_fn),
True)
self.failUnlessEqual(self.retries, 2)
def testCompleteNontrivialTimes(self):
self.time_for_time_fn = 0.01
self.time_for_retry_and_succeed = 0.1
self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2],
wait_fn = self._wait_fn,
_time_fn = self._time_fn),
True)
self.failUnlessEqual(self.retries, 2)
def testNestedLoop(self):
try:
self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
self._WrongNestedLoop, 0, 1)
self._WrongNestedLoop, 0, 1,
wait_fn = self._wait_fn, _time_fn = self._time_fn)
except utils.RetryTimeout:
self.fail("Didn't detect inner loop's exception")
def testTimeoutArgument(self):
retry_arg="my_important_debugging_message"
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]],
wait_fn = self._wait_fn, _time_fn = self._time_fn)
except utils.RetryTimeout, err:
self.failUnlessEqual(err.args, (retry_arg, ))
else:
self.fail("Expected timeout didn't happen")
def testTimeout(self):
self.time_for_time_fn = 0.01
self.time_for_retry_and_succeed = 10
try:
utils.Retry(self._RetryAndSucceed, 1, 18, args=[2],
wait_fn = self._wait_fn, _time_fn = self._time_fn)
except utils.RetryTimeout, err:
self.failUnlessEqual(err.args, ())
else:
self.fail("Expected timeout didn't happen")
def testNoTimeout(self):
self.time_for_time_fn = 0.01
self.time_for_retry_and_succeed = 8
self.failUnlessEqual(
utils.Retry(self._RetryAndSucceed, 1, 18, args=[2],
wait_fn = self._wait_fn, _time_fn = self._time_fn),
True)
def testRaiseInnerWithExc(self):
retry_arg="my_important_debugging_message"
try:
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
args=[[errors.GenericError(retry_arg, retry_arg)]])
args=[[errors.GenericError(retry_arg, retry_arg)]],
wait_fn = self._wait_fn, _time_fn = self._time_fn)
except utils.RetryTimeout, err:
err.RaiseInner()
else:
......@@ -111,7 +160,8 @@ class TestRetry(testutils.GanetiTestCase):
try:
try:
utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
args=[[retry_arg, retry_arg]])
args=[[retry_arg, retry_arg]],
wait_fn = self._wait_fn, _time_fn = self._time_fn)
except utils.RetryTimeout, err:
err.RaiseInner()
else:
......@@ -122,17 +172,27 @@ class TestRetry(testutils.GanetiTestCase):
self.fail("Expected RetryTimeout didn't happen")
def testSimpleRetry(self):
self.assertFalse(utils.SimpleRetry(True, lambda: False, 0.01, 0.02))
self.assertFalse(utils.SimpleRetry(lambda x: x, lambda: False, 0.01, 0.02))
self.assertTrue(utils.SimpleRetry(True, lambda: True, 0, 1))
self.assertTrue(utils.SimpleRetry(lambda x: x, lambda: True, 0, 1))
self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed,
0, 1, args=[1]))
self.assertFalse(utils.SimpleRetry(True, lambda: False, 0.01, 0.02,
wait_fn = self._wait_fn,
_time_fn = self._time_fn))
self.assertFalse(utils.SimpleRetry(lambda x: x, lambda: False, 0.01, 0.02,
wait_fn = self._wait_fn,
_time_fn = self._time_fn))
self.assertTrue(utils.SimpleRetry(True, lambda: True, 0, 1,
wait_fn = self._wait_fn,
_time_fn = self._time_fn))
self.assertTrue(utils.SimpleRetry(lambda x: x, lambda: True, 0, 1,
wait_fn = self._wait_fn,
_time_fn = self._time_fn))
self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1,
args=[1], wait_fn = self._wait_fn,
_time_fn = self._time_fn))
self.assertEqual(self.retries, 1)
self.assertEqual(self.called, 2)
self.called = self.retries = 0
self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed,
0, 1, args=[2]))
self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1,
args=[2], wait_fn = self._wait_fn,
_time_fn = self._time_fn))
self.assertEqual(self.called, 3)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment