Commit 28a6fbc8 authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Run ganeti-watcher script from QA code instead of cron.

This makes the tests much more reliably because it avoids race conditions.
It also helps to speed them up a lot.

Reviewed-by: iustinp
parent 7b195d9b
......@@ -55,18 +55,14 @@ tests:
instance-import: True
instance-reinstall: True
instance-shutdown: True
instance-automatic-restart: False
instance-consecutive-failures: False
# Make sure not to include the disk(s) required for Dom0 to be included in
# the volume group used for instances. Otherwise the whole system may stop
# working until restarted.
instance-disk-failure: False
# This test takes up to 6 minutes to complete
instance-automatic-restart: False
# This test takes at least 35 minutes to complete
instance-consecutive-failures: False
# Other settings
options:
burnin-instances: 2
......
......@@ -67,47 +67,55 @@ def _XmShutdownInstance(node, name):
raise qa_error.Error("xm shutdown failed")
def _ResetWatcherDaemon(node):
def _ResetWatcherDaemon():
"""Removes the watcher daemon's state file.
Args:
node: Node to be reset
"""
master = qa_config.GetMasterNode()
cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
AssertEqual(StartSSH(node['primary'],
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def _RunWatcherDaemon():
"""Runs the ganeti-watcher daemon on the master node.
"""
master = qa_config.GetMasterNode()
cmd = ['ganeti-watcher', '-d']
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def PrintCronWarning():
"""Shows a warning about the required cron job.
"""Shows a warning about the cron job.
"""
msg = ("For the following tests it's recommended to turn off the "
"ganeti-watcher cronjob.")
print
print qa_utils.FormatWarning("The following tests require the cron script "
"for ganeti-watcher to be set up.")
print qa_utils.FormatWarning(msg)
def TestInstanceAutomaticRestart(node, instance):
"""Test automatic restart of instance by ganeti-watcher.
Note: takes up to 6 minutes to complete.
"""
master = qa_config.GetMasterNode()
inst_name = qa_utils.ResolveInstanceName(instance)
_ResetWatcherDaemon(node)
_ResetWatcherDaemon()
_XmShutdownInstance(node, inst_name)
# Give it a bit more than five minutes to start again
restart_at = time.time() + 330
_RunWatcherDaemon()
time.sleep(5)
# Wait until it's running again
while time.time() <= restart_at:
if _InstanceRunning(node, inst_name):
break
time.sleep(15)
else:
raise qa_error.Error("Daemon didn't restart instance in time")
if not _InstanceRunning(node, inst_name):
raise qa_error.Error("Daemon didn't restart instance")
cmd = ['gnt-instance', 'info', inst_name]
AssertEqual(StartSSH(master['primary'],
......@@ -117,28 +125,23 @@ def TestInstanceAutomaticRestart(node, instance):
def TestInstanceConsecutiveFailures(node, instance):
"""Test five consecutive instance failures.
Note: takes at least 35 minutes to complete.
"""
master = qa_config.GetMasterNode()
inst_name = qa_utils.ResolveInstanceName(instance)
_ResetWatcherDaemon(node)
_XmShutdownInstance(node, inst_name)
# Do shutdowns for 30 minutes
finished_at = time.time() + (35 * 60)
_ResetWatcherDaemon()
while time.time() <= finished_at:
if _InstanceRunning(node, inst_name):
_XmShutdownInstance(node, inst_name)
time.sleep(30)
for should_start in ([True] * 5) + [False]:
_XmShutdownInstance(node, inst_name)
_RunWatcherDaemon()
time.sleep(5)
# Check for some time whether the instance doesn't start again
check_until = time.time() + 330
while time.time() <= check_until:
if _InstanceRunning(node, inst_name):
raise qa_error.Error("Instance started when it shouldn't")
time.sleep(30)
if bool(_InstanceRunning(node, inst_name)) != should_start:
if should_start:
msg = "Instance not started when it should"
else:
msg = "Instance started when it shouldn't"
raise qa_error.Error(msg)
cmd = ['gnt-instance', 'info', inst_name]
AssertEqual(StartSSH(master['primary'],
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment