Commit cec9845c authored by Michael Hanselmann's avatar Michael Hanselmann

Split QA script into different modules.

Reviewed-by: iustinp
parent 8a23d2d3
EXTRA_DIST = ganeti-qa.py qa-sample.yaml
EXTRA_DIST = ganeti-qa.py qa-sample.yaml \
qa_cluster.py \
qa_config.py \
qa_daemon.py \
qa_env.py \
qa_error.py \
qa_instance.py \
qa_node.py
qa_other.py \
qa_utils.py
CLEANFILES = *.py[co]
This diff is collapsed.
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Cluster related QA tests.
"""
import tempfile
from ganeti import utils
import qa_config
import qa_utils
import qa_error
from qa_utils import AssertEqual, StartSSH
def TestClusterInit():
"""gnt-cluster init"""
master = qa_config.GetMasterNode()
cmd = ['gnt-cluster', 'init']
if master.get('secondary', None):
cmd.append('--secondary-ip=%s' % master['secondary'])
bridge = qa_config.get('bridge', None)
if bridge:
cmd.append('--bridge=%s' % bridge)
cmd.append('--master-netdev=%s' % bridge)
cmd.append(qa_config.get('name'))
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestClusterVerify():
"""gnt-cluster verify"""
cmd = ['gnt-cluster', 'verify']
master = qa_config.GetMasterNode()
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestClusterInfo():
"""gnt-cluster info"""
cmd = ['gnt-cluster', 'info']
master = qa_config.GetMasterNode()
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestClusterBurnin():
"""Burnin"""
master = qa_config.GetMasterNode()
# Get as many instances as we need
instances = []
try:
num = qa_config.get('options', {}).get('burnin-instances', 1)
for _ in xrange(0, num):
instances.append(qa_config.AcquireInstance())
except qa_error.OutOfInstancesError:
print "Not enough instances, continuing anyway."
if len(instances) < 1:
raise qa_error.Error("Burnin needs at least one instance")
# Run burnin
try:
script = qa_utils.UploadFile(master['primary'], '../tools/burnin')
try:
cmd = [script,
'--os=%s' % qa_config.get('os'),
'--os-size=%s' % qa_config.get('os-size'),
'--swap-size=%s' % qa_config.get('swap-size')]
cmd += [inst['name'] for inst in instances]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
finally:
cmd = ['rm', '-f', script]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
finally:
for inst in instances:
qa_config.ReleaseInstance(inst)
def TestClusterMasterFailover():
"""gnt-cluster masterfailover"""
master = qa_config.GetMasterNode()
failovermaster = qa_config.AcquireNode(exclude=master)
try:
cmd = ['gnt-cluster', 'masterfailover']
AssertEqual(StartSSH(failovermaster['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
cmd = ['gnt-cluster', 'masterfailover']
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
finally:
qa_config.ReleaseNode(failovermaster)
def TestClusterCopyfile():
"""gnt-cluster copyfile"""
master = qa_config.GetMasterNode()
# Create temporary file
f = tempfile.NamedTemporaryFile()
f.write("I'm a testfile.\n")
f.flush()
f.seek(0)
# Upload file to master node
testname = qa_utils.UploadFile(master['primary'], f.name)
try:
# Copy file to all nodes
cmd = ['gnt-cluster', 'copyfile', testname]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
finally:
# Remove file from all nodes
for node in qa_config.get('nodes'):
cmd = ['rm', '-f', testname]
AssertEqual(StartSSH(node['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestClusterDestroy():
"""gnt-cluster destroy"""
cmd = ['gnt-cluster', 'destroy', '--yes-do-it']
master = qa_config.GetMasterNode()
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""QA configuration.
"""
import yaml
import qa_error
cfg = None
options = None
def Load(path):
"""Loads the passed configuration file.
"""
global cfg
f = open(path, 'r')
try:
cfg = yaml.load(f.read())
finally:
f.close()
Validate()
def Validate():
if len(cfg['nodes']) < 1:
raise qa_error.Error("Need at least one node")
if len(cfg['instances']) < 1:
raise qa_error.Error("Need at least one instance")
def get(name, default=None):
return cfg.get(name, default)
def TestEnabled(test):
"""Returns True if the given test is enabled."""
return cfg.get('tests', {}).get(test, False)
def GetMasterNode():
return cfg['nodes'][0]
def AcquireInstance():
"""Returns an instance which isn't in use.
"""
# Filter out unwanted instances
tmp_flt = lambda inst: not inst.get('_used', False)
instances = filter(tmp_flt, cfg['instances'])
del tmp_flt
if len(instances) == 0:
raise qa_error.OutOfInstancesError("No instances left")
inst = instances[0]
inst['_used'] = True
return inst
def ReleaseInstance(inst):
inst['_used'] = False
def AcquireNode(exclude=None):
"""Returns the least used node.
"""
master = GetMasterNode()
# Filter out unwanted nodes
# TODO: Maybe combine filters
if exclude is None:
nodes = cfg['nodes'][:]
else:
nodes = filter(lambda node: node != exclude, cfg['nodes'])
tmp_flt = lambda node: node.get('_added', False) or node == master
nodes = filter(tmp_flt, nodes)
del tmp_flt
if len(nodes) == 0:
raise qa_error.OutOfNodesError("No nodes left")
# Get node with least number of uses
def compare(a, b):
result = cmp(a.get('_count', 0), b.get('_count', 0))
if result == 0:
result = cmp(a['primary'], b['primary'])
return result
nodes.sort(cmp=compare)
node = nodes[0]
node['_count'] = node.get('_count', 0) + 1
return node
def ReleaseNode(node):
node['_count'] = node.get('_count', 0) - 1
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Daemon related QA tests.
"""
import time
import subprocess
from ganeti import utils
from ganeti import constants
import qa_config
import qa_utils
import qa_error
from qa_utils import AssertEqual, StartSSH
def _ResolveInstanceName(instance):
"""Gets the full Xen name of an instance.
"""
master = qa_config.GetMasterNode()
info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])
cmd = '%s | %s' % (info_cmd, sed_cmd)
ssh_cmd = qa_utils.GetSSHCommand(master['primary'], cmd)
p = subprocess.Popen(ssh_cmd, shell=False, stdout=subprocess.PIPE)
AssertEqual(p.wait(), 0)
return p.stdout.read().strip()
def _InstanceRunning(node, name):
"""Checks whether an instance is running.
Args:
node: Node the instance runs on
name: Full name of Xen instance
"""
cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
ret = StartSSH(node['primary'], cmd).wait()
return ret == 0
def _XmShutdownInstance(node, name):
"""Shuts down instance using "xm" and waits for completion.
Args:
node: Node the instance runs on
name: Full name of Xen instance
"""
master = qa_config.GetMasterNode()
cmd = ['xm', 'shutdown', name]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
# Wait up to a minute
end = time.time() + 60
while time.time() <= end:
if not _InstanceRunning(node, name):
break
time.sleep(5)
else:
raise qa_error.Error("xm shutdown failed")
def _ResetWatcherDaemon(node):
"""Removes the watcher daemon's state file.
Args:
node: Node to be reset
"""
cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
AssertEqual(StartSSH(node['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestInstanceAutomaticRestart(node, instance):
"""Test automatic restart of instance by ganeti-watcher.
Note: takes up to 6 minutes to complete.
"""
master = qa_config.GetMasterNode()
inst_name = _ResolveInstanceName(instance)
_ResetWatcherDaemon(node)
_XmShutdownInstance(node, inst_name)
# Give it a bit more than five minutes to start again
restart_at = time.time() + 330
# Wait until it's running again
while time.time() <= restart_at:
if _InstanceRunning(node, inst_name):
break
time.sleep(15)
else:
raise qa_error.Error("Daemon didn't restart instance in time")
cmd = ['gnt-instance', 'info', inst_name]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestInstanceConsecutiveFailures(node, instance):
"""Test five consecutive instance failures.
Note: takes at least 35 minutes to complete.
"""
master = qa_config.GetMasterNode()
inst_name = _ResolveInstanceName(instance)
_ResetWatcherDaemon(node)
_XmShutdownInstance(node, inst_name)
# Do shutdowns for 30 minutes
finished_at = time.time() + (35 * 60)
while time.time() <= finished_at:
if _InstanceRunning(node, inst_name):
_XmShutdownInstance(node, inst_name)
time.sleep(30)
# Check for some time whether the instance doesn't start again
check_until = time.time() + 330
while time.time() <= check_until:
if _InstanceRunning(node, inst_name):
raise qa_error.Error("Instance started when it shouldn't")
time.sleep(30)
cmd = ['gnt-instance', 'info', inst_name]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Cluster environment related QA tests.
"""
from ganeti import utils
import qa_config
from qa_utils import AssertEqual, StartSSH
def TestSshConnection():
"""Test SSH connection.
"""
for node in qa_config.get('nodes'):
AssertEqual(StartSSH(node['primary'], 'exit').wait(), 0)
def TestGanetiCommands():
"""Test availibility of Ganeti commands.
"""
cmds = ( ['gnt-cluster', '--version'],
['gnt-os', '--version'],
['gnt-node', '--version'],
['gnt-instance', '--version'],
['gnt-backup', '--version'],
['ganeti-noded', '--version'],
['ganeti-watcher', '--version'] )
cmd = ' && '.join([utils.ShellQuoteArgs(i) for i in cmds])
for node in qa_config.get('nodes'):
AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
def TestIcmpPing():
"""ICMP ping each node.
"""
nodes = qa_config.get('nodes')
for node in nodes:
check = []
for i in nodes:
check.append(i['primary'])
if i.has_key('secondary'):
check.append(i['secondary'])
ping = lambda ip: utils.ShellQuoteArgs(['ping', '-w', '3', '-c', '1', ip])
cmd = ' && '.join([ping(i) for i in check])
AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
class Error(Exception):
"""An error occurred during Q&A testing.
"""
pass
class OutOfNodesError(Error):
"""Out of nodes.
"""
pass
class OutOfInstancesError(Error):
"""Out of instances.
"""
pass
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Instance related QA tests.
"""
from ganeti import utils
import qa_config
from qa_utils import AssertEqual, StartSSH
def _DiskTest(node, args):
master = qa_config.GetMasterNode()
instance = qa_config.AcquireInstance()
try:
cmd = ['gnt-instance', 'add',
'--os-type=%s' % qa_config.get('os'),
'--os-size=%s' % qa_config.get('os-size'),
'--swap-size=%s' % qa_config.get('swap-size'),
'--memory=%s' % qa_config.get('mem'),
'--node=%s' % node['primary']]
if args:
cmd += args
cmd.append(instance['name'])
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
return instance
except:
qa_config.ReleaseInstance(instance)
raise
def TestInstanceAddWithPlainDisk(node):
"""gnt-instance add -t plain"""
return _DiskTest(node, ['--disk-template=plain'])
def TestInstanceAddWithLocalMirrorDisk(node):
"""gnt-instance add -t local_raid1"""
return _DiskTest(node, ['--disk-template=local_raid1'])
def TestInstanceAddWithRemoteRaidDisk(node, node2):
"""gnt-instance add -t remote_raid1"""
return _DiskTest(node,
['--disk-template=remote_raid1',
'--secondary-node=%s' % node2['primary']])
def TestInstanceRemove(instance):
"""gnt-instance remove"""
master = qa_config.GetMasterNode()
cmd = ['gnt-instance', 'remove', '-f', instance['name']]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
qa_config.ReleaseInstance(instance)
def TestInstanceStartup(instance):
"""gnt-instance startup"""
master = qa_config.GetMasterNode()
cmd = ['gnt-instance', 'startup', instance['name']]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)
def TestInstanceShutdown(instance):
"""gnt-instance shutdown"""
master = qa_config.GetMasterNode()
cmd = ['gnt-instance', 'shutdown', instance['name']]
AssertEqual(StartSSH(master['primary'],
utils.ShellQuoteArgs(cmd)).wait(), 0)