Split QA script into different modules.

Reviewed-by: iustinp

qa/Makefile.am

-EXTRA_DIST = ganeti-qa.py qa-sample.yaml
+EXTRA_DIST = ganeti-qa.py qa-sample.yaml \
+	qa_cluster.py \
+	qa_config.py \
+	qa_daemon.py \
+	qa_env.py \
+	qa_error.py \
+	qa_instance.py \
+	qa_node.py
+	qa_other.py \
+	qa_utils.py
 CLEANFILES = *.py[co]

qa/qa_cluster.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Cluster related QA tests.
+
+"""
+
+import tempfile
+
+from ganeti import utils
+
+import qa_config
+import qa_utils
+import qa_error
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def TestClusterInit():
+  """gnt-cluster init"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-cluster', 'init']
+
+  if master.get('secondary', None):
+    cmd.append('--secondary-ip=%s' % master['secondary'])
+
+  bridge = qa_config.get('bridge', None)
+  if bridge:
+    cmd.append('--bridge=%s' % bridge)
+    cmd.append('--master-netdev=%s' % bridge)
+
+  cmd.append(qa_config.get('name'))
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterVerify():
+  """gnt-cluster verify"""
+  cmd = ['gnt-cluster', 'verify']
+  master = qa_config.GetMasterNode()
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterInfo():
+  """gnt-cluster info"""
+  cmd = ['gnt-cluster', 'info']
+  master = qa_config.GetMasterNode()
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterBurnin():
+  """Burnin"""
+  master = qa_config.GetMasterNode()
+
+  # Get as many instances as we need
+  instances = []
+  try:
+    num = qa_config.get('options', {}).get('burnin-instances', 1)
+    for _ in xrange(0, num):
+      instances.append(qa_config.AcquireInstance())
+  except qa_error.OutOfInstancesError:
+    print "Not enough instances, continuing anyway."
+
+  if len(instances) < 1:
+    raise qa_error.Error("Burnin needs at least one instance")
+
+  # Run burnin
+  try:
+    script = qa_utils.UploadFile(master['primary'], '../tools/burnin')
+    try:
+      cmd = [script,
+             '--os=%s' % qa_config.get('os'),
+             '--os-size=%s' % qa_config.get('os-size'),
+             '--swap-size=%s' % qa_config.get('swap-size')]
+      cmd += [inst['name'] for inst in instances]
+      AssertEqual(StartSSH(master['primary'],
+                           utils.ShellQuoteArgs(cmd)).wait(), 0)
+    finally:
+      cmd = ['rm', '-f', script]
+      AssertEqual(StartSSH(master['primary'],
+                           utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    for inst in instances:
+      qa_config.ReleaseInstance(inst)
+
+
+def TestClusterMasterFailover():
+  """gnt-cluster masterfailover"""
+  master = qa_config.GetMasterNode()
+
+  failovermaster = qa_config.AcquireNode(exclude=master)
+  try:
+    cmd = ['gnt-cluster', 'masterfailover']
+    AssertEqual(StartSSH(failovermaster['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+    cmd = ['gnt-cluster', 'masterfailover']
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    qa_config.ReleaseNode(failovermaster)
+
+
+def TestClusterCopyfile():
+  """gnt-cluster copyfile"""
+  master = qa_config.GetMasterNode()
+
+  # Create temporary file
+  f = tempfile.NamedTemporaryFile()
+  f.write("I'm a testfile.\n")
+  f.flush()
+  f.seek(0)
+
+  # Upload file to master node
+  testname = qa_utils.UploadFile(master['primary'], f.name)
+  try:
+    # Copy file to all nodes
+    cmd = ['gnt-cluster', 'copyfile', testname]
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    # Remove file from all nodes
+    for node in qa_config.get('nodes'):
+      cmd = ['rm', '-f', testname]
+      AssertEqual(StartSSH(node['primary'],
+                           utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterDestroy():
+  """gnt-cluster destroy"""
+  cmd = ['gnt-cluster', 'destroy', '--yes-do-it']
+  master = qa_config.GetMasterNode()
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)

qa/qa_config.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""QA configuration.
+
+"""
+
+
+import yaml
+
+import qa_error
+
+
+cfg = None
+options = None
+
+
+def Load(path):
+  """Loads the passed configuration file.
+
+  """
+  global cfg
+
+  f = open(path, 'r')
+  try:
+    cfg = yaml.load(f.read())
+  finally:
+    f.close()
+
+  Validate()
+
+
+def Validate():
+  if len(cfg['nodes']) < 1:
+    raise qa_error.Error("Need at least one node")
+  if len(cfg['instances']) < 1:
+    raise qa_error.Error("Need at least one instance")
+
+
+def get(name, default=None):
+  return cfg.get(name, default)
+
+
+def TestEnabled(test):
+  """Returns True if the given test is enabled."""
+  return cfg.get('tests', {}).get(test, False)
+
+
+def GetMasterNode():
+  return cfg['nodes'][0]
+
+
+def AcquireInstance():
+  """Returns an instance which isn't in use.
+
+  """
+  # Filter out unwanted instances
+  tmp_flt = lambda inst: not inst.get('_used', False)
+  instances = filter(tmp_flt, cfg['instances'])
+  del tmp_flt
+
+  if len(instances) == 0:
+    raise qa_error.OutOfInstancesError("No instances left")
+
+  inst = instances[0]
+  inst['_used'] = True
+  return inst
+
+
+def ReleaseInstance(inst):
+  inst['_used'] = False
+
+
+def AcquireNode(exclude=None):
+  """Returns the least used node.
+
+  """
+  master = GetMasterNode()
+
+  # Filter out unwanted nodes
+  # TODO: Maybe combine filters
+  if exclude is None:
+    nodes = cfg['nodes'][:]
+  else:
+    nodes = filter(lambda node: node != exclude, cfg['nodes'])
+
+  tmp_flt = lambda node: node.get('_added', False) or node == master
+  nodes = filter(tmp_flt, nodes)
+  del tmp_flt
+
+  if len(nodes) == 0:
+    raise qa_error.OutOfNodesError("No nodes left")
+
+  # Get node with least number of uses
+  def compare(a, b):
+    result = cmp(a.get('_count', 0), b.get('_count', 0))
+    if result == 0:
+      result = cmp(a['primary'], b['primary'])
+    return result
+
+  nodes.sort(cmp=compare)
+
+  node = nodes[0]
+  node['_count'] = node.get('_count', 0) + 1
+  return node
+
+
+def ReleaseNode(node):
+  node['_count'] = node.get('_count', 0) - 1

qa/qa_daemon.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Daemon related QA tests.
+
+"""
+
+import time
+import subprocess
+
+from ganeti import utils
+from ganeti import constants
+
+import qa_config
+import qa_utils
+import qa_error
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def _ResolveInstanceName(instance):
+  """Gets the full Xen name of an instance.
+
+  """
+  master = qa_config.GetMasterNode()
+
+  info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
+  sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])
+
+  cmd = '%s | %s' % (info_cmd, sed_cmd)
+  ssh_cmd = qa_utils.GetSSHCommand(master['primary'], cmd)
+  p = subprocess.Popen(ssh_cmd, shell=False, stdout=subprocess.PIPE)
+  AssertEqual(p.wait(), 0)
+
+  return p.stdout.read().strip()
+
+
+def _InstanceRunning(node, name):
+  """Checks whether an instance is running.
+
+  Args:
+    node: Node the instance runs on
+    name: Full name of Xen instance
+  """
+  cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
+  ret = StartSSH(node['primary'], cmd).wait()
+  return ret == 0
+
+
+def _XmShutdownInstance(node, name):
+  """Shuts down instance using "xm" and waits for completion.
+
+  Args:
+    node: Node the instance runs on
+    name: Full name of Xen instance
+  """
+  master = qa_config.GetMasterNode()
+
+  cmd = ['xm', 'shutdown', name]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  # Wait up to a minute
+  end = time.time() + 60
+  while time.time() <= end:
+    if not _InstanceRunning(node, name):
+      break
+    time.sleep(5)
+  else:
+    raise qa_error.Error("xm shutdown failed")
+
+
+def _ResetWatcherDaemon(node):
+  """Removes the watcher daemon's state file.
+
+  Args:
+    node: Node to be reset
+  """
+  cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
+  AssertEqual(StartSSH(node['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceAutomaticRestart(node, instance):
+  """Test automatic restart of instance by ganeti-watcher.
+
+  Note: takes up to 6 minutes to complete.
+  """
+  master = qa_config.GetMasterNode()
+  inst_name = _ResolveInstanceName(instance)
+
+  _ResetWatcherDaemon(node)
+  _XmShutdownInstance(node, inst_name)
+
+  # Give it a bit more than five minutes to start again
+  restart_at = time.time() + 330
+
+  # Wait until it's running again
+  while time.time() <= restart_at:
+    if _InstanceRunning(node, inst_name):
+      break
+    time.sleep(15)
+  else:
+    raise qa_error.Error("Daemon didn't restart instance in time")
+
+  cmd = ['gnt-instance', 'info', inst_name]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceConsecutiveFailures(node, instance):
+  """Test five consecutive instance failures.
+
+  Note: takes at least 35 minutes to complete.
+  """
+  master = qa_config.GetMasterNode()
+  inst_name = _ResolveInstanceName(instance)
+
+  _ResetWatcherDaemon(node)
+  _XmShutdownInstance(node, inst_name)
+
+  # Do shutdowns for 30 minutes
+  finished_at = time.time() + (35 * 60)
+
+  while time.time() <= finished_at:
+    if _InstanceRunning(node, inst_name):
+      _XmShutdownInstance(node, inst_name)
+    time.sleep(30)
+
+  # Check for some time whether the instance doesn't start again
+  check_until = time.time() + 330
+  while time.time() <= check_until:
+    if _InstanceRunning(node, inst_name):
+      raise qa_error.Error("Instance started when it shouldn't")
+    time.sleep(30)
+
+  cmd = ['gnt-instance', 'info', inst_name]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)

qa/qa_env.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Cluster environment related QA tests.
+
+"""
+
+from ganeti import utils
+
+import qa_config
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def TestSshConnection():
+  """Test SSH connection.
+
+  """
+  for node in qa_config.get('nodes'):
+    AssertEqual(StartSSH(node['primary'], 'exit').wait(), 0)
+
+
+def TestGanetiCommands():
+  """Test availibility of Ganeti commands.
+
+  """
+  cmds = ( ['gnt-cluster', '--version'],
+           ['gnt-os', '--version'],
+           ['gnt-node', '--version'],
+           ['gnt-instance', '--version'],
+           ['gnt-backup', '--version'],
+           ['ganeti-noded', '--version'],
+           ['ganeti-watcher', '--version'] )
+
+  cmd = ' && '.join([utils.ShellQuoteArgs(i) for i in cmds])
+
+  for node in qa_config.get('nodes'):
+    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
+
+
+def TestIcmpPing():
+  """ICMP ping each node.
+
+  """
+  nodes = qa_config.get('nodes')
+
+  for node in nodes:
+    check = []
+    for i in nodes:
+      check.append(i['primary'])
+      if i.has_key('secondary'):
+        check.append(i['secondary'])
+
+    ping = lambda ip: utils.ShellQuoteArgs(['ping', '-w', '3', '-c', '1', ip])
+    cmd = ' && '.join([ping(i) for i in check])
+
+    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)

qa/qa_error.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+class Error(Exception):
+  """An error occurred during Q&A testing.
+
+  """
+  pass
+
+
+class OutOfNodesError(Error):
+  """Out of nodes.
+
+  """
+  pass
+
+
+class OutOfInstancesError(Error):
+  """Out of instances.
+
+  """
+  pass

qa/qa_instance.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Instance related QA tests.
+
+"""
+
+from ganeti import utils
+
+import qa_config
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def _DiskTest(node, args):
+  master = qa_config.GetMasterNode()
+
+  instance = qa_config.AcquireInstance()
+  try:
+    cmd = ['gnt-instance', 'add',
+           '--os-type=%s' % qa_config.get('os'),
+           '--os-size=%s' % qa_config.get('os-size'),
+           '--swap-size=%s' % qa_config.get('swap-size'),
+           '--memory=%s' % qa_config.get('mem'),
+           '--node=%s' % node['primary']]
+    if args:
+      cmd += args
+    cmd.append(instance['name'])
+
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+    return instance
+  except:
+    qa_config.ReleaseInstance(instance)
+    raise
+
+
+def TestInstanceAddWithPlainDisk(node):
+  """gnt-instance add -t plain"""
+  return _DiskTest(node, ['--disk-template=plain'])
+
+
+def TestInstanceAddWithLocalMirrorDisk(node):
+  """gnt-instance add -t local_raid1"""
+  return _DiskTest(node, ['--disk-template=local_raid1'])
+
+
+def TestInstanceAddWithRemoteRaidDisk(node, node2):
+  """gnt-instance add -t remote_raid1"""
+  return _DiskTest(node,
+                   ['--disk-template=remote_raid1',
+                    '--secondary-node=%s' % node2['primary']])
+
+
+def TestInstanceRemove(instance):
+  """gnt-instance remove"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  qa_config.ReleaseInstance(instance)
+
+
+def TestInstanceStartup(instance):
+  """gnt-instance startup"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'startup', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceShutdown(instance):
+  """gnt-instance shutdown"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'shutdown', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceFailover(instance):
+  """gnt-instance failover"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceInfo(instance):
+  """gnt-instance info"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'info', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)

qa/qa_node.py

+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+from ganeti import utils
+
+import qa_config
+import qa_error
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def _NodeAdd(node):
+  master = qa_config.GetMasterNode()
+
+  if node.get('_added', False):
+    raise qa_error.Error("Node %s already in cluster" % node['primary'])
+
+  cmd = ['gnt-node', 'add']
+  if node.get('secondary', None):
+    cmd.append('--secondary-ip=%s' % node['secondary'])
+  cmd.append(node['primary'])
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  node['_added'] = True
+
+
+def _NodeRemove(node):
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-node', 'remove', node['primary']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+  node['_added'] = False
+
+
+def TestNodeAddAll():
+  """Adding all nodes to cluster."""
+  master = qa_config.GetMasterNode()
+  for node in qa_config.get('nodes'):
+    if node != master:
+      _NodeAdd(node)
+
+
+def TestNodeRemoveAll():
+  """Removing all nodes from cluster."""