Test “gnt-node evacuate” and “gnt-node failover” in QA.

Reviewed-by: schreiberal

qa/ganeti-qa.py

@@ -211,6 +211,12 @@ def main():
         if qa_config.TestEnabled('instance-failover'):
           RunTest(qa_instance.TestInstanceFailover, instance)
 
+        if qa_config.TestEnabled('node-evacuate'):
+          RunTest(qa_node.TestNodeEvacuate, node, node2)
+
+        if qa_config.TestEnabled('node-failover'):
+          RunTest(qa_node.TestNodeFailover, node, node2)
+
         if qa_config.TestEnabled('node-volumes'):
           RunTest(qa_node.TestNodeVolumes)
 

qa/qa-sample.yaml

@@ -38,6 +38,10 @@ tests:
   node-info: True
   node-volumes: True
 
+  # These tests need at least three nodes
+  node-evacuate: False
+  node-failover: False
+
   instance-add-plain-disk: True
   instance-add-local-mirror-disk: True
   instance-add-remote-raid-disk: True

qa/qa_config.py

@@ -96,6 +96,8 @@ def AcquireNode(exclude=None):
   # TODO: Maybe combine filters
   if exclude is None:
     nodes = cfg['nodes'][:]
+  elif isinstance(exclude, (list, tuple)):
+    nodes = filter(lambda node: node not in exclude, cfg['nodes'])
   else:
     nodes = filter(lambda node: node != exclude, cfg['nodes'])
 

qa/qa_error.py

@@ -35,3 +35,10 @@ class OutOfInstancesError(Error):
 
   """
   pass
+
+
+class UnusableNodeError(Error):
+  """Unusable node.
+
+  """
+  pass

qa/qa_node.py

@@ -20,6 +20,7 @@ from ganeti import utils
 
 import qa_config
 import qa_error
+import qa_utils
 
 from qa_utils import AssertEqual, StartSSH
 
@@ -81,3 +82,47 @@ def TestNodeVolumes():
   cmd = ['gnt-node', 'volumes']
   AssertEqual(StartSSH(master['primary'],
                        utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestNodeFailover(node, node2):
+  """gnt-node failover"""
+  master = qa_config.GetMasterNode()
+
+  if qa_utils.GetNodeInstances(node2):
+    raise qa_errors.UnusableNodeError("Secondary node has at least one "
+                                      "primary instance. This test requires "
+                                      "it to have no primary instances.")
+
+  # Fail over to secondary node
+  cmd = ['gnt-node', 'failover', '-f', node['primary']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  # ... and back again.
+  cmd = ['gnt-node', 'failover', '-f', node2['primary']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestNodeEvacuate(node, node2):
+  """gnt-node evacuate"""
+  master = qa_config.GetMasterNode()
+
+  node3 = qa_config.AcquireNode(exclude=[node, node2])
+  try:
+    if qa_utils.GetNodeInstances(node3):
+      raise qa_errors.UnusableNodeError("Evacuation node has at least one "
+                                        "secondary instance. This test requires "
+                                        "it to have no secondary instances.")
+
+    # Evacuate all secondary instances
+    cmd = ['gnt-node', 'evacuate', '-f', node2['primary'], node3['primary']]
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+    # ... and back again.
+    cmd = ['gnt-node', 'evacuate', '-f', node3['primary'], node2['primary']]
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    qa_config.ReleaseNode(node3)

qa/qa_utils.py

@@ -100,8 +100,18 @@ def StartSSH(node, cmd, strict=True):
   """Starts SSH.
 
   """
-  args = GetSSHCommand(node, cmd, strict=strict)
-  return subprocess.Popen(args, shell=False)
+  return subprocess.Popen(GetSSHCommand(node, cmd, strict=strict),
+                          shell=False)
+
+
+def GetCommandOutput(node, cmd):
+  """Returns the output of a command executed on the given node.
+
+  """
+  p = subprocess.Popen(GetSSHCommand(node, cmd),
+                       shell=False, stdout=subprocess.PIPE)
+  AssertEqual(p.wait(), 0)
+  return p.stdout.read()
 
 
 def UploadFile(node, src):
@@ -130,21 +140,57 @@ def UploadFile(node, src):
     f.close()
 
 
+def _ResolveName(cmd, key):
+  """Helper function.
+
+  """
+  master = qa_config.GetMasterNode()
+
+  output = GetCommandOutput(master['primary'], utils.ShellQuoteArgs(cmd))
+  for line in output.splitlines():
+    (lkey, lvalue) = line.split(':', 1)
+    if lkey == key:
+      return lvalue.lstrip()
+  raise KeyError("Key not found")
+
+
 def ResolveInstanceName(instance):
   """Gets the full name of an instance.
 
+  """
+  return _ResolveName(['gnt-instance', 'info', instance['info']],
+                      'Instance name')
+
+
+def ResolveNodeName(node):
+  """Gets the full name of a node.
+
+  """
+  return _ResolveName(['gnt-node', 'info', node['primary']],
+                      'Node name')
+
+
+def GetNodeInstances(node, secondaries=False):
+  """Gets a list of instances on a node.
+
   """
   master = qa_config.GetMasterNode()
 
-  info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
-  sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])
+  node_name = ResolveNodeName(node)
 
-  cmd = '%s | %s' % (info_cmd, sed_cmd)
-  ssh_cmd = GetSSHCommand(master['primary'], cmd)
-  p = subprocess.Popen(ssh_cmd, shell=False, stdout=subprocess.PIPE)
-  AssertEqual(p.wait(), 0)
+  # Get list of all instances
+  cmd = ['gnt-instance', 'list', '--separator=:', '--no-headers',
+         '--output=name,pnode,snodes']
+  output = GetCommandOutput(master['primary'], utils.ShellQuoteArgs(cmd))
+
+  instances = []
+  for line in output.splitlines():
+    (name, pnode, snodes) = line.split(':', 2)
+    if ((not secondaries and pnode == node_name) or
+        (secondaries and node_name in snodes.split(','))):
+      instances.append(name)
 
-  return p.stdout.read().strip()
+  return instances
 
 
 def _PrintWithColor(text, seq):