qa_node.py

#
#

# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


"""Node-related QA tests.

"""

from ganeti import utils
from ganeti import constants
from ganeti import query
from ganeti import serializer

import qa_config
import qa_error
import qa_utils

from qa_utils import AssertCommand, AssertRedirectedCommand, AssertEqual, \
  AssertIn, GetCommandOutput


def NodeAdd(node, readd=False, group=None):
  if not readd and node.added:
    raise qa_error.Error("Node %s already in cluster" % node.primary)
  elif readd and not node.added:
    raise qa_error.Error("Node %s not yet in cluster" % node.primary)

  cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  if node.secondary:
    cmd.append("--secondary-ip=%s" % node.secondary)
  if readd:
    cmd.append("--readd")
  if group is not None:
    cmd.extend(["--node-group", group])

  if not qa_config.GetModifySshSetup():
    cmd.append("--no-node-setup")

  cmd.append(node.primary)

  AssertCommand(cmd)

  if readd:
    AssertRedirectedCommand(["gnt-cluster", "verify"])

  if readd:
    assert node.added
  else:
    node.MarkAdded()


def NodeRemove(node):
  AssertCommand(["gnt-node", "remove", node.primary])
  node.MarkRemoved()


def MakeNodeOffline(node, value):
  """gnt-node modify --offline=value"""
  # value in ["yes", "no"]
  AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])


def TestNodeAddAll():
  """Adding all nodes to cluster."""
  master = qa_config.GetMasterNode()
  for node in qa_config.get("nodes"):
    if node != master:
      NodeAdd(node, readd=False)


def MarkNodeAddedAll():
  """Mark all nodes as added.

  This is useful if we don't create the cluster ourselves (in qa).

  """
  master = qa_config.GetMasterNode()
  for node in qa_config.get("nodes"):
    if node != master:
      node.MarkAdded()


def TestNodeRemoveAll():
  """Removing all nodes from cluster."""
  master = qa_config.GetMasterNode()
  for node in qa_config.get("nodes"):
    if node != master:
      NodeRemove(node)


def TestNodeReadd(node):
  """gnt-node add --readd"""
  NodeAdd(node, readd=True)


def TestNodeInfo():
  """gnt-node info"""
  AssertCommand(["gnt-node", "info"])


def TestNodeVolumes():
  """gnt-node volumes"""
  AssertCommand(["gnt-node", "volumes"])


def TestNodeStorage():
  """gnt-node storage"""
  master = qa_config.GetMasterNode()

  # FIXME: test all storage_types in constants.STORAGE_TYPES
  # as soon as they are implemented.
  enabled_storage_types = qa_config.GetEnabledStorageTypes()
  testable_storage_types = list(set(enabled_storage_types).intersection(
      set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))

  for storage_type in testable_storage_types:

    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]

    # Test simple list
    AssertCommand(cmd)

    # Test all storage fields
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
           "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS))]
    AssertCommand(cmd)

    # Get list of valid storage devices
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
           "--output=node,name,allocatable", "--separator=|",
           "--no-headers"]
    output = qa_utils.GetCommandOutput(master.primary,
                                       utils.ShellQuoteArgs(cmd))

    # Test with up to two devices
    testdevcount = 2

    for line in output.splitlines()[:testdevcount]:
      (node_name, st_name, st_allocatable) = line.split("|")

      # Dummy modification without any changes
      cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
      AssertCommand(cmd)

      # Make sure we end up with the same value as before
      if st_allocatable.lower() == "y":
        test_allocatable = ["no", "yes"]
      else:
        test_allocatable = ["yes", "no"]

      fail = (constants.SF_ALLOCATABLE not in
              constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))

      for i in test_allocatable:
        AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
                       node_name, storage_type, st_name], fail=fail)

        # Verify list output
        cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
               "--output=name,allocatable", "--separator=|",
               "--no-headers", node_name]
        listout = qa_utils.GetCommandOutput(master.primary,
                                            utils.ShellQuoteArgs(cmd))
        for line in listout.splitlines():
          (vfy_name, vfy_allocatable) = line.split("|")
          if vfy_name == st_name and not fail:
            AssertEqual(vfy_allocatable, i[0].upper())
          else:
            AssertEqual(vfy_allocatable, st_allocatable)

      # Test repair functionality
      fail = (constants.SO_FIX_CONSISTENCY not in
              constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
      AssertCommand(["gnt-node", "repair-storage", node_name,
                     storage_type, st_name], fail=fail)


def TestNodeFailover(node, node2):
  """gnt-node failover"""
  if qa_utils.GetNodeInstances(node2, secondaries=False):
    raise qa_error.UnusableNodeError("Secondary node has at least one"
                                     " primary instance. This test requires"
                                     " it to have no primary instances.")

  # Fail over to secondary node
  AssertCommand(["gnt-node", "failover", "-f", node.primary])

  # ... and back again.
  AssertCommand(["gnt-node", "failover", "-f", node2.primary])


def TestNodeMigrate(node, node2):
  """gnt-node migrate"""
  if qa_utils.GetNodeInstances(node2, secondaries=False):
    raise qa_error.UnusableNodeError("Secondary node has at least one"
                                     " primary instance. This test requires"
                                     " it to have no primary instances.")

  # Migrate to secondary node
  AssertCommand(["gnt-node", "migrate", "-f", node.primary])

  # ... and back again.
  AssertCommand(["gnt-node", "migrate", "-f", node2.primary])


def TestNodeEvacuate(node, node2):
  """gnt-node evacuate"""
  node3 = qa_config.AcquireNode(exclude=[node, node2])
  try:
    if qa_utils.GetNodeInstances(node3, secondaries=True):
      raise qa_error.UnusableNodeError("Evacuation node has at least one"
                                       " secondary instance. This test requires"
                                       " it to have no secondary instances.")

    # Evacuate all secondary instances
    AssertCommand(["gnt-node", "evacuate", "-f",
                   "--new-secondary=%s" % node3.primary, node2.primary])

    # ... and back again.
    AssertCommand(["gnt-node", "evacuate", "-f",
                   "--new-secondary=%s" % node2.primary, node3.primary])
  finally:
    node3.Release()


def TestNodeModify(node):
  """gnt-node modify"""

  # make sure enough master candidates will be available by disabling the
  # master candidate role first with --auto-promote
  AssertCommand(["gnt-node", "modify", "--master-candidate=no",
                "--auto-promote", node.primary])

  # now it's save to force-remove the master candidate role
  for flag in ["master-candidate", "drained", "offline"]:
    for value in ["yes", "no"]:
      AssertCommand(["gnt-node", "modify", "--force",
                     "--%s=%s" % (flag, value), node.primary])

  AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])

  # Test setting secondary IP address
  AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
                 node.primary])

  AssertRedirectedCommand(["gnt-cluster", "verify"])


def _CreateOobScriptStructure():
  """Create a simple OOB handling script and its structure."""
  master = qa_config.GetMasterNode()

  data_path = qa_utils.UploadData(master.primary, "")
  verify_path = qa_utils.UploadData(master.primary, "")
  exit_code_path = qa_utils.UploadData(master.primary, "")

  oob_script = (("#!/bin/bash\n"
                 "echo \"$@\" > %s\n"
                 "cat %s\n"
                 "exit $(< %s)\n") %
                (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
                 utils.ShellQuote(exit_code_path)))
  oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)

  return [oob_path, verify_path, data_path, exit_code_path]


def _UpdateOobFile(path, data):
  """Updates the data file with data."""
  master = qa_config.GetMasterNode()
  qa_utils.UploadData(master.primary, data, filename=path)


def _AssertOobCall(verify_path, expected_args):
  """Assert the OOB call was performed with expetected args."""
  master = qa_config.GetMasterNode()

  verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
  output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
                                     tty=False)

  AssertEqual(expected_args, output.strip())


def TestOutOfBand():
  """gnt-node power"""
  master = qa_config.GetMasterNode()

  node = qa_config.AcquireNode(exclude=master)

  master_name = master.primary
  node_name = node.primary
  full_node_name = qa_utils.ResolveNodeName(node)

  (oob_path, verify_path,
   data_path, exit_code_path) = _CreateOobScriptStructure()

  try:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program=%s" % oob_path])

    # No data, exit 0
    _UpdateOobFile(exit_code_path, "0")

    AssertCommand(["gnt-node", "power", "on", node_name])
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)

    AssertCommand(["gnt-node", "power", "-f", "off", node_name])
    _AssertOobCall(verify_path, "power-off %s" % full_node_name)

    # Power off on master without options should fail
    AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
    # With force master it should still fail
    AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
                   master_name],
                  fail=True)

    # Verify we can't transform back to online when not yet powered on
    AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
                  fail=True)
    # Now reset state
    AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
                   node_name])

    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)

    # Those commands should fail as they expect output which isn't provided yet
    # But they should have called the oob helper nevermind
    AssertCommand(["gnt-node", "power", "status", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)

    AssertCommand(["gnt-node", "health", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"], fail=True)

    # Correct Data, exit 0
    _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))

    AssertCommand(["gnt-node", "power", "status", node_name])
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)

    _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
                                                   ["disk0", "CRITICAL"]]))

    AssertCommand(["gnt-node", "health", node_name])
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"])

    # Those commands should fail as they expect no data regardless of exit 0
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)

    try:
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
    finally:
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])

    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)

    # Data, exit 1 (all should fail)
    _UpdateOobFile(exit_code_path, "1")

    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)

    try:
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
    finally:
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])

    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)

    AssertCommand(["gnt-node", "power", "status", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)

    AssertCommand(["gnt-node", "health", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"], fail=True)

    # No data, exit 1 (all should fail)
    _UpdateOobFile(data_path, "")
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)

    try:
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
    finally:
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])

    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)

    AssertCommand(["gnt-node", "power", "status", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)

    AssertCommand(["gnt-node", "health", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"], fail=True)

    # Different OOB script for node
    verify_path2 = qa_utils.UploadData(master.primary, "")
    oob_script = ("#!/bin/sh\n"
                  "echo \"$@\" > %s\n") % verify_path2
    oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)

    try:
      AssertCommand(["gnt-node", "modify", "--node-parameters",
                     "oob_program=%s" % oob_path2, node_name])
      AssertCommand(["gnt-node", "power", "on", node_name])
      _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
    finally:
      AssertCommand(["gnt-node", "modify", "--node-parameters",
                     "oob_program=default", node_name])
      AssertCommand(["rm", "-f", oob_path2, verify_path2])
  finally:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program="])
    AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
                   exit_code_path])


def TestNodeList():
  """gnt-node list"""
  qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())


def TestNodeListFields():
  """gnt-node list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())


def TestNodeListDrbd(node, is_drbd):
  """gnt-node list-drbd"""
  master = qa_config.GetMasterNode()
  result_output = GetCommandOutput(master.primary,
                                   "gnt-node list-drbd --no-header %s" %
                                   node.primary)
  # Meaningful to note: there is but one instance, and the node is either the
  # primary or one of the secondaries
  if is_drbd:
    # Invoked for both primary and secondary
    per_disk_info = result_output.splitlines()
    for line in per_disk_info:
      try:
        drbd_node, _, _, _, _, drbd_peer = line.split()
      except ValueError:
        raise qa_error.Error("Could not examine list-drbd output: expected a"
                             " single row of 6 entries, found the following:"
                             " %s" % line)

      AssertIn(node.primary, [drbd_node, drbd_peer],
               msg="The output %s does not contain the node" % line)
  else:
    # Output should be empty, barring newlines
    AssertEqual(result_output.strip(), "")


def _BuildSetESCmd(action, value, node_name):
  cmd = ["gnt-node"]
  if action == "add":
    cmd.extend(["add", "--readd"])
    if not qa_config.GetModifySshSetup():
      cmd.append("--no-node-setup")
  else:
    cmd.append("modify")
  cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
  return cmd


def TestExclStorSingleNode(node):
  """gnt-node add/modify cannot change the exclusive_storage flag.

  """
  for action in ["add", "modify"]:
    for value in (True, False, "default"):
      AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)