qa_utils.py 25.5 KB
Newer Older
1 2 3
#
#

4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Utilities for QA tests.

"""

26
import copy
27
import operator
28
import os
29
import random
30
import re
31
import socket
32
import subprocess
33
import sys
34
import tempfile
35
import yaml
36

37 38 39 40 41
try:
  import functools
except ImportError, err:
  raise ImportError("Python 2.5 or higher is required: %s" % err)

42
from ganeti import utils
43
from ganeti import compat
44
from ganeti import constants
45
from ganeti import ht
46
from ganeti import pathutils
47
from ganeti import vcluster
48

49
import colors
50 51 52 53
import qa_config
import qa_error


Michael Hanselmann's avatar
Michael Hanselmann committed
54 55 56 57 58
_INFO_SEQ = None
_WARNING_SEQ = None
_ERROR_SEQ = None
_RESET_SEQ = None

59 60
_MULTIPLEXERS = {}

61 62 63
#: Unique ID per QA run
_RUN_UUID = utils.NewUUID()

64 65 66
#: Path to the QA query output log file
_QA_OUTPUT = pathutils.GetLogFilename("qa-output")

67 68 69 70 71 72 73

(INST_DOWN,
 INST_UP) = range(500, 502)

(FIRST_ARG,
 RETURN_VALUE) = range(1000, 1002)

Michael Hanselmann's avatar
Michael Hanselmann committed
74 75 76 77 78

def _SetupColours():
  """Initializes the colour constants.

  """
79
  # pylint: disable=W0603
Iustin Pop's avatar
Iustin Pop committed
80
  # due to global usage
Michael Hanselmann's avatar
Michael Hanselmann committed
81 82
  global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ

83 84 85 86
  # Don't use colours if stdout isn't a terminal
  if not sys.stdout.isatty():
    return

Michael Hanselmann's avatar
Michael Hanselmann committed
87 88 89 90 91 92
  try:
    import curses
  except ImportError:
    # Don't use colours if curses module can't be imported
    return

93 94 95 96 97
  try:
    curses.setupterm()
  except curses.error:
    # Probably a non-standard terminal, don't use colours then
    return
Michael Hanselmann's avatar
Michael Hanselmann committed
98 99 100 101 102 103 104 105 106 107 108 109

  _RESET_SEQ = curses.tigetstr("op")

  setaf = curses.tigetstr("setaf")
  _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
  _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
  _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)


_SetupColours()


110 111 112 113 114
def AssertIn(item, sequence):
  """Raises an error when item is not in sequence.

  """
  if item not in sequence:
Iustin Pop's avatar
Iustin Pop committed
115
    raise qa_error.Error("%r not in %r" % (item, sequence))
116 117


118 119 120 121 122
def AssertNotIn(item, sequence):
  """Raises an error when item is in sequence.

  """
  if item in sequence:
Iustin Pop's avatar
Iustin Pop committed
123
    raise qa_error.Error("%r in %r" % (item, sequence))
124 125


126
def AssertEqual(first, second, msg=""):
127 128 129 130
  """Raises an error when values aren't equal.

  """
  if not first == second:
131 132 133 134
    if msg:
      raise qa_error.Error("%s: %r == %r" % (msg, first, second))
    else:
      raise qa_error.Error("%r == %r" % (first, second))
135 136


137 138 139 140 141 142 143 144
def AssertMatch(string, pattern):
  """Raises an error when string doesn't match regexp pattern.

  """
  if not re.match(pattern, string):
    raise qa_error.Error("%r doesn't match /%r/" % (string, pattern))


145
def _GetName(entity, fn):
146 147 148
  """Tries to get name of an entity.

  @type entity: string or dict
149
  @param fn: Function retrieving name from entity
150 151 152 153 154

  """
  if isinstance(entity, basestring):
    result = entity
  else:
155
    result = fn(entity)
156 157 158 159 160 161 162

  if not ht.TNonEmptyString(result):
    raise Exception("Invalid name '%s'" % result)

  return result


163 164 165 166 167 168 169 170 171 172 173 174
def _AssertRetCode(rcode, fail, cmdstr, nodename):
  """Check the return value from a command and possibly raise an exception.

  """
  if fail and rcode == 0:
    raise qa_error.Error("Command '%s' on node %s was expected to fail but"
                         " didn't" % (cmdstr, nodename))
  elif not fail and rcode != 0:
    raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
                         (cmdstr, nodename, rcode))


175
def AssertCommand(cmd, fail=False, node=None, log_cmd=True):
Iustin Pop's avatar
Iustin Pop committed
176 177 178 179 180 181 182 183 184
  """Checks that a remote command succeeds.

  @param cmd: either a string (the command to execute) or a list (to
      be converted using L{utils.ShellQuoteArgs} into a string)
  @type fail: boolean
  @param fail: if the command is expected to fail instead of succeeding
  @param node: if passed, it should be the node on which the command
      should be executed, instead of the master node (can be either a
      dict or a string)
185 186
  @param log_cmd: if False, the command won't be logged (simply passed to
      StartSSH)
187 188
  @return: the return code of the command
  @raise qa_error.Error: if the command fails when it shouldn't or vice versa
Iustin Pop's avatar
Iustin Pop committed
189 190 191 192 193

  """
  if node is None:
    node = qa_config.GetMasterNode()

194
  nodename = _GetName(node, operator.attrgetter("primary"))
Iustin Pop's avatar
Iustin Pop committed
195 196 197 198 199 200

  if isinstance(cmd, basestring):
    cmdstr = cmd
  else:
    cmdstr = utils.ShellQuoteArgs(cmd)

201
  rcode = StartSSH(nodename, cmdstr, log_cmd=log_cmd).wait()
202
  _AssertRetCode(rcode, fail, cmdstr, nodename)
Iustin Pop's avatar
Iustin Pop committed
203

204 205
  return rcode

Iustin Pop's avatar
Iustin Pop committed
206

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True):
  """Executes a command with redirected output.

  The log will go to the qa-output log file in the ganeti log
  directory on the node where the command is executed. The fail and
  node parameters are passed unchanged to AssertCommand.

  @param cmd: the command to be executed, as a list; a string is not
      supported

  """
  if not isinstance(cmd, list):
    raise qa_error.Error("Non-list passed to AssertRedirectedCommand")
  ofile = utils.ShellQuote(_QA_OUTPUT)
  cmdstr = utils.ShellQuoteArgs(cmd)
  AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile),
                fail=False, node=node, log_cmd=False)
  return AssertCommand(cmdstr + " >> %s" % ofile,
                       fail=fail, node=node, log_cmd=log_cmd)


Iustin Pop's avatar
Iustin Pop committed
228
def GetSSHCommand(node, cmd, strict=True, opts=None, tty=None):
229 230
  """Builds SSH command to be executed.

231 232 233
  @type node: string
  @param node: node the command should run on
  @type cmd: string
234 235
  @param cmd: command to be executed in the node; if None or empty
      string, no command will be executed
236 237
  @type strict: boolean
  @param strict: whether to enable strict host key checking
238 239
  @type opts: list
  @param opts: list of additional options
Iustin Pop's avatar
Iustin Pop committed
240 241
  @type tty: boolean or None
  @param tty: if we should use tty; if None, will be auto-detected
242

243
  """
244
  args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-lroot"]
245

Iustin Pop's avatar
Iustin Pop committed
246 247 248
  if tty is None:
    tty = sys.stdout.isatty()

249 250
  if tty:
    args.append("-t")
251 252

  if strict:
Iustin Pop's avatar
Iustin Pop committed
253
    tmp = "yes"
254
  else:
Iustin Pop's avatar
Iustin Pop committed
255 256 257 258
    tmp = "no"
  args.append("-oStrictHostKeyChecking=%s" % tmp)
  args.append("-oClearAllForwardings=yes")
  args.append("-oForwardAgent=yes")
259 260 261 262
  if opts:
    args.extend(opts)
  if node in _MULTIPLEXERS:
    spath = _MULTIPLEXERS[node][0]
Iustin Pop's avatar
Iustin Pop committed
263 264
    args.append("-oControlPath=%s" % spath)
    args.append("-oControlMaster=no")
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283

  (vcluster_master, vcluster_basedir) = \
    qa_config.GetVclusterSettings()

  if vcluster_master:
    args.append(vcluster_master)
    args.append("%s/%s/cmd" % (vcluster_basedir, node))

    if cmd:
      # For virtual clusters the whole command must be wrapped using the "cmd"
      # script, as that script sets a number of environment variables. If the
      # command contains shell meta characters the whole command needs to be
      # quoted.
      args.append(utils.ShellQuote(cmd))
  else:
    args.append(node)

    if cmd:
      args.append(cmd)
284 285 286 287

  return args


288
def StartLocalCommand(cmd, _nolog_opts=False, log_cmd=True, **kwargs):
289 290 291
  """Starts a local command.

  """
292 293 294 295 296
  if log_cmd:
    if _nolog_opts:
      pcmd = [i for i in cmd if not i.startswith("-")]
    else:
      pcmd = cmd
297 298
    print "%s %s" % (colors.colorize("Command:", colors.CYAN),
                     utils.ShellQuoteArgs(pcmd))
299 300 301
  return subprocess.Popen(cmd, shell=False, **kwargs)


302
def StartSSH(node, cmd, strict=True, log_cmd=True):
303 304 305
  """Starts SSH.

  """
306
  return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict),
307
                           _nolog_opts=True, log_cmd=log_cmd)
308 309


310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
def StartMultiplexer(node):
  """Starts a multiplexer command.

  @param node: the node for which to open the multiplexer

  """
  if node in _MULTIPLEXERS:
    return

  # Note: yes, we only need mktemp, since we'll remove the file anyway
  sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
  utils.RemoveFile(sname)
  opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
  print "Created socket at %s" % sname
  child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
  _MULTIPLEXERS[node] = (sname, child)


def CloseMultiplexers():
  """Closes all current multiplexers and cleans up.

  """
  for node in _MULTIPLEXERS.keys():
    (sname, child) = _MULTIPLEXERS.pop(node)
    utils.KillProcess(child.pid, timeout=10, waitpid=True)
    utils.RemoveFile(sname)


338
def GetCommandOutput(node, cmd, tty=None, fail=False):
339 340
  """Returns the output of a command executed on the given node.

341 342 343 344 345 346 347 348
  @type node: string
  @param node: node the command should run on
  @type cmd: string
  @param cmd: command to be executed in the node (cannot be empty or None)
  @type tty: bool or None
  @param tty: if we should use tty; if None, it will be auto-detected
  @type fail: bool
  @param fail: whether the command is expected to fail
349
  """
350
  assert cmd
351 352
  p = StartLocalCommand(GetSSHCommand(node, cmd, tty=tty),
                        stdout=subprocess.PIPE)
353
  rcode = p.wait()
354
  _AssertRetCode(rcode, fail, cmd, node)
355
  return p.stdout.read()
356 357


358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
def GetObjectInfo(infocmd):
  """Get and parse information about a Ganeti object.

  @type infocmd: list of strings
  @param infocmd: command to be executed, e.g. ["gnt-cluster", "info"]
  @return: the information parsed, appropriately stored in dictionaries,
      lists...

  """
  master = qa_config.GetMasterNode()
  cmdline = utils.ShellQuoteArgs(infocmd)
  info_out = GetCommandOutput(master.primary, cmdline)
  return yaml.load(info_out)


373 374 375 376 377
def UploadFile(node, src):
  """Uploads a file to a node and returns the filename.

  Caller needs to remove the returned file on the node when it's not needed
  anymore.
378

379 380 381 382
  """
  # Make sure nobody else has access to it while preserving local permissions
  mode = os.stat(src).st_mode & 0700

383 384
  cmd = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
         'chmod %o "${tmp}" && '
385 386 387 388
         '[[ -f "${tmp}" ]] && '
         'cat > "${tmp}" && '
         'echo "${tmp}"') % mode

Iustin Pop's avatar
Iustin Pop committed
389
  f = open(src, "r")
390 391 392 393 394 395 396 397 398
  try:
    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
                         stdout=subprocess.PIPE)
    AssertEqual(p.wait(), 0)

    # Return temporary filename
    return p.stdout.read().strip()
  finally:
    f.close()
Michael Hanselmann's avatar
Michael Hanselmann committed
399 400


401 402 403 404 405 406 407 408 409 410
def UploadData(node, data, mode=0600, filename=None):
  """Uploads data to a node and returns the filename.

  Caller needs to remove the returned file on the node when it's not needed
  anymore.

  """
  if filename:
    tmp = "tmp=%s" % utils.ShellQuote(filename)
  else:
411 412
    tmp = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
           'chmod %o "${tmp}"') % mode
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
  cmd = ("%s && "
         "[[ -f \"${tmp}\" ]] && "
         "cat > \"${tmp}\" && "
         "echo \"${tmp}\"") % tmp

  p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  p.stdin.write(data)
  p.stdin.close()
  AssertEqual(p.wait(), 0)

  # Return temporary filename
  return p.stdout.read().strip()


428 429 430 431 432 433 434
def BackupFile(node, path):
  """Creates a backup of a file on the node and returns the filename.

  Caller needs to remove the returned file on the node when it's not needed
  anymore.

  """
435 436
  vpath = MakeNodePath(node, path)

437
  cmd = ("tmp=$(mktemp .gnt.XXXXXX --tmpdir=$(dirname %s)) && "
438 439
         "[[ -f \"$tmp\" ]] && "
         "cp %s $tmp && "
440
         "echo $tmp") % (utils.ShellQuote(vpath), utils.ShellQuote(vpath))
441 442

  # Return temporary filename
443 444 445 446 447
  result = GetCommandOutput(node, cmd).strip()

  print "Backup filename: %s" % result

  return result
448 449


Michael Hanselmann's avatar
Michael Hanselmann committed
450 451 452
def ResolveInstanceName(instance):
  """Gets the full name of an instance.

453 454 455
  @type instance: string
  @param instance: Instance name

456
  """
457 458
  info = GetObjectInfo(["gnt-instance", "info", instance])
  return info[0]["Instance name"]
459 460 461 462 463 464


def ResolveNodeName(node):
  """Gets the full name of a node.

  """
465 466
  info = GetObjectInfo(["gnt-node", "info", node.primary])
  return info[0]["Node name"]
467 468 469 470 471


def GetNodeInstances(node, secondaries=False):
  """Gets a list of instances on a node.

Michael Hanselmann's avatar
Michael Hanselmann committed
472 473
  """
  master = qa_config.GetMasterNode()
474
  node_name = ResolveNodeName(node)
Michael Hanselmann's avatar
Michael Hanselmann committed
475

476
  # Get list of all instances
Iustin Pop's avatar
Iustin Pop committed
477 478
  cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
         "--output=name,pnode,snodes"]
479
  output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
480 481 482

  instances = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
483
    (name, pnode, snodes) = line.split(":", 2)
484
    if ((not secondaries and pnode == node_name) or
Iustin Pop's avatar
Iustin Pop committed
485
        (secondaries and node_name in snodes.split(","))):
486
      instances.append(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
487

488
  return instances
Michael Hanselmann's avatar
Michael Hanselmann committed
489 490


491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
def _SelectQueryFields(rnd, fields):
  """Generates a list of fields for query tests.

  """
  # Create copy for shuffling
  fields = list(fields)
  rnd.shuffle(fields)

  # Check all fields
  yield fields
  yield sorted(fields)

  # Duplicate fields
  yield fields + fields

  # Check small groups of fields
  while fields:
    yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]


def _List(listcmd, fields, names):
  """Runs a list command.

  """
  master = qa_config.GetMasterNode()

517
  cmd = [listcmd, "list", "--separator=|", "--no-headers",
518 519 520 521 522
         "--output", ",".join(fields)]

  if names:
    cmd.extend(names)

523
  return GetCommandOutput(master.primary,
524 525 526
                          utils.ShellQuoteArgs(cmd)).splitlines()


527
def GenericQueryTest(cmd, fields, namefield="name", test_unknown=True):
528 529 530 531 532 533 534 535
  """Runs a number of tests on query commands.

  @param cmd: Command name
  @param fields: List of field names

  """
  rnd = random.Random(hash(cmd))

Iustin Pop's avatar
Iustin Pop committed
536
  fields = list(fields)
537 538 539 540
  rnd.shuffle(fields)

  # Test a number of field combinations
  for testfields in _SelectQueryFields(rnd, fields):
541
    AssertRedirectedCommand([cmd, "list", "--output", ",".join(testfields)])
542

543 544
  if namefield is not None:
    namelist_fn = compat.partial(_List, cmd, [namefield])
545

546 547 548
    # When no names were requested, the list must be sorted
    names = namelist_fn(None)
    AssertEqual(names, utils.NiceSort(names))
549

550 551 552
    # When requesting specific names, the order must be kept
    revnames = list(reversed(names))
    AssertEqual(namelist_fn(revnames), revnames)
553

554 555 556
    randnames = list(names)
    rnd.shuffle(randnames)
    AssertEqual(namelist_fn(randnames), randnames)
557

558 559 560 561
  if test_unknown:
    # Listing unknown items must fail
    AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"],
                  fail=True)
562 563

  # Check exit code for listing unknown field
564 565 566
  AssertEqual(AssertRedirectedCommand([cmd, "list",
                                       "--output=field/does/not/exist"],
                                      fail=True),
567 568 569 570 571 572 573
              constants.EXIT_UNKNOWN_FIELD)


def GenericQueryFieldsTest(cmd, fields):
  master = qa_config.GetMasterNode()

  # Listing fields
574 575
  AssertRedirectedCommand([cmd, "list-fields"])
  AssertRedirectedCommand([cmd, "list-fields"] + fields)
576 577 578

  # Check listed fields (all, must be sorted)
  realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
579
  output = GetCommandOutput(master.primary,
580 581
                            utils.ShellQuoteArgs(realcmd)).splitlines()
  AssertEqual([line.split("|", 1)[0] for line in output],
582
              utils.NiceSort(fields))
583 584 585 586 587 588

  # Check exit code for listing unknown field
  AssertEqual(AssertCommand([cmd, "list-fields", "field/does/not/exist"],
                            fail=True),
              constants.EXIT_UNKNOWN_FIELD)

589

590 591 592 593
def _FormatWithColor(text, seq):
  if not seq:
    return text
  return "%s%s%s" % (seq, text, _RESET_SEQ)
Michael Hanselmann's avatar
Michael Hanselmann committed
594 595


596 597 598
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
599 600 601 602 603 604 605 606 607


def AddToEtcHosts(hostnames):
  """Adds hostnames to /etc/hosts.

  @param hostnames: List of hostnames first used A records, all other CNAMEs

  """
  master = qa_config.GetMasterNode()
608
  tmp_hosts = UploadData(master.primary, "", mode=0644)
609 610 611 612 613 614

  data = []
  for localhost in ("::1", "127.0.0.1"):
    data.append("%s %s" % (localhost, " ".join(hostnames)))

  try:
615 616 617 618 619 620 621 622 623
    AssertCommand("{ cat %s && echo -e '%s'; } > %s && mv %s %s" %
                  (utils.ShellQuote(pathutils.ETC_HOSTS),
                   "\\n".join(data),
                   utils.ShellQuote(tmp_hosts),
                   utils.ShellQuote(tmp_hosts),
                   utils.ShellQuote(pathutils.ETC_HOSTS)))
  except Exception:
    AssertCommand(["rm", "-f", tmp_hosts])
    raise
624 625 626 627 628 629 630 631 632


def RemoveFromEtcHosts(hostnames):
  """Remove hostnames from /etc/hosts.

  @param hostnames: List of hostnames first used A records, all other CNAMEs

  """
  master = qa_config.GetMasterNode()
633
  tmp_hosts = UploadData(master.primary, "", mode=0644)
634 635 636 637
  quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)

  sed_data = " ".join(hostnames)
  try:
638 639
    AssertCommand((r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
                   r" && mv %s %s") %
640 641 642 643 644 645
                   (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS),
                    quoted_tmp_hosts, quoted_tmp_hosts,
                    utils.ShellQuote(pathutils.ETC_HOSTS)))
  except Exception:
    AssertCommand(["rm", "-f", tmp_hosts])
    raise
646 647 648 649 650 651


def RunInstanceCheck(instance, running):
  """Check if instance is running or not.

  """
652
  instance_name = _GetName(instance, operator.attrgetter("name"))
653

654 655 656 657 658 659 660
  script = qa_config.GetInstanceCheckScript()
  if not script:
    return

  master_node = qa_config.GetMasterNode()

  # Build command to connect to master node
661
  master_ssh = GetSSHCommand(master_node.primary, "--")
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726

  if running:
    running_shellval = "1"
    running_text = ""
  else:
    running_shellval = ""
    running_text = "not "

  print FormatInfo("Checking if instance '%s' is %srunning" %
                   (instance_name, running_text))

  args = [script, instance_name]
  env = {
    "PATH": constants.HOOKS_PATH,
    "RUN_UUID": _RUN_UUID,
    "MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
    "INSTANCE_NAME": instance_name,
    "INSTANCE_RUNNING": running_shellval,
    }

  result = os.spawnve(os.P_WAIT, script, args, env)
  if result != 0:
    raise qa_error.Error("Instance check failed with result %s" % result)


def _InstanceCheckInner(expected, instarg, args, result):
  """Helper function used by L{InstanceCheck}.

  """
  if instarg == FIRST_ARG:
    instance = args[0]
  elif instarg == RETURN_VALUE:
    instance = result
  else:
    raise Exception("Invalid value '%s' for instance argument" % instarg)

  if expected in (INST_DOWN, INST_UP):
    RunInstanceCheck(instance, (expected == INST_UP))
  elif expected is not None:
    raise Exception("Invalid value '%s'" % expected)


def InstanceCheck(before, after, instarg):
  """Decorator to check instance status before and after test.

  @param before: L{INST_DOWN} if instance must be stopped before test,
    L{INST_UP} if instance must be running before test, L{None} to not check.
  @param after: L{INST_DOWN} if instance must be stopped after test,
    L{INST_UP} if instance must be running after test, L{None} to not check.
  @param instarg: L{FIRST_ARG} to use first argument to test as instance (a
    dictionary), L{RETURN_VALUE} to use return value (disallows pre-checks)

  """
  def decorator(fn):
    @functools.wraps(fn)
    def wrapper(*args, **kwargs):
      _InstanceCheckInner(before, instarg, args, NotImplemented)

      result = fn(*args, **kwargs)

      _InstanceCheckInner(after, instarg, args, result)

      return result
    return wrapper
  return decorator
727 728 729 730 731 732


def GetNonexistentGroups(count):
  """Gets group names which shouldn't exist on the cluster.

  @param count: Number of groups to get
Helga Velroyen's avatar
Helga Velroyen committed
733
  @rtype: integer
734 735

  """
Helga Velroyen's avatar
Helga Velroyen committed
736
  return GetNonexistentEntityNames(count, "groups", "group")
737

Helga Velroyen's avatar
Helga Velroyen committed
738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759

def GetNonexistentEntityNames(count, name_config, name_prefix):
  """Gets entity names which shouldn't exist on the cluster.

  The actualy names can refer to arbitrary entities (for example
  groups, networks).

  @param count: Number of names to get
  @rtype: integer
  @param name_config: name of the leaf in the config containing
    this entity's configuration, including a 'inexistent-'
    element
  @rtype: string
  @param name_prefix: prefix of the entity's names, used to compose
    the default values; for example for groups, the prefix is
    'group' and the generated names are then group1, group2, ...
  @rtype: string

  """
  entities = qa_config.get(name_config, {})

  default = [name_prefix + str(i) for i in range(count)]
760 761
  assert count <= len(default)

Helga Velroyen's avatar
Helga Velroyen committed
762 763
  name_config_inexistent = "inexistent-" + name_config
  candidates = entities.get(name_config_inexistent, default)[:count]
764 765

  if len(candidates) < count:
Helga Velroyen's avatar
Helga Velroyen committed
766 767
    raise Exception("At least %s non-existent %s are needed" %
                    (count, name_config))
768 769

  return candidates
770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792


def MakeNodePath(node, path):
  """Builds an absolute path for a virtual node.

  @type node: string or L{qa_config._QaNode}
  @param node: Node
  @type path: string
  @param path: Path without node-specific prefix

  """
  (_, basedir) = qa_config.GetVclusterSettings()

  if isinstance(node, basestring):
    name = node
  else:
    name = node.primary

  if basedir:
    assert path.startswith("/")
    return "%s%s" % (vcluster.MakeNodeRoot(basedir, name), path)
  else:
    return path
793 794


795
def _GetParameterOptions(specs):
796
  """Helper to build policy options."""
797 798
  values = ["%s=%s" % (par, val)
            for (par, val) in specs.items()]
799 800 801
  return ",".join(values)


802 803
def TestSetISpecs(new_specs=None, diff_specs=None, get_policy_fn=None,
                  build_cmd_fn=None, fail=False, old_values=None):
804 805
  """Change instance specs for an object.

806 807 808 809 810 811
  At most one of new_specs or diff_specs can be specified.

  @type new_specs: dict
  @param new_specs: new complete specs, in the same format returned by
      L{ParseIPolicy}.
  @type diff_specs: dict
812 813 814
  @param diff_specs: partial specs, it can be an incomplete specifications, but
      if min/max specs are specified, their number must match the number of the
      existing specs
815 816
  @type get_policy_fn: function
  @param get_policy_fn: function that returns the current policy as in
817
      L{ParseIPolicy}
818 819 820 821 822 823 824
  @type build_cmd_fn: function
  @param build_cmd_fn: function that return the full command line from the
      options alone
  @type fail: bool
  @param fail: if the change is expected to fail
  @type old_values: tuple
  @param old_values: (old_policy, old_specs), as returned by
825 826
     L{ParseIPolicy}
  @return: same as L{ParseIPolicy}
827 828 829 830

  """
  assert get_policy_fn is not None
  assert build_cmd_fn is not None
831
  assert new_specs is None or diff_specs is None
832 833 834 835 836

  if old_values:
    (old_policy, old_specs) = old_values
  else:
    (old_policy, old_specs) = get_policy_fn()
837 838 839

  if diff_specs:
    new_specs = copy.deepcopy(old_specs)
840 841 842 843 844 845 846 847 848 849
    if constants.ISPECS_MINMAX in diff_specs:
      AssertEqual(len(new_specs[constants.ISPECS_MINMAX]),
                  len(diff_specs[constants.ISPECS_MINMAX]))
      for (new_minmax, diff_minmax) in zip(new_specs[constants.ISPECS_MINMAX],
                                           diff_specs[constants.ISPECS_MINMAX]):
        for (key, parvals) in diff_minmax.items():
          for (par, val) in parvals.items():
            new_minmax[key][par] = val
    for (par, val) in diff_specs.get(constants.ISPECS_STD, {}).items():
      new_specs[constants.ISPECS_STD][par] = val
850

851 852
  if new_specs:
    cmd = []
853
    if (diff_specs is None or constants.ISPECS_MINMAX in diff_specs):
854
      minmax_opt_items = []
855 856 857 858 859 860
      for minmax in new_specs[constants.ISPECS_MINMAX]:
        minmax_opts = []
        for key in ["min", "max"]:
          keyopt = _GetParameterOptions(minmax[key])
          minmax_opts.append("%s:%s" % (key, keyopt))
        minmax_opt_items.append("/".join(minmax_opts))
861 862
      cmd.extend([
        "--ipolicy-bounds-specs",
863
        "//".join(minmax_opt_items)
864
        ])
865
    if diff_specs is None:
866
      std_source = new_specs
867 868
    else:
      std_source = diff_specs
869
    std_opt = _GetParameterOptions(std_source.get("std", {}))
870 871 872 873
    if std_opt:
      cmd.extend(["--ipolicy-std-specs", std_opt])
    AssertCommand(build_cmd_fn(cmd), fail=fail)

874 875 876 877 878 879 880 881
    # Check the new state
    (eff_policy, eff_specs) = get_policy_fn()
    AssertEqual(eff_policy, old_policy)
    if fail:
      AssertEqual(eff_specs, old_specs)
    else:
      AssertEqual(eff_specs, new_specs)

882
  else:
883 884
    (eff_policy, eff_specs) = (old_policy, old_specs)

885 886 887 888 889 890 891 892 893 894 895
  return (eff_policy, eff_specs)


def ParseIPolicy(policy):
  """Parse and split instance an instance policy.

  @type policy: dict
  @param policy: policy, as returned by L{GetObjectInfo}
  @rtype: tuple
  @return: (policy, specs), where:
      - policy is a dictionary of the policy values, instance specs excluded
896 897
      - specs is a dictionary containing only the specs, using the internal
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
898 899 900 901 902

  """
  ret_specs = {}
  ret_policy = {}
  for (key, val) in policy.items():
903 904 905 906 907 908 909 910 911 912
    if key == "bounds specs":
      ret_specs[constants.ISPECS_MINMAX] = []
      for minmax in val:
        ret_minmax = {}
        for key in minmax:
          keyparts = key.split("/", 1)
          assert len(keyparts) > 1
          ret_minmax[keyparts[0]] = minmax[key]
        ret_specs[constants.ISPECS_MINMAX].append(ret_minmax)
    elif key == constants.ISPECS_STD:
913
      ret_specs[key] = val
914 915 916
    else:
      ret_policy[key] = val
  return (ret_policy, ret_specs)
917 918 919 920 921 922 923


def UsesIPv6Connection(host, port):
  """Returns True if the connection to a given host/port could go through IPv6.

  """
  return any(t[0] == socket.AF_INET6 for t in socket.getaddrinfo(host, port))