qa_cluster.py 21.5 KB
Newer Older
1 2 3
#
#

4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Cluster related QA tests.

"""

26
import re
27
import tempfile
28
import os.path
29

30
from ganeti import constants
René Nussbaumer's avatar
René Nussbaumer committed
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34 35 36 37 38

import qa_config
import qa_utils
import qa_error

René Nussbaumer's avatar
René Nussbaumer committed
39
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
40 41


42 43 44
# Prefix for LVM volumes created by QA code during tests
_QA_LV_PREFIX = "qa-"

45 46 47
#: cluster verify command
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]

Andrea Spadaccini's avatar
Andrea Spadaccini committed
48

49 50 51 52
def _RemoveFileFromAllNodes(filename):
  """Removes a file from all nodes.

  """
Iustin Pop's avatar
Iustin Pop committed
53 54
  for node in qa_config.get("nodes"):
    AssertCommand(["rm", "-f", filename], node=node)
55 56 57 58 59 60 61


def _CheckFileOnAllNodes(filename, content):
  """Verifies the content of the given file on all nodes.

  """
  cmd = utils.ShellQuoteArgs(["cat", filename])
Iustin Pop's avatar
Iustin Pop committed
62
  for node in qa_config.get("nodes"):
63
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
64 65


66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
# "gnt-cluster info" fields
_CIFIELD_RE = re.compile(r"^[-\s]*(?P<field>[^\s:]+):\s*(?P<value>\S.*)$")


def _GetBoolClusterField(field):
  """Get the Boolean value of a cluster field.

  This function currently assumes that the field name is unique in the cluster
  configuration. An assertion checks this assumption.

  @type field: string
  @param field: Name of the field
  @rtype: bool
  @return: The effective value of the field

  """
  master = qa_config.GetMasterNode()
  infocmd = "gnt-cluster info"
84
  info_out = qa_utils.GetCommandOutput(master.primary, infocmd)
85 86 87 88 89 90 91 92 93 94 95 96 97
  ret = None
  for l in info_out.splitlines():
    m = _CIFIELD_RE.match(l)
    # FIXME: There should be a way to specify a field through a hierarchy
    if m and m.group("field") == field:
      # Make sure that ignoring the hierarchy doesn't cause a double match
      assert ret is None
      ret = (m.group("value").lower() == "true")
  if ret is not None:
    return ret
  raise qa_error.Error("Field not found in cluster configuration: %s" % field)


98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
# Cluster-verify errors (date, "ERROR", then error code)
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- ERROR:([A-Z0-9_-]+):")


def _GetCVErrorCodes(cvout):
  ret = set()
  for l in cvout.splitlines():
    m = _CVERROR_RE.match(l)
    if m:
      ecode = m.group(1)
      ret.add(ecode)
  return ret


def AssertClusterVerify(fail=False, errors=None):
  """Run cluster-verify and check the result

  @type fail: bool
  @param fail: if cluster-verify is expected to fail instead of succeeding
  @type errors: list of tuples
  @param errors: List of CV_XXX errors that are expected; if specified, all the
      errors listed must appear in cluster-verify output. A non-empty value
      implies C{fail=True}.

  """
  cvcmd = "gnt-cluster verify"
  mnode = qa_config.GetMasterNode()
  if errors:
126
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
127 128 129 130 131 132 133 134 135 136 137
                             fail=True)
    actual = _GetCVErrorCodes(cvout)
    expected = compat.UniqueFrozenset(e for (_, e, _) in errors)
    if not actual.issuperset(expected):
      missing = expected.difference(actual)
      raise qa_error.Error("Cluster-verify didn't return these expected"
                           " errors: %s" % utils.CommaJoin(missing))
  else:
    AssertCommand(cvcmd, fail=fail, node=mnode)


138 139 140 141 142 143 144 145 146 147 148 149 150 151
# data for testing failures due to bad keys/values for disk parameters
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
                "drbd:nonexistent=1",
                "drbd:resync-rate=invalid",
                ]


def TestClusterInitDisk():
  """gnt-cluster init -D"""
  name = qa_config.get("name")
  for param in _FAIL_PARAMS:
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)


152
def TestClusterInit(rapi_user, rapi_secret):
153 154 155
  """gnt-cluster init"""
  master = qa_config.GetMasterNode()

156
  rapi_dir = os.path.dirname(pathutils.RAPI_USERS_FILE)
157

158
  # First create the RAPI credentials
159 160 161 162 163
  fh = tempfile.NamedTemporaryFile()
  try:
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
    fh.flush()

164
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
165
    try:
166
      AssertCommand(["mkdir", "-p", rapi_dir])
167
      AssertCommand(["mv", tmpru, pathutils.RAPI_USERS_FILE])
168
    finally:
Iustin Pop's avatar
Iustin Pop committed
169
      AssertCommand(["rm", "-f", tmpru])
170 171 172 173
  finally:
    fh.close()

  # Initialize cluster
174 175 176
  cmd = [
    "gnt-cluster", "init",
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
177
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
178
    ]
179 180 181 182 183

  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
                    "nic-count"):
    for spec_val in ("min", "max", "std"):
      spec = qa_config.get("ispec_%s_%s" %
184
                           (spec_type.replace("-", "_"), spec_val), None)
185 186
      if spec:
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
Manuel Franceschini's avatar
Manuel Franceschini committed
187

188 189
  if master.secondary:
    cmd.append("--secondary-ip=%s" % master.secondary)
190

191 192 193 194
  vgname = qa_config.get("vg-name", None)
  if vgname:
    cmd.append("--vg-name=%s" % vgname)

195 196 197 198 199 200 201 202
  master_netdev = qa_config.get("master-netdev", None)
  if master_netdev:
    cmd.append("--master-netdev=%s" % master_netdev)

  nicparams = qa_config.get("default-nicparams", None)
  if nicparams:
    cmd.append("--nic-parameters=%s" %
               ",".join(utils.FormatKeyValue(nicparams)))
203

204 205 206 207 208 209 210 211
  # Cluster value of the exclusive-storage node parameter
  e_s = qa_config.get("exclusive-storage")
  if e_s is not None:
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
  else:
    e_s = False
  qa_config.SetExclusiveStorage(e_s)

212 213 214 215
  extra_args = qa_config.get("cluster-init-args")
  if extra_args:
    cmd.extend(extra_args)

216
  cmd.append(qa_config.get("name"))
217

Iustin Pop's avatar
Iustin Pop committed
218
  AssertCommand(cmd)
219

220
  cmd = ["gnt-cluster", "modify"]
221

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
  # hypervisor parameter modifications
  hvp = qa_config.get("hypervisor-parameters", {})
  for k, v in hvp.items():
    cmd.extend(["-H", "%s:%s" % (k, v)])
  # backend parameter modifications
  bep = qa_config.get("backend-parameters", "")
  if bep:
    cmd.extend(["-B", bep])

  if len(cmd) > 2:
    AssertCommand(cmd)

  # OS parameters
  osp = qa_config.get("os-parameters", {})
  for k, v in osp.items():
    AssertCommand(["gnt-os", "modify", "-O", v, k])

  # OS hypervisor parameters
  os_hvp = qa_config.get("os-hvp", {})
  for os_name in os_hvp:
    for hv, hvp in os_hvp[os_name].items():
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])

245

246 247
def TestClusterRename():
  """gnt-cluster rename"""
Iustin Pop's avatar
Iustin Pop committed
248
  cmd = ["gnt-cluster", "rename", "-f"]
249

Iustin Pop's avatar
Iustin Pop committed
250 251
  original_name = qa_config.get("name")
  rename_target = qa_config.get("rename", None)
252 253 254 255
  if rename_target is None:
    print qa_utils.FormatError('"rename" entry is missing')
    return

Iustin Pop's avatar
Iustin Pop committed
256 257
  for data in [
    cmd + [rename_target],
258
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
259
    cmd + [original_name],
260
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
261 262
    ]:
    AssertCommand(data)
263 264


Iustin Pop's avatar
Iustin Pop committed
265 266
def TestClusterOob():
  """out-of-band framework"""
267 268
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()

269
  AssertCommand(_CLUSTER_VERIFY)
270 271 272 273
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
                 utils.NewUUID()])

274
  AssertCommand(_CLUSTER_VERIFY, fail=True)
275

Iustin Pop's avatar
Iustin Pop committed
276 277 278
  AssertCommand(["touch", oob_path_exists])
  AssertCommand(["chmod", "0400", oob_path_exists])
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
279 280 281 282 283

  try:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program=%s" % oob_path_exists])

284
    AssertCommand(_CLUSTER_VERIFY, fail=True)
285

Iustin Pop's avatar
Iustin Pop committed
286 287
    AssertCommand(["chmod", "0500", oob_path_exists])
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
288

289
    AssertCommand(_CLUSTER_VERIFY)
290
  finally:
Iustin Pop's avatar
Iustin Pop committed
291
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
292 293 294

  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program="])
Iustin Pop's avatar
Iustin Pop committed
295 296


René Nussbaumer's avatar
René Nussbaumer committed
297 298 299 300 301
def TestClusterEpo():
  """gnt-cluster epo"""
  master = qa_config.GetMasterNode()

  # Assert that OOB is unavailable for all nodes
302
  result_output = GetCommandOutput(master.primary,
303
                                   "gnt-node list --verbose --no-headers -o"
René Nussbaumer's avatar
René Nussbaumer committed
304 305 306 307 308 309 310 311 312 313
                                   " powered")
  AssertEqual(compat.all(powered == "(unavail)"
                         for powered in result_output.splitlines()), True)

  # Conflicting
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
  # --all doesn't expect arguments
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)

  # Unless --all is given master is not allowed to be in the list
314
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
René Nussbaumer's avatar
René Nussbaumer committed
315 316 317 318 319

  # This shouldn't fail
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])

  # All instances should have been stopped now
320
  result_output = GetCommandOutput(master.primary,
321
                                   "gnt-instance list --no-headers -o status")
322 323
  # ERROR_down because the instance is stopped but not recorded as such
  AssertEqual(compat.all(status == "ERROR_down"
René Nussbaumer's avatar
René Nussbaumer committed
324 325 326 327 328 329
                         for status in result_output.splitlines()), True)

  # Now start everything again
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])

  # All instances should have been started now
330
  result_output = GetCommandOutput(master.primary,
331
                                   "gnt-instance list --no-headers -o status")
René Nussbaumer's avatar
René Nussbaumer committed
332 333 334 335
  AssertEqual(compat.all(status == "running"
                         for status in result_output.splitlines()), True)


Iustin Pop's avatar
Iustin Pop committed
336 337
def TestClusterVerify():
  """gnt-cluster verify"""
338
  AssertCommand(_CLUSTER_VERIFY)
339
  AssertCommand(["gnt-cluster", "verify-disks"])
340

341 342 343

def TestJobqueue():
  """gnt-debug test-jobqueue"""
Iustin Pop's avatar
Iustin Pop committed
344
  AssertCommand(["gnt-debug", "test-jobqueue"])
345 346


347 348 349 350 351
def TestDelay(node):
  """gnt-debug delay"""
  AssertCommand(["gnt-debug", "delay", "1"])
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
  AssertCommand(["gnt-debug", "delay", "--no-master",
352
                 "-n", node.primary, "1"])
353 354


355 356
def TestClusterReservedLvs():
  """gnt-cluster reserved lvs"""
357 358 359
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
  lvname = _QA_LV_PREFIX + "test"
  lvfullname = "/".join([vgname, lvname])
Iustin Pop's avatar
Iustin Pop committed
360
  for fail, cmd in [
361
    (False, _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
362
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
363
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
364
    (True, _CLUSTER_VERIFY),
365
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
366
             "%s,.*/other-test" % lvfullname]),
367
    (False, _CLUSTER_VERIFY),
368 369
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
             ".*/%s.*" % _QA_LV_PREFIX]),
370
    (False, _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
371
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
372
    (True, _CLUSTER_VERIFY),
373
    (False, ["lvremove", "-f", lvfullname]),
374
    (False, _CLUSTER_VERIFY),
375
    ]:
Iustin Pop's avatar
Iustin Pop committed
376
    AssertCommand(cmd, fail=fail)
377

378

379 380 381 382 383
def TestClusterModifyEmpty():
  """gnt-cluster modify"""
  AssertCommand(["gnt-cluster", "modify"], fail=True)


384 385 386 387 388 389
def TestClusterModifyDisk():
  """gnt-cluster modify -D"""
  for param in _FAIL_PARAMS:
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)


390 391
def TestClusterModifyBe():
  """gnt-cluster modify -B"""
Iustin Pop's avatar
Iustin Pop committed
392
  for fail, cmd in [
393 394 395 396 397 398 399 400 401 402 403 404
    # max/min mem
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
405
    # vcpus
Iustin Pop's avatar
Iustin Pop committed
406 407
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
408
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
Iustin Pop's avatar
Iustin Pop committed
409 410
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
411
    # auto_balance
Iustin Pop's avatar
Iustin Pop committed
412 413
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
414
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
Iustin Pop's avatar
Iustin Pop committed
415 416
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
417
    ]:
Iustin Pop's avatar
Iustin Pop committed
418
    AssertCommand(cmd, fail=fail)
419

420 421 422 423
  # redo the original-requested BE parameters, if any
  bep = qa_config.get("backend-parameters", "")
  if bep:
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
424

Andrea Spadaccini's avatar
Andrea Spadaccini committed
425

426 427
def TestClusterInfo():
  """gnt-cluster info"""
Iustin Pop's avatar
Iustin Pop committed
428
  AssertCommand(["gnt-cluster", "info"])
Michael Hanselmann's avatar
Michael Hanselmann committed
429 430


Iustin Pop's avatar
Iustin Pop committed
431 432 433 434 435
def TestClusterRedistConf():
  """gnt-cluster redist-conf"""
  AssertCommand(["gnt-cluster", "redist-conf"])


Michael Hanselmann's avatar
Michael Hanselmann committed
436 437
def TestClusterGetmaster():
  """gnt-cluster getmaster"""
Iustin Pop's avatar
Iustin Pop committed
438
  AssertCommand(["gnt-cluster", "getmaster"])
Michael Hanselmann's avatar
Michael Hanselmann committed
439 440 441 442


def TestClusterVersion():
  """gnt-cluster version"""
Iustin Pop's avatar
Iustin Pop committed
443
  AssertCommand(["gnt-cluster", "version"])
444 445


446 447 448 449 450 451
def TestClusterRenewCrypto():
  """gnt-cluster renew-crypto"""
  master = qa_config.GetMasterNode()

  # Conflicting options
  cmd = ["gnt-cluster", "renew-crypto", "--force",
Michael Hanselmann's avatar
Michael Hanselmann committed
452 453 454 455 456 457
         "--new-cluster-certificate", "--new-confd-hmac-key"]
  conflicting = [
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
    ]
  for i in conflicting:
Andrea Spadaccini's avatar
Andrea Spadaccini committed
458
    AssertCommand(cmd + i, fail=True)
459 460 461 462

  # Invalid RAPI certificate
  cmd = ["gnt-cluster", "renew-crypto", "--force",
         "--rapi-certificate=/dev/null"]
Iustin Pop's avatar
Iustin Pop committed
463
  AssertCommand(cmd, fail=True)
464

465
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
466
                                         pathutils.RAPI_CERT_FILE)
467 468 469
  try:
    # Custom RAPI certificate
    fh = tempfile.NamedTemporaryFile()
470

471 472
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
473

Michael Hanselmann's avatar
Michael Hanselmann committed
474
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
475

476
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
477
    try:
Iustin Pop's avatar
Iustin Pop committed
478 479
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--rapi-certificate=%s" % tmpcert])
480
    finally:
Iustin Pop's avatar
Iustin Pop committed
481
      AssertCommand(["rm", "-f", tmpcert])
482

Michael Hanselmann's avatar
Michael Hanselmann committed
483 484 485 486 487 488
    # Custom cluster domain secret
    cds_fh = tempfile.NamedTemporaryFile()
    cds_fh.write(utils.GenerateSecret())
    cds_fh.write("\n")
    cds_fh.flush()

489
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
Michael Hanselmann's avatar
Michael Hanselmann committed
490
    try:
Iustin Pop's avatar
Iustin Pop committed
491 492
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--cluster-domain-secret=%s" % tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
493
    finally:
Iustin Pop's avatar
Iustin Pop committed
494
      AssertCommand(["rm", "-f", tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
495

496
    # Normal case
Iustin Pop's avatar
Iustin Pop committed
497 498 499
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--new-cluster-certificate", "--new-confd-hmac-key",
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
Michael Hanselmann's avatar
Michael Hanselmann committed
500

501
    # Restore RAPI certificate
Iustin Pop's avatar
Iustin Pop committed
502 503
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--rapi-certificate=%s" % rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
504
  finally:
Iustin Pop's avatar
Iustin Pop committed
505
    AssertCommand(["rm", "-f", rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
506

507

508 509 510 511
def TestClusterBurnin():
  """Burnin"""
  master = qa_config.GetMasterNode()

Iustin Pop's avatar
Iustin Pop committed
512
  options = qa_config.get("options", {})
513
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
Iustin Pop's avatar
Iustin Pop committed
514 515 516 517
  parallel = options.get("burnin-in-parallel", False)
  check_inst = options.get("burnin-check-instances", False)
  do_rename = options.get("burnin-rename", "")
  do_reboot = options.get("burnin-reboot", True)
518
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
519

520 521 522
  # Get as many instances as we need
  instances = []
  try:
523
    try:
Iustin Pop's avatar
Iustin Pop committed
524
      num = qa_config.get("options", {}).get("burnin-instances", 1)
525
      for _ in range(0, num):
526 527 528
        instances.append(qa_config.AcquireInstance())
    except qa_error.OutOfInstancesError:
      print "Not enough instances, continuing anyway."
529

530 531
    if len(instances) < 1:
      raise qa_error.Error("Burnin needs at least one instance")
532

533
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
534
    try:
535
      # Run burnin
536
      cmd = [script,
Iustin Pop's avatar
Iustin Pop committed
537
             "--os=%s" % qa_config.get("os"),
538 539
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
Iustin Pop's avatar
Iustin Pop committed
540 541 542
             "--disk-size=%s" % ",".join(qa_config.get("disk")),
             "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
             "--disk-template=%s" % disk_template]
543
      if parallel:
Iustin Pop's avatar
Iustin Pop committed
544 545
        cmd.append("--parallel")
        cmd.append("--early-release")
546
      if check_inst:
Iustin Pop's avatar
Iustin Pop committed
547
        cmd.append("--http-check")
Iustin Pop's avatar
Iustin Pop committed
548
      if do_rename:
Iustin Pop's avatar
Iustin Pop committed
549
        cmd.append("--rename=%s" % do_rename)
550
      if not do_reboot:
Iustin Pop's avatar
Iustin Pop committed
551
        cmd.append("--no-reboot")
552
      else:
Iustin Pop's avatar
Iustin Pop committed
553
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
554
      cmd += [inst.name for inst in instances]
Iustin Pop's avatar
Iustin Pop committed
555
      AssertCommand(cmd)
556
    finally:
Iustin Pop's avatar
Iustin Pop committed
557 558
      AssertCommand(["rm", "-f", script])

559 560
  finally:
    for inst in instances:
561
      inst.Release()
562 563 564


def TestClusterMasterFailover():
565
  """gnt-cluster master-failover"""
566 567 568
  master = qa_config.GetMasterNode()
  failovermaster = qa_config.AcquireNode(exclude=master)

Iustin Pop's avatar
Iustin Pop committed
569 570 571
  cmd = ["gnt-cluster", "master-failover"]
  try:
    AssertCommand(cmd, node=failovermaster)
572
    # Back to original master node
Iustin Pop's avatar
Iustin Pop committed
573
    AssertCommand(cmd, node=master)
574
  finally:
575
    failovermaster.Release()
576 577


578 579
def TestClusterMasterFailoverWithDrainedQueue():
  """gnt-cluster master-failover with drained queue"""
580
  drain_check = ["test", "-f", pathutils.JOB_QUEUE_DRAIN_FILE]
581 582 583 584 585 586 587 588 589

  master = qa_config.GetMasterNode()
  failovermaster = qa_config.AcquireNode(exclude=master)

  # Ensure queue is not drained
  for node in [master, failovermaster]:
    AssertCommand(drain_check, node=node, fail=True)

  # Drain queue on failover master
590
  AssertCommand(["touch", pathutils.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
591 592 593 594 595 596 597 598 599 600 601

  cmd = ["gnt-cluster", "master-failover"]
  try:
    AssertCommand(drain_check, node=failovermaster)
    AssertCommand(cmd, node=failovermaster)
    AssertCommand(drain_check, fail=True)
    AssertCommand(drain_check, node=failovermaster, fail=True)

    # Back to original master node
    AssertCommand(cmd, node=master)
  finally:
602
    failovermaster.Release()
603 604 605 606 607

  AssertCommand(drain_check, fail=True)
  AssertCommand(drain_check, node=failovermaster, fail=True)


608 609 610 611
def TestClusterCopyfile():
  """gnt-cluster copyfile"""
  master = qa_config.GetMasterNode()

612
  uniqueid = utils.NewUUID()
613

614 615
  # Create temporary file
  f = tempfile.NamedTemporaryFile()
616
  f.write(uniqueid)
617 618 619 620
  f.flush()
  f.seek(0)

  # Upload file to master node
621
  testname = qa_utils.UploadFile(master.primary, f.name)
622 623
  try:
    # Copy file to all nodes
Iustin Pop's avatar
Iustin Pop committed
624
    AssertCommand(["gnt-cluster", "copyfile", testname])
625
    _CheckFileOnAllNodes(testname, uniqueid)
626
  finally:
627 628 629 630 631
    _RemoveFileFromAllNodes(testname)


def TestClusterCommand():
  """gnt-cluster command"""
632 633
  uniqueid = utils.NewUUID()
  rfile = "/tmp/gnt%s" % utils.NewUUID()
Iustin Pop's avatar
Iustin Pop committed
634 635
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
636 637 638
                              "%s >%s" % (rcmd, rfile)])

  try:
Iustin Pop's avatar
Iustin Pop committed
639
    AssertCommand(cmd)
640 641 642
    _CheckFileOnAllNodes(rfile, uniqueid)
  finally:
    _RemoveFileFromAllNodes(rfile)
643 644 645 646


def TestClusterDestroy():
  """gnt-cluster destroy"""
Iustin Pop's avatar
Iustin Pop committed
647
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
648 649 650 651 652


def TestClusterRepairDiskSizes():
  """gnt-cluster repair-disk-sizes"""
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670


def TestSetExclStorCluster(newvalue):
  """Set the exclusive_storage node parameter at the cluster level.

  @type newvalue: bool
  @param newvalue: New value of exclusive_storage
  @rtype: bool
  @return: The old value of exclusive_storage

  """
  oldvalue = _GetBoolClusterField("exclusive_storage")
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "exclusive_storage=%s" % newvalue])
  effvalue = _GetBoolClusterField("exclusive_storage")
  if effvalue != newvalue:
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
                         " of %s" % (effvalue, newvalue))
671
  qa_config.SetExclusiveStorage(newvalue)
672
  return oldvalue
673 674


675 676 677 678 679 680 681
def TestExclStorSharedPv(node):
  """cluster-verify reports LVs that share the same PV with exclusive_storage.

  """
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
  lvname1 = _QA_LV_PREFIX + "vol1"
  lvname2 = _QA_LV_PREFIX + "vol2"
682
  node_name = node.primary
683 684 685 686 687 688 689 690
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
                                         constants.CV_ENODEORPHANLV])
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
  AssertClusterVerify()