qa_instance.py 22.4 KB
Newer Older
1 2 3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26 27 28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32 33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35 36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40 41 42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43 44


45
def _GetGenericAddParameters(inst, force_mac=None):
46 47 48 49 50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53 54

  # Set static MAC address if configured
55 56 57 58
  if force_mac:
    nic0_mac = force_mac
  else:
    nic0_mac = qa_config.GetInstanceNicMac(inst)
59 60 61
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

62
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
63 64


65
def _DiskTest(node, disk_template):
66 67
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
68 69 70 71
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
72
           _GetGenericAddParameters(instance))
Iustin Pop's avatar
Iustin Pop committed
73
    cmd.append(instance["name"])
74

Iustin Pop's avatar
Iustin Pop committed
75
    AssertCommand(cmd)
76 77 78

    _CheckSsconfInstanceList(instance["name"])

79 80 81 82 83 84
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
def _DestroyInstanceVolumes(instance):
  """Remove all the LVM volumes of an instance.

  This is used to simulate HW errors (dead nodes, broken disks...); the
  configuration of the instance is not affected.

  """
  master = qa_config.GetMasterNode()
  infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance["name"]])
  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
  re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
  node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
  # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
  #  node1.fqdn
  #  node2.fqdn,node3.fqdn
  #  node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
  # FIXME This works with no more than 2 secondaries
  re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
  re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
  nodes = []
  vols = []
  for line in info_out.splitlines():
    m = re_node.match(line)
    if m:
      nodestr = m.group(1)
      m2 = re_nodelist.match(nodestr)
      if m2:
        nodes.extend(filter(None, m2.groups()))
      else:
        nodes.append(nodestr)
    m = re_vol.match(line)
    if m:
      vols.append(m.group(1))
  assert vols
  assert nodes
  for node in nodes:
    AssertCommand(["lvremove", "-f"] + vols, node=node)


124
@InstanceCheck(None, INST_UP, RETURN_VALUE)
125 126
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
127
  return _DiskTest(node["primary"], "plain")
128 129


130
@InstanceCheck(None, INST_UP, RETURN_VALUE)
131 132
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
133 134
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
135 136


137
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
138 139
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
140
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
141 142 143 144

  qa_config.ReleaseInstance(instance)


145
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
146 147
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
148
  AssertCommand(["gnt-instance", "startup", instance["name"]])
149 150


151
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
152 153
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
154
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
155 156


157
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
158 159
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
160
  options = qa_config.get("options", {})
161
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
162
  name = instance["name"]
163
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
164
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
165

166
  AssertCommand(["gnt-instance", "shutdown", name])
167
  qa_utils.RunInstanceCheck(instance, False)
168 169 170
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
171
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
172 173 174 175
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

176

177
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
178 179
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
180
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
181 182


183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


207
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
208 209 210 211 212 213 214
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
215
  _CheckSsconfInstanceList(rename_source)
216

Iustin Pop's avatar
Iustin Pop committed
217
  # first do a rename to a different actual name, expecting it to fail
218 219 220 221 222 223 224
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
225

Iustin Pop's avatar
Iustin Pop committed
226
  # and now rename instance to rename_target...
227 228
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
229
  qa_utils.RunInstanceCheck(rename_source, False)
230
  qa_utils.RunInstanceCheck(rename_target, False)
231

Iustin Pop's avatar
Iustin Pop committed
232 233 234
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
235
  qa_utils.RunInstanceCheck(rename_target, False)
236 237


238
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
239 240
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
241
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
242

Iustin Pop's avatar
Iustin Pop committed
243 244
  # failover ...
  AssertCommand(cmd)
245 246
  qa_utils.RunInstanceCheck(instance, True)

247
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
248
  AssertCommand(cmd)
249

250

251
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
252 253 254
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
255

Iustin Pop's avatar
Iustin Pop committed
256 257
  # migrate ...
  AssertCommand(cmd)
258 259
  qa_utils.RunInstanceCheck(instance, True)

260
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
261
  AssertCommand(cmd)
262 263

  # TODO: Split into multiple tests
264
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
265
  qa_utils.RunInstanceCheck(instance, False)
266 267 268 269 270
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
271 272
  qa_utils.RunInstanceCheck(instance, True)

273 274 275 276
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
277

278
  AssertCommand(cmd, fail=True)
279
  qa_utils.RunInstanceCheck(instance, True)
280 281
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
282 283

  # TODO: Verify whether the default value is restored here (not hardcoded)
284 285 286 287
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
288

289
  AssertCommand(cmd)
290
  qa_utils.RunInstanceCheck(instance, True)
291 292


293 294
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
295
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
296 297


298
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
299 300
def TestInstanceModify(instance):
  """gnt-instance modify"""
301 302
  default_hv = qa_config.GetDefaultHypervisor()

303 304 305 306
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

307 308
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
309
  #orig_bridge = qa_config.get("bridge", "xen-br0")
310

311
  args = [
312 313 314 315
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
316 317 318
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
319 320
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
321 322 323 324 325 326 327

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],
328
    ]
329 330 331 332 333 334 335 336 337 338 339 340 341

  if default_hv == constants.HT_XEN_PVM:
    args.extend([
      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
      ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
      ])
  elif default_hv == constants.HT_XEN_HVM:
    args.extend([
      ["-H", "%s=acn" % constants.HV_BOOT_ORDER],
      ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
      ])

342
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
343
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
344 345

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
346
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
347

348 349 350 351 352
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


353
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
354 355 356 357
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

358 359
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
360 361 362 363 364 365 366

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

367

368
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
369 370
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
371 372 373 374
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
375 376


377
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
400 401
def TestInstanceList():
  """gnt-instance list"""
402
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
403 404


405 406 407 408 409
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


410
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
411 412
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
413
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
414 415


416
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
417 418
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
419
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
420 421
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
422
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
423
    cmd = ["gnt-instance", "replace-disks"]
424 425 426 427
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
428 429 430 431 432 433 434 435
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
436

437 438 439 440
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
441 442
  AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
                 instance["name"]])
443 444 445
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

446

447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
                         destroy=True):
  """Execute gnt-instance recreate-disks and check the result

  @param cmdargs: Arguments (instance name excluded)
  @param instance: Instance to operate on
  @param fail: True if the command is expected to fail
  @param check: If True and fail is False, check that the disks work
  @prama destroy: If True, destroy the old disks first

  """
  if destroy:
    _DestroyInstanceVolumes(instance)
  AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
                 [instance["name"]]), fail)
  if not fail and check:
    # Quick check that the disks are there
    AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
465 466
    AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
                   instance["name"]])
467 468
    AssertCommand(["gnt-instance", "deactivate-disks", instance["name"]])

Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
469

470 471 472 473 474 475 476 477 478 479 480 481 482 483
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
def TestRecreateDisks(instance, pnode, snode, othernodes):
  """gnt-instance recreate-disks

  @param instance: Instance to work on
  @param pnode: Primary node
  @param snode: Secondary node, or None for sigle-homed instances
  @param othernodes: list/tuple of nodes where to temporarily recreate disks

  """
  other_seq = ":".join([n["primary"] for n in othernodes])
  orig_seq = pnode["primary"]
  if snode:
    orig_seq = orig_seq + ":" + snode["primary"]
484
  # These fail because the instance is running
485
  _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
486
  _AssertRecreateDisks(["-I", "hail"], instance, fail=True, destroy=False)
487 488 489 490 491 492
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  # Disks exist: this should fail
  _AssertRecreateDisks([], instance, fail=True, destroy=False)
  # Recreate disks in place
  _AssertRecreateDisks([], instance)
  # Move disks away
493
  _AssertRecreateDisks(["-I", "hail"], instance)
494 495 496 497 498 499 500
  # Move disks back
  _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
  # This and InstanceCheck decoration check that the disks are working
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])


501
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
502
def TestInstanceExport(instance, node):
503
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
504 505 506
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
507 508


509
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
510 511
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
512 513
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
514 515


516
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
517 518
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
519
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
520 521


522
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
523
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
524
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
525 526 527 528 529 530
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
531
         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
Iustin Pop's avatar
Iustin Pop committed
532
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
533
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
534 535 536 537


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
538
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
539

540 541 542 543 544 545 546 547
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

548 549 550 551 552 553

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

554
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
555 556 557
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

558
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
559 560 561 562
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
563 564

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
565
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
566 567
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
568
    (node_name, phys, inst) = line.split("|")
569 570 571 572 573 574
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
575
        raise qa_error.Error("Unknown disk name format: %s" % phys)
576 577 578 579 580 581

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
582 583
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
584

585 586
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
587 588 589 590
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
591
    AssertCommand(" && ".join(cmds), node=node_name)
592

593
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
594 595
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
596 597
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
598
    (_, _, tmpdevpath) = line.split(":")
599
    devpath.append(tmpdevpath)
600
  print devpath
601

602
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
603 604 605 606
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
607
  drbddevs = re.findall(pattern, output, re.M)
608
  print drbddevs
609 610 611

  halted_disks = []
  try:
612
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
613 614 615 616
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
617
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
618

619
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
620
                              " the problem")
621 622 623 624 625
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
626
      AssertCommand(" && ".join(cmds), node=node)
627 628
      time.sleep(3)

629
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
630
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
631
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
632

Iustin Pop's avatar
Iustin Pop committed
633
    AssertCommand(["gnt-instance", "info", instance["name"]])
634 635

  finally:
636
    print qa_utils.FormatInfo("Activating disks again")
637 638 639
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
640
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
641

Michael Hanselmann's avatar
Michael Hanselmann committed
642 643
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
644
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
645 646
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
647
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
648

649
  # TODO
Iustin Pop's avatar
Iustin Pop committed
650
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
651

652
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
653
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
654 655

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
656 657
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
658

Iustin Pop's avatar
Iustin Pop committed
659
  AssertCommand(["gnt-cluster", "verify"])
660 661 662 663


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
664
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
665
  # due to unused args
666 667
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
668
  # The following can cause crashes, thus it's disabled until fixed
669
  #return _TestInstanceDiskFailure(instance, node, node2, True)
670 671 672 673 674


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)