qa_instance.py 15.7 KB
Newer Older
1 2 3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26 27 28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32 33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35 36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39 40 41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42 43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45 46 47 48 49
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
50
  for idx, size in enumerate(qa_config.get("disk")):
51 52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
53 54


55
def _DiskTest(node, disk_template):
56 57
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
58 59 60 61
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
62
           _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
63
    cmd.append(instance["name"])
64

Iustin Pop's avatar
Iustin Pop committed
65
    AssertCommand(cmd)
66 67 68

    _CheckSsconfInstanceList(instance["name"])

69 70 71 72 73 74 75 76
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
77
  return _DiskTest(node["primary"], "plain")
78 79


80 81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
82 83
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
84 85


86 87
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
88
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
89 90 91 92 93 94

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
95
  AssertCommand(["gnt-instance", "startup", instance["name"]])
96 97 98 99


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
100
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
101 102


103 104
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
105
  options = qa_config.get("options", {})
106
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
107
  name = instance["name"]
108
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
109
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
110

111 112 113 114
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
115
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
116 117 118 119
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

120

Michael Hanselmann's avatar
Michael Hanselmann committed
121 122
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
123
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
124 125


126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


Iustin Pop's avatar
Iustin Pop committed
150 151 152 153 154 155 156
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
157
  _CheckSsconfInstanceList(rename_source)
Iustin Pop's avatar
Iustin Pop committed
158
  # first do a rename to a different actual name, expecting it to fail
159 160 161 162 163 164 165
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
Iustin Pop's avatar
Iustin Pop committed
166
  # and now rename instance to rename_target...
167 168
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
Iustin Pop's avatar
Iustin Pop committed
169 170 171
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
172 173


174 175
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
176
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
177 178
  # failover ...
  AssertCommand(cmd)
179
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
180
  AssertCommand(cmd)
181

182

183 184 185
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
186 187
  # migrate ...
  AssertCommand(cmd)
188
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
189
  AssertCommand(cmd)
190 191 192 193 194 195
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
196 197 198 199 200 201 202 203 204 205 206 207
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
  AssertCommand(cmd)
208 209


210 211
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
212
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
213 214


215 216
def TestInstanceModify(instance):
  """gnt-instance modify"""
217 218 219 220
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

221 222
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
223
  #orig_bridge = qa_config.get("bridge", "xen-br0")
224
  args = [
225 226 227 228
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
229 230 231
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
232 233
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
234 235 236 237

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
238
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
239 240 241 242 243 244 245 246 247
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
248 249
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
250
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
251 252

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
253
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
254

255 256 257 258 259 260 261 262 263
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

264 265
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
266 267 268 269 270 271 272

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

273

274 275
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
276 277 278 279
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
280 281


Michael Hanselmann's avatar
Michael Hanselmann committed
282 283
def TestInstanceList():
  """gnt-instance list"""
284
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
285 286


287 288 289 290 291
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


292 293
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
294
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
295 296


297 298
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
299
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
300 301
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
302
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
303
    cmd = ["gnt-instance", "replace-disks"]
304 305 306 307
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
308 309 310 311 312 313 314 315
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
316

317 318 319 320 321 322 323
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

324

Michael Hanselmann's avatar
Michael Hanselmann committed
325
def TestInstanceExport(instance, node):
326
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
327 328 329
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
330 331


332 333
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
334 335
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
336 337


338 339
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
340
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
341 342


Michael Hanselmann's avatar
Michael Hanselmann committed
343 344
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
345 346 347 348 349 350 351
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--net", "0:mac=generate",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
Michael Hanselmann's avatar
Michael Hanselmann committed
352
         _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
353
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
354
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
355 356 357 358


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
359
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
360 361 362 363 364 365 366


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

367
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
368 369 370
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

371
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
372 373 374 375
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
376 377

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
378
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
379 380
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
381
    (node_name, phys, inst) = line.split("|")
382 383 384 385 386 387
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
388
        raise qa_error.Error("Unknown disk name format: %s" % phys)
389 390 391 392 393 394

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
395 396
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
397

398 399
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
400 401 402 403
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
404
    AssertCommand(" && ".join(cmds), node=node_name)
405

406
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
407 408
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
409 410
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
411
    (_, _, tmpdevpath) = line.split(":")
412
    devpath.append(tmpdevpath)
413
  print devpath
414

415
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
416 417 418 419
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
420
  drbddevs = re.findall(pattern, output, re.M)
421
  print drbddevs
422 423 424

  halted_disks = []
  try:
425
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
426 427 428 429
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
430
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
431

432 433
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
434 435 436 437 438
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
439
      AssertCommand(" && ".join(cmds), node=node)
440 441
      time.sleep(3)

442
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
443
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
444
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
445

Iustin Pop's avatar
Iustin Pop committed
446
    AssertCommand(["gnt-instance", "info", instance["name"]])
447 448

  finally:
449
    print qa_utils.FormatInfo("Activating disks again")
450 451 452
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
453
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
454

Michael Hanselmann's avatar
Michael Hanselmann committed
455 456
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
457
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
458 459
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
460
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
461

462
  # TODO
Iustin Pop's avatar
Iustin Pop committed
463
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
464

465
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
466
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
467 468

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
469 470
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
471

Iustin Pop's avatar
Iustin Pop committed
472
  AssertCommand(["gnt-cluster", "verify"])
473 474 475 476


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
477
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
478
  # due to unused args
479 480
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
481
  # The following can cause crashes, thus it's disabled until fixed
482
  #return _TestInstanceDiskFailure(instance, node, node2, True)
483 484 485 486 487


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)