qa_instance.py 19.9 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40
41
42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43
44


45
def _GetGenericAddParameters(inst, force_mac=None):
46
47
48
49
50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53
54

  # Set static MAC address if configured
55
56
57
58
  if force_mac:
    nic0_mac = force_mac
  else:
    nic0_mac = qa_config.GetInstanceNicMac(inst)
59
60
61
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

62
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
63
64


65
def _DiskTest(node, disk_template):
66
67
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
68
69
70
71
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
72
           _GetGenericAddParameters(instance))
Iustin Pop's avatar
Iustin Pop committed
73
    cmd.append(instance["name"])
74

Iustin Pop's avatar
Iustin Pop committed
75
    AssertCommand(cmd)
76
77
78

    _CheckSsconfInstanceList(instance["name"])

79
80
81
82
83
84
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def _DestroyInstanceVolumes(instance):
  """Remove all the LVM volumes of an instance.

  This is used to simulate HW errors (dead nodes, broken disks...); the
  configuration of the instance is not affected.

  """
  master = qa_config.GetMasterNode()
  infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance["name"]])
  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
  re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
  node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
  # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
  #  node1.fqdn
  #  node2.fqdn,node3.fqdn
  #  node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
  # FIXME This works with no more than 2 secondaries
  re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
  re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
  nodes = []
  vols = []
  for line in info_out.splitlines():
    m = re_node.match(line)
    if m:
      nodestr = m.group(1)
      m2 = re_nodelist.match(nodestr)
      if m2:
        nodes.extend(filter(None, m2.groups()))
      else:
        nodes.append(nodestr)
    m = re_vol.match(line)
    if m:
      vols.append(m.group(1))
  assert vols
  assert nodes
  for node in nodes:
    AssertCommand(["lvremove", "-f"] + vols, node=node)


124
@InstanceCheck(None, INST_UP, RETURN_VALUE)
125
126
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
127
  return _DiskTest(node["primary"], "plain")
128
129


130
@InstanceCheck(None, INST_UP, RETURN_VALUE)
131
132
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
133
134
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
135
136


137
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
138
139
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
140
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
141
142
143
144

  qa_config.ReleaseInstance(instance)


145
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
146
147
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
148
  AssertCommand(["gnt-instance", "startup", instance["name"]])
149
150


151
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
152
153
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
154
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
155
156


157
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
158
159
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
160
  options = qa_config.get("options", {})
161
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
162
  name = instance["name"]
163
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
164
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
165

166
  AssertCommand(["gnt-instance", "shutdown", name])
167
  qa_utils.RunInstanceCheck(instance, False)
168
169
170
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
171
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
172
173
174
175
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

176

177
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
178
179
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
180
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
181
182


183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


207
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
208
209
210
211
212
213
214
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
215
  _CheckSsconfInstanceList(rename_source)
216

Iustin Pop's avatar
Iustin Pop committed
217
  # first do a rename to a different actual name, expecting it to fail
218
219
220
221
222
223
224
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
225

Iustin Pop's avatar
Iustin Pop committed
226
  # and now rename instance to rename_target...
227
228
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
229
  qa_utils.RunInstanceCheck(rename_source, False)
230
  qa_utils.RunInstanceCheck(rename_target, False)
231

Iustin Pop's avatar
Iustin Pop committed
232
233
234
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
235
  qa_utils.RunInstanceCheck(rename_target, False)
236
237


238
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
239
240
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
241
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
242

Iustin Pop's avatar
Iustin Pop committed
243
244
  # failover ...
  AssertCommand(cmd)
245
246
  qa_utils.RunInstanceCheck(instance, True)

247
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
248
  AssertCommand(cmd)
249

250

251
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
252
253
254
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
255

Iustin Pop's avatar
Iustin Pop committed
256
257
  # migrate ...
  AssertCommand(cmd)
258
259
  qa_utils.RunInstanceCheck(instance, True)

260
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
261
  AssertCommand(cmd)
262
263

  # TODO: Split into multiple tests
264
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
265
  qa_utils.RunInstanceCheck(instance, False)
266
267
268
269
270
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
271
272
  qa_utils.RunInstanceCheck(instance, True)

273
274
275
276
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
277

278
  AssertCommand(cmd, fail=True)
279
  qa_utils.RunInstanceCheck(instance, True)
280
281
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
282
283

  # TODO: Verify whether the default value is restored here (not hardcoded)
284
285
286
287
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
288

289
  AssertCommand(cmd)
290
  qa_utils.RunInstanceCheck(instance, True)
291
292


293
294
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
295
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
296
297


298
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
299
300
def TestInstanceModify(instance):
  """gnt-instance modify"""
301
302
303
304
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

305
306
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
307
  #orig_bridge = qa_config.get("bridge", "xen-br0")
308
  args = [
309
310
311
312
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
313
314
315
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
316
317
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
318
319
320
321

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
322
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
323
324
325
326
327
328
329
330
331
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
332
333
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
334
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
335
336

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
337
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
338

339
340
341
342
343
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


344
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
345
346
347
348
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

349
350
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
351
352
353
354
355
356
357

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

358

359
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
360
361
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
362
363
364
365
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
366
367


368
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
391
392
def TestInstanceList():
  """gnt-instance list"""
393
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
394
395


396
397
398
399
400
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


401
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
402
403
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
404
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
405
406


407
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
408
409
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
410
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
411
412
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
413
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
414
    cmd = ["gnt-instance", "replace-disks"]
415
416
417
418
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
419
420
421
422
423
424
425
426
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
427

428
429
430
431
432
433
434
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

435

436
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
437
def TestInstanceExport(instance, node):
438
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
439
440
441
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
442
443


444
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
445
446
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
447
448
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
449
450


451
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
452
453
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
454
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
455
456


457
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
458
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
459
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
460
461
462
463
464
465
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
466
         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
Iustin Pop's avatar
Iustin Pop committed
467
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
468
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
469
470
471
472


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
473
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
474

475
476
477
478
479
480
481
482
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

483
484
485
486
487
488

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

489
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
490
491
492
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

493
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
494
495
496
497
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
498
499

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
500
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
501
502
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
503
    (node_name, phys, inst) = line.split("|")
504
505
506
507
508
509
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
510
        raise qa_error.Error("Unknown disk name format: %s" % phys)
511
512
513
514
515
516

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
517
518
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
519

520
521
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
522
523
524
525
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
526
    AssertCommand(" && ".join(cmds), node=node_name)
527

528
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
529
530
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
531
532
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
533
    (_, _, tmpdevpath) = line.split(":")
534
    devpath.append(tmpdevpath)
535
  print devpath
536

537
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
538
539
540
541
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
542
  drbddevs = re.findall(pattern, output, re.M)
543
  print drbddevs
544
545
546

  halted_disks = []
  try:
547
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
548
549
550
551
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
552
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
553

554
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
555
                              " the problem")
556
557
558
559
560
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
561
      AssertCommand(" && ".join(cmds), node=node)
562
563
      time.sleep(3)

564
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
565
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
566
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
567

Iustin Pop's avatar
Iustin Pop committed
568
    AssertCommand(["gnt-instance", "info", instance["name"]])
569
570

  finally:
571
    print qa_utils.FormatInfo("Activating disks again")
572
573
574
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
575
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
576

Michael Hanselmann's avatar
Michael Hanselmann committed
577
578
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
579
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
580
581
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
582
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
583

584
  # TODO
Iustin Pop's avatar
Iustin Pop committed
585
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
586

587
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
588
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
589
590

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
591
592
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
593

Iustin Pop's avatar
Iustin Pop committed
594
  AssertCommand(["gnt-cluster", "verify"])
595
596
597
598


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
599
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
600
  # due to unused args
601
602
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
603
  # The following can cause crashes, thus it's disabled until fixed
604
  #return _TestInstanceDiskFailure(instance, node, node2, True)
605
606
607
608
609


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)