qa_instance.py 18.6 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40
41
42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43
44


45
def _GetGenericAddParameters(inst, force_mac=None):
46
47
48
49
50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53
54

  # Set static MAC address if configured
55
56
57
58
  if force_mac:
    nic0_mac = force_mac
  else:
    nic0_mac = qa_config.GetInstanceNicMac(inst)
59
60
61
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

62
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
63
64


65
def _DiskTest(node, disk_template):
66
67
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
68
69
70
71
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
72
           _GetGenericAddParameters(instance))
Iustin Pop's avatar
Iustin Pop committed
73
    cmd.append(instance["name"])
74

Iustin Pop's avatar
Iustin Pop committed
75
    AssertCommand(cmd)
76
77
78

    _CheckSsconfInstanceList(instance["name"])

79
80
81
82
83
84
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


85
@InstanceCheck(None, INST_UP, RETURN_VALUE)
86
87
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
88
  return _DiskTest(node["primary"], "plain")
89
90


91
@InstanceCheck(None, INST_UP, RETURN_VALUE)
92
93
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
94
95
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
96
97


98
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
99
100
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
101
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
102
103
104
105

  qa_config.ReleaseInstance(instance)


106
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
107
108
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
109
  AssertCommand(["gnt-instance", "startup", instance["name"]])
110
111


112
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
113
114
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
115
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
116
117


118
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
119
120
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
121
  options = qa_config.get("options", {})
122
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
123
  name = instance["name"]
124
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
125
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
126

127
  AssertCommand(["gnt-instance", "shutdown", name])
128
  qa_utils.RunInstanceCheck(instance, False)
129
130
131
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
132
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
133
134
135
136
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

137

138
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
139
140
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
141
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
142
143


144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


168
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
169
170
171
172
173
174
175
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
176
  _CheckSsconfInstanceList(rename_source)
177

Iustin Pop's avatar
Iustin Pop committed
178
  # first do a rename to a different actual name, expecting it to fail
179
180
181
182
183
184
185
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
186

Iustin Pop's avatar
Iustin Pop committed
187
  # and now rename instance to rename_target...
188
189
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
190
  qa_utils.RunInstanceCheck(rename_source, False)
191
  qa_utils.RunInstanceCheck(rename_target, False)
192

Iustin Pop's avatar
Iustin Pop committed
193
194
195
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
196
  qa_utils.RunInstanceCheck(rename_target, False)
197
198


199
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
200
201
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
202
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
203

Iustin Pop's avatar
Iustin Pop committed
204
205
  # failover ...
  AssertCommand(cmd)
206
207
  qa_utils.RunInstanceCheck(instance, True)

208
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
209
  AssertCommand(cmd)
210

211

212
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
213
214
215
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
216

Iustin Pop's avatar
Iustin Pop committed
217
218
  # migrate ...
  AssertCommand(cmd)
219
220
  qa_utils.RunInstanceCheck(instance, True)

221
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
222
  AssertCommand(cmd)
223
224

  # TODO: Split into multiple tests
225
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
226
  qa_utils.RunInstanceCheck(instance, False)
227
228
229
230
231
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
232
233
  qa_utils.RunInstanceCheck(instance, True)

234
235
236
237
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
238

239
  AssertCommand(cmd, fail=True)
240
  qa_utils.RunInstanceCheck(instance, True)
241
242
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
243
244

  # TODO: Verify whether the default value is restored here (not hardcoded)
245
246
247
248
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
249

250
  AssertCommand(cmd)
251
  qa_utils.RunInstanceCheck(instance, True)
252
253


254
255
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
256
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
257
258


259
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
260
261
def TestInstanceModify(instance):
  """gnt-instance modify"""
262
263
264
265
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

266
267
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
268
  #orig_bridge = qa_config.get("bridge", "xen-br0")
269
  args = [
270
271
272
273
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
274
275
276
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
277
278
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
279
280
281
282

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
283
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
284
285
286
287
288
289
290
291
292
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
293
294
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
295
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
296
297

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
298
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
299

300
301
302
303
304
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


305
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
306
307
308
309
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

310
311
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
312
313
314
315
316
317
318

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

319

320
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
321
322
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
323
324
325
326
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
327
328


329
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
352
353
def TestInstanceList():
  """gnt-instance list"""
354
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
355
356


357
358
359
360
361
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


362
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
363
364
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
365
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
366
367


368
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
369
370
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
371
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
372
373
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
374
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
375
    cmd = ["gnt-instance", "replace-disks"]
376
377
378
379
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
380
381
382
383
384
385
386
387
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
388

389
390
391
392
393
394
395
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

396

397
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
398
def TestInstanceExport(instance, node):
399
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
400
401
402
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
403
404


405
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
406
407
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
408
409
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
410
411


412
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
413
414
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
415
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
416
417


418
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
419
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
420
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
421
422
423
424
425
426
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
427
         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
Iustin Pop's avatar
Iustin Pop committed
428
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
429
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
430
431
432
433


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
434
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
435

436
437
438
439
440
441
442
443
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

444
445
446
447
448
449

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

450
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
451
452
453
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

454
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
455
456
457
458
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
459
460

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
461
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
462
463
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
464
    (node_name, phys, inst) = line.split("|")
465
466
467
468
469
470
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
471
        raise qa_error.Error("Unknown disk name format: %s" % phys)
472
473
474
475
476
477

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
478
479
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
480

481
482
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
483
484
485
486
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
487
    AssertCommand(" && ".join(cmds), node=node_name)
488

489
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
490
491
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
492
493
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
494
    (_, _, tmpdevpath) = line.split(":")
495
    devpath.append(tmpdevpath)
496
  print devpath
497

498
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
499
500
501
502
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
503
  drbddevs = re.findall(pattern, output, re.M)
504
  print drbddevs
505
506
507

  halted_disks = []
  try:
508
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
509
510
511
512
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
513
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
514

515
516
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
517
518
519
520
521
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
522
      AssertCommand(" && ".join(cmds), node=node)
523
524
      time.sleep(3)

525
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
526
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
527
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
528

Iustin Pop's avatar
Iustin Pop committed
529
    AssertCommand(["gnt-instance", "info", instance["name"]])
530
531

  finally:
532
    print qa_utils.FormatInfo("Activating disks again")
533
534
535
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
536
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
537

Michael Hanselmann's avatar
Michael Hanselmann committed
538
539
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
540
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
541
542
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
543
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
544

545
  # TODO
Iustin Pop's avatar
Iustin Pop committed
546
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
547

548
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
549
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
550
551

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
552
553
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
554

Iustin Pop's avatar
Iustin Pop committed
555
  AssertCommand(["gnt-cluster", "verify"])
556
557
558
559


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
560
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
561
  # due to unused args
562
563
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
564
  # The following can cause crashes, thus it's disabled until fixed
565
  #return _TestInstanceDiskFailure(instance, node, node2, True)
566
567
568
569
570


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)