qa_instance.py 18.4 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40
41
42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43
44


Michael Hanselmann's avatar
Michael Hanselmann committed
45
def _GetGenericAddParameters():
46
47
48
49
50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
53
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
54
55


56
def _DiskTest(node, disk_template):
57
58
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
59
60
61
62
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
63
           _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
64
    cmd.append(instance["name"])
65

Iustin Pop's avatar
Iustin Pop committed
66
    AssertCommand(cmd)
67
68
69

    _CheckSsconfInstanceList(instance["name"])

70
71
72
73
74
75
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


76
@InstanceCheck(None, INST_UP, RETURN_VALUE)
77
78
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
79
  return _DiskTest(node["primary"], "plain")
80
81


82
@InstanceCheck(None, INST_UP, RETURN_VALUE)
83
84
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
85
86
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
87
88


89
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
90
91
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
92
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
93
94
95
96

  qa_config.ReleaseInstance(instance)


97
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
98
99
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
100
  AssertCommand(["gnt-instance", "startup", instance["name"]])
101
102


103
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
104
105
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
106
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
107
108


109
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
110
111
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
112
  options = qa_config.get("options", {})
113
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
114
  name = instance["name"]
115
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
116
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
117

118
  AssertCommand(["gnt-instance", "shutdown", name])
119
  qa_utils.RunInstanceCheck(instance, False)
120
121
122
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
123
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
124
125
126
127
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

128

129
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
130
131
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
132
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
133
134


135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


159
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
160
161
162
163
164
165
166
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
167
  _CheckSsconfInstanceList(rename_source)
168

Iustin Pop's avatar
Iustin Pop committed
169
  # first do a rename to a different actual name, expecting it to fail
170
171
172
173
174
175
176
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
177

Iustin Pop's avatar
Iustin Pop committed
178
  # and now rename instance to rename_target...
179
180
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
181
  qa_utils.RunInstanceCheck(rename_source, False)
182
  qa_utils.RunInstanceCheck(rename_target, False)
183

Iustin Pop's avatar
Iustin Pop committed
184
185
186
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
187
  qa_utils.RunInstanceCheck(rename_target, False)
188
189


190
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
191
192
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
193
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
194

Iustin Pop's avatar
Iustin Pop committed
195
196
  # failover ...
  AssertCommand(cmd)
197
198
  qa_utils.RunInstanceCheck(instance, True)

199
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
200
  AssertCommand(cmd)
201

202

203
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
204
205
206
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
207

Iustin Pop's avatar
Iustin Pop committed
208
209
  # migrate ...
  AssertCommand(cmd)
210
211
  qa_utils.RunInstanceCheck(instance, True)

212
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
213
  AssertCommand(cmd)
214
215

  # TODO: Split into multiple tests
216
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
217
  qa_utils.RunInstanceCheck(instance, False)
218
219
220
221
222
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
223
224
  qa_utils.RunInstanceCheck(instance, True)

225
226
227
228
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
229

230
  AssertCommand(cmd, fail=True)
231
  qa_utils.RunInstanceCheck(instance, True)
232
233
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
234
235

  # TODO: Verify whether the default value is restored here (not hardcoded)
236
237
238
239
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
240

241
  AssertCommand(cmd)
242
  qa_utils.RunInstanceCheck(instance, True)
243
244


245
246
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
247
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
248
249


250
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
251
252
def TestInstanceModify(instance):
  """gnt-instance modify"""
253
254
255
256
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

257
258
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
259
  #orig_bridge = qa_config.get("bridge", "xen-br0")
260
  args = [
261
262
263
264
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
265
266
267
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
268
269
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
270
271
272
273

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
274
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
275
276
277
278
279
280
281
282
283
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
284
285
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
286
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
287
288

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
289
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
290

291
292
293
294
295
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


296
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
297
298
299
300
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

301
302
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
303
304
305
306
307
308
309

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

310

311
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
312
313
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
314
315
316
317
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
318
319


320
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
343
344
def TestInstanceList():
  """gnt-instance list"""
345
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
346
347


348
349
350
351
352
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


353
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
354
355
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
356
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
357
358


359
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
360
361
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
362
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
363
364
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
365
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
366
    cmd = ["gnt-instance", "replace-disks"]
367
368
369
370
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
371
372
373
374
375
376
377
378
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
379

380
381
382
383
384
385
386
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

387

388
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
389
def TestInstanceExport(instance, node):
390
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
391
392
393
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
394
395


396
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
397
398
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
399
400
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
401
402


403
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
404
405
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
406
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
407
408


409
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
410
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
411
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
412
413
414
415
416
417
418
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--net", "0:mac=generate",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
Michael Hanselmann's avatar
Michael Hanselmann committed
419
         _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
420
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
421
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
422
423
424
425


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
426
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
427

428
429
430
431
432
433
434
435
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

436
437
438
439
440
441

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

442
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
443
444
445
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

446
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
447
448
449
450
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
451
452

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
453
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
454
455
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
456
    (node_name, phys, inst) = line.split("|")
457
458
459
460
461
462
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
463
        raise qa_error.Error("Unknown disk name format: %s" % phys)
464
465
466
467
468
469

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
470
471
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
472

473
474
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
475
476
477
478
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
479
    AssertCommand(" && ".join(cmds), node=node_name)
480

481
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
482
483
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
484
485
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
486
    (_, _, tmpdevpath) = line.split(":")
487
    devpath.append(tmpdevpath)
488
  print devpath
489

490
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
491
492
493
494
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
495
  drbddevs = re.findall(pattern, output, re.M)
496
  print drbddevs
497
498
499

  halted_disks = []
  try:
500
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
501
502
503
504
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
505
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
506

507
508
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
509
510
511
512
513
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
514
      AssertCommand(" && ".join(cmds), node=node)
515
516
      time.sleep(3)

517
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
518
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
519
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
520

Iustin Pop's avatar
Iustin Pop committed
521
    AssertCommand(["gnt-instance", "info", instance["name"]])
522
523

  finally:
524
    print qa_utils.FormatInfo("Activating disks again")
525
526
527
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
528
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
529

Michael Hanselmann's avatar
Michael Hanselmann committed
530
531
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
532
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
533
534
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
535
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
536

537
  # TODO
Iustin Pop's avatar
Iustin Pop committed
538
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
539

540
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
541
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
542
543

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
544
545
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
546

Iustin Pop's avatar
Iustin Pop committed
547
  AssertCommand(["gnt-cluster", "verify"])
548
549
550
551


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
552
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
553
  # due to unused args
554
555
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
556
  # The following can cause crashes, thus it's disabled until fixed
557
  #return _TestInstanceDiskFailure(instance, node, node2, True)
558
559
560
561
562


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)