qa_instance.py 18.5 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40
41
42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43
44


45
def _GetGenericAddParameters(inst):
46
47
48
49
50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53
54
55
56
57
58

  # Set static MAC address if configured
  nic0_mac = qa_config.GetInstanceNicMac(inst)
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

59
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
60
61


62
def _DiskTest(node, disk_template):
63
64
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
65
66
67
68
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
69
           _GetGenericAddParameters(instance))
Iustin Pop's avatar
Iustin Pop committed
70
    cmd.append(instance["name"])
71

Iustin Pop's avatar
Iustin Pop committed
72
    AssertCommand(cmd)
73
74
75

    _CheckSsconfInstanceList(instance["name"])

76
77
78
79
80
81
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


82
@InstanceCheck(None, INST_UP, RETURN_VALUE)
83
84
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
85
  return _DiskTest(node["primary"], "plain")
86
87


88
@InstanceCheck(None, INST_UP, RETURN_VALUE)
89
90
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
91
92
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
93
94


95
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
96
97
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
98
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
99
100
101
102

  qa_config.ReleaseInstance(instance)


103
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
104
105
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
106
  AssertCommand(["gnt-instance", "startup", instance["name"]])
107
108


109
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
110
111
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
112
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
113
114


115
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
116
117
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
118
  options = qa_config.get("options", {})
119
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
120
  name = instance["name"]
121
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
122
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
123

124
  AssertCommand(["gnt-instance", "shutdown", name])
125
  qa_utils.RunInstanceCheck(instance, False)
126
127
128
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
129
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
130
131
132
133
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

134

135
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
136
137
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
138
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
139
140


141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


165
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
166
167
168
169
170
171
172
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
173
  _CheckSsconfInstanceList(rename_source)
174

Iustin Pop's avatar
Iustin Pop committed
175
  # first do a rename to a different actual name, expecting it to fail
176
177
178
179
180
181
182
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
183

Iustin Pop's avatar
Iustin Pop committed
184
  # and now rename instance to rename_target...
185
186
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
187
  qa_utils.RunInstanceCheck(rename_source, False)
188
  qa_utils.RunInstanceCheck(rename_target, False)
189

Iustin Pop's avatar
Iustin Pop committed
190
191
192
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
193
  qa_utils.RunInstanceCheck(rename_target, False)
194
195


196
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
197
198
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
199
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
200

Iustin Pop's avatar
Iustin Pop committed
201
202
  # failover ...
  AssertCommand(cmd)
203
204
  qa_utils.RunInstanceCheck(instance, True)

205
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
206
  AssertCommand(cmd)
207

208

209
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
210
211
212
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
213

Iustin Pop's avatar
Iustin Pop committed
214
215
  # migrate ...
  AssertCommand(cmd)
216
217
  qa_utils.RunInstanceCheck(instance, True)

218
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
219
  AssertCommand(cmd)
220
221

  # TODO: Split into multiple tests
222
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
223
  qa_utils.RunInstanceCheck(instance, False)
224
225
226
227
228
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
229
230
  qa_utils.RunInstanceCheck(instance, True)

231
232
233
234
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
235

236
  AssertCommand(cmd, fail=True)
237
  qa_utils.RunInstanceCheck(instance, True)
238
239
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
240
241

  # TODO: Verify whether the default value is restored here (not hardcoded)
242
243
244
245
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
246

247
  AssertCommand(cmd)
248
  qa_utils.RunInstanceCheck(instance, True)
249
250


251
252
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
253
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
254
255


256
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
257
258
def TestInstanceModify(instance):
  """gnt-instance modify"""
259
260
261
262
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

263
264
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
265
  #orig_bridge = qa_config.get("bridge", "xen-br0")
266
  args = [
267
268
269
270
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
271
272
273
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
274
275
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
276
277
278
279

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
280
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
281
282
283
284
285
286
287
288
289
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
290
291
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
292
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
293
294

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
295
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
296

297
298
299
300
301
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


302
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
303
304
305
306
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

307
308
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
309
310
311
312
313
314
315

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

316

317
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
318
319
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
320
321
322
323
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
324
325


326
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
349
350
def TestInstanceList():
  """gnt-instance list"""
351
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
352
353


354
355
356
357
358
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


359
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
360
361
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
362
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
363
364


365
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
366
367
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
368
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
369
370
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
371
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
372
    cmd = ["gnt-instance", "replace-disks"]
373
374
375
376
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
377
378
379
380
381
382
383
384
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
385

386
387
388
389
390
391
392
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

393

394
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
395
def TestInstanceExport(instance, node):
396
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
397
398
399
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
400
401


402
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
403
404
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
405
406
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
407
408


409
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
410
411
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
412
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
413
414


415
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
416
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
417
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
418
419
420
421
422
423
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
424
         _GetGenericAddParameters(newinst))
Iustin Pop's avatar
Iustin Pop committed
425
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
426
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
427
428
429
430


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
431
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
432

433
434
435
436
437
438
439
440
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

441
442
443
444
445
446

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

447
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
448
449
450
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

451
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
452
453
454
455
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
456
457

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
458
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
459
460
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
461
    (node_name, phys, inst) = line.split("|")
462
463
464
465
466
467
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
468
        raise qa_error.Error("Unknown disk name format: %s" % phys)
469
470
471
472
473
474

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
475
476
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
477

478
479
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
480
481
482
483
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
484
    AssertCommand(" && ".join(cmds), node=node_name)
485

486
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
487
488
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
489
490
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
491
    (_, _, tmpdevpath) = line.split(":")
492
    devpath.append(tmpdevpath)
493
  print devpath
494

495
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
496
497
498
499
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
500
  drbddevs = re.findall(pattern, output, re.M)
501
  print drbddevs
502
503
504

  halted_disks = []
  try:
505
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
506
507
508
509
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
510
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
511

512
513
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
514
515
516
517
518
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
519
      AssertCommand(" && ".join(cmds), node=node)
520
521
      time.sleep(3)

522
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
523
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
524
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
525

Iustin Pop's avatar
Iustin Pop committed
526
    AssertCommand(["gnt-instance", "info", instance["name"]])
527
528

  finally:
529
    print qa_utils.FormatInfo("Activating disks again")
530
531
532
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
533
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
534

Michael Hanselmann's avatar
Michael Hanselmann committed
535
536
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
537
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
538
539
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
540
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
541

542
  # TODO
Iustin Pop's avatar
Iustin Pop committed
543
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
544

545
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
546
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
547
548

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
549
550
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
551

Iustin Pop's avatar
Iustin Pop committed
552
  AssertCommand(["gnt-cluster", "verify"])
553
554
555
556


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
557
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
558
  # due to unused args
559
560
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
561
  # The following can cause crashes, thus it's disabled until fixed
562
  #return _TestInstanceDiskFailure(instance, node, node2, True)
563
564
565
566
567


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)