qa_instance.py 16.9 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
49
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
50
  for idx, size in enumerate(qa_config.get("disk")):
51
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
53
54


55
def _DiskTest(node, disk_template):
56
57
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
58
59
60
61
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
62
           _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
63
    cmd.append(instance["name"])
64

Iustin Pop's avatar
Iustin Pop committed
65
    AssertCommand(cmd)
66
67
68

    _CheckSsconfInstanceList(instance["name"])

69
70
71
72
73
74
75
76
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
77
  return _DiskTest(node["primary"], "plain")
78
79


80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
82
83
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
84
85


86
87
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
88
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
89
90
91
92
93
94

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
95
  AssertCommand(["gnt-instance", "startup", instance["name"]])
96
97
98
99


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
100
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
101
102


103
104
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
105
  options = qa_config.get("options", {})
106
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
107
  name = instance["name"]
108
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
109
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
110

111
112
113
114
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
115
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
116
117
118
119
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

120

Michael Hanselmann's avatar
Michael Hanselmann committed
121
122
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
123
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
124
125


126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


Iustin Pop's avatar
Iustin Pop committed
150
151
152
153
154
155
156
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
157
  _CheckSsconfInstanceList(rename_source)
Iustin Pop's avatar
Iustin Pop committed
158
  # first do a rename to a different actual name, expecting it to fail
159
160
161
162
163
164
165
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
Iustin Pop's avatar
Iustin Pop committed
166
  # and now rename instance to rename_target...
167
168
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
Iustin Pop's avatar
Iustin Pop committed
169
170
171
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
172
173


174
175
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
176
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
177
178
  # failover ...
  AssertCommand(cmd)
179
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
180
  AssertCommand(cmd)
181

182

183
184
185
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
186
187
  # migrate ...
  AssertCommand(cmd)
188
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
189
  AssertCommand(cmd)
190
191
192
193
194
195
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
196
197
198
199
200
201
202
203
204
205
206
207
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
  AssertCommand(cmd)
208
209


210
211
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
212
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
213
214


215
216
def TestInstanceModify(instance):
  """gnt-instance modify"""
217
218
219
220
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

221
222
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
223
  #orig_bridge = qa_config.get("bridge", "xen-br0")
224
  args = [
225
226
227
228
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
229
230
231
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
232
233
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
234
235
236
237

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
238
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
239
240
241
242
243
244
245
246
247
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
248
249
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
250
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
251
252

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
253
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
254

255
256
257
258
259
260
261
262
263
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

264
265
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
266
267
268
269
270
271
272

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

273

274
275
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
276
277
278
279
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
280
281


Iustin Pop's avatar
Iustin Pop committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
304
305
def TestInstanceList():
  """gnt-instance list"""
306
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
307
308


309
310
311
312
313
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


314
315
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
316
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
317
318


319
320
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
321
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
322
323
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
324
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
325
    cmd = ["gnt-instance", "replace-disks"]
326
327
328
329
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
330
331
332
333
334
335
336
337
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
338

339
340
341
342
343
344
345
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

346

Michael Hanselmann's avatar
Michael Hanselmann committed
347
def TestInstanceExport(instance, node):
348
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
349
350
351
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
352
353


354
355
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
356
357
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
358
359


360
361
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
362
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
363
364


Michael Hanselmann's avatar
Michael Hanselmann committed
365
366
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
367
368
369
370
371
372
373
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--net", "0:mac=generate",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
Michael Hanselmann's avatar
Michael Hanselmann committed
374
         _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
375
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
376
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
377
378
379
380


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
381
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
382

383
384
385
386
387
388
389
390
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

391
392
393
394
395
396

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

397
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
398
399
400
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

401
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
402
403
404
405
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
406
407

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
408
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
409
410
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
411
    (node_name, phys, inst) = line.split("|")
412
413
414
415
416
417
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
418
        raise qa_error.Error("Unknown disk name format: %s" % phys)
419
420
421
422
423
424

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
425
426
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
427

428
429
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
430
431
432
433
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
434
    AssertCommand(" && ".join(cmds), node=node_name)
435

436
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
437
438
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
439
440
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
441
    (_, _, tmpdevpath) = line.split(":")
442
    devpath.append(tmpdevpath)
443
  print devpath
444

445
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
446
447
448
449
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
450
  drbddevs = re.findall(pattern, output, re.M)
451
  print drbddevs
452
453
454

  halted_disks = []
  try:
455
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
456
457
458
459
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
460
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
461

462
463
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
464
465
466
467
468
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
469
      AssertCommand(" && ".join(cmds), node=node)
470
471
      time.sleep(3)

472
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
473
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
474
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
475

Iustin Pop's avatar
Iustin Pop committed
476
    AssertCommand(["gnt-instance", "info", instance["name"]])
477
478

  finally:
479
    print qa_utils.FormatInfo("Activating disks again")
480
481
482
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
483
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
484

Michael Hanselmann's avatar
Michael Hanselmann committed
485
486
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
487
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
488
489
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
490
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
491

492
  # TODO
Iustin Pop's avatar
Iustin Pop committed
493
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
494

495
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
496
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
497
498

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
499
500
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
501

Iustin Pop's avatar
Iustin Pop committed
502
  AssertCommand(["gnt-cluster", "verify"])
503
504
505
506


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
507
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
508
  # due to unused args
509
510
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
511
  # The following can cause crashes, thus it's disabled until fixed
512
  #return _TestInstanceDiskFailure(instance, node, node2, True)
513
514
515
516
517


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)