qa_instance.py 21.9 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40
41
42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43
44


45
def _GetGenericAddParameters(inst, force_mac=None):
46
47
48
49
50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53
54

  # Set static MAC address if configured
55
56
57
58
  if force_mac:
    nic0_mac = force_mac
  else:
    nic0_mac = qa_config.GetInstanceNicMac(inst)
59
60
61
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

62
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
63
64


65
def _DiskTest(node, disk_template):
66
67
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
68
69
70
71
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
72
           _GetGenericAddParameters(instance))
Iustin Pop's avatar
Iustin Pop committed
73
    cmd.append(instance["name"])
74

Iustin Pop's avatar
Iustin Pop committed
75
    AssertCommand(cmd)
76
77
78

    _CheckSsconfInstanceList(instance["name"])

79
80
81
82
83
84
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def _DestroyInstanceVolumes(instance):
  """Remove all the LVM volumes of an instance.

  This is used to simulate HW errors (dead nodes, broken disks...); the
  configuration of the instance is not affected.

  """
  master = qa_config.GetMasterNode()
  infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance["name"]])
  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
  re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
  node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
  # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
  #  node1.fqdn
  #  node2.fqdn,node3.fqdn
  #  node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
  # FIXME This works with no more than 2 secondaries
  re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
  re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
  nodes = []
  vols = []
  for line in info_out.splitlines():
    m = re_node.match(line)
    if m:
      nodestr = m.group(1)
      m2 = re_nodelist.match(nodestr)
      if m2:
        nodes.extend(filter(None, m2.groups()))
      else:
        nodes.append(nodestr)
    m = re_vol.match(line)
    if m:
      vols.append(m.group(1))
  assert vols
  assert nodes
  for node in nodes:
    AssertCommand(["lvremove", "-f"] + vols, node=node)


124
@InstanceCheck(None, INST_UP, RETURN_VALUE)
125
126
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
127
  return _DiskTest(node["primary"], "plain")
128
129


130
@InstanceCheck(None, INST_UP, RETURN_VALUE)
131
132
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
133
134
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
135
136


137
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
138
139
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
140
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
141
142
143
144

  qa_config.ReleaseInstance(instance)


145
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
146
147
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
148
  AssertCommand(["gnt-instance", "startup", instance["name"]])
149
150


151
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
152
153
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
154
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
155
156


157
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
158
159
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
160
  options = qa_config.get("options", {})
161
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
162
  name = instance["name"]
163
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
164
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
165

166
  AssertCommand(["gnt-instance", "shutdown", name])
167
  qa_utils.RunInstanceCheck(instance, False)
168
169
170
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
171
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
172
173
174
175
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

176

177
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
178
179
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
180
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
181
182


183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


207
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
208
209
210
211
212
213
214
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
215
  _CheckSsconfInstanceList(rename_source)
216

Iustin Pop's avatar
Iustin Pop committed
217
  # first do a rename to a different actual name, expecting it to fail
218
219
220
221
222
223
224
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
225

Iustin Pop's avatar
Iustin Pop committed
226
  # and now rename instance to rename_target...
227
228
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
229
  qa_utils.RunInstanceCheck(rename_source, False)
230
  qa_utils.RunInstanceCheck(rename_target, False)
231

Iustin Pop's avatar
Iustin Pop committed
232
233
234
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
235
  qa_utils.RunInstanceCheck(rename_target, False)
236
237


238
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
239
240
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
241
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
242

Iustin Pop's avatar
Iustin Pop committed
243
244
  # failover ...
  AssertCommand(cmd)
245
246
  qa_utils.RunInstanceCheck(instance, True)

247
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
248
  AssertCommand(cmd)
249

250

251
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
252
253
254
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
255

Iustin Pop's avatar
Iustin Pop committed
256
257
  # migrate ...
  AssertCommand(cmd)
258
259
  qa_utils.RunInstanceCheck(instance, True)

260
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
261
  AssertCommand(cmd)
262
263

  # TODO: Split into multiple tests
264
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
265
  qa_utils.RunInstanceCheck(instance, False)
266
267
268
269
270
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
271
272
  qa_utils.RunInstanceCheck(instance, True)

273
274
275
276
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
277

278
  AssertCommand(cmd, fail=True)
279
  qa_utils.RunInstanceCheck(instance, True)
280
281
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
282
283

  # TODO: Verify whether the default value is restored here (not hardcoded)
284
285
286
287
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
288

289
  AssertCommand(cmd)
290
  qa_utils.RunInstanceCheck(instance, True)
291
292


293
294
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
295
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
296
297


298
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
299
300
def TestInstanceModify(instance):
  """gnt-instance modify"""
301
302
303
304
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

305
306
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
307
  #orig_bridge = qa_config.get("bridge", "xen-br0")
308
  args = [
309
310
311
312
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
313
314
315
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
316
317
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
318
319
320
321

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
322
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
323
324
325
326
327
328
329
330
331
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
332
333
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
334
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
335
336

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
337
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
338

339
340
341
342
343
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


344
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
345
346
347
348
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

349
350
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
351
352
353
354
355
356
357

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

358

359
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
360
361
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
362
363
364
365
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
366
367


368
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
391
392
def TestInstanceList():
  """gnt-instance list"""
393
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
394
395


396
397
398
399
400
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


401
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
402
403
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
404
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
405
406


407
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
408
409
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
410
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
411
412
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
413
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
414
    cmd = ["gnt-instance", "replace-disks"]
415
416
417
418
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
419
420
421
422
423
424
425
426
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
427

428
429
430
431
432
433
434
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

435

436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
                         destroy=True):
  """Execute gnt-instance recreate-disks and check the result

  @param cmdargs: Arguments (instance name excluded)
  @param instance: Instance to operate on
  @param fail: True if the command is expected to fail
  @param check: If True and fail is False, check that the disks work
  @prama destroy: If True, destroy the old disks first

  """
  if destroy:
    _DestroyInstanceVolumes(instance)
  AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
                 [instance["name"]]), fail)
  if not fail and check:
    # Quick check that the disks are there
    AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
    AssertCommand(["gnt-instance", "deactivate-disks", instance["name"]])

Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
456

457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
def TestRecreateDisks(instance, pnode, snode, othernodes):
  """gnt-instance recreate-disks

  @param instance: Instance to work on
  @param pnode: Primary node
  @param snode: Secondary node, or None for sigle-homed instances
  @param othernodes: list/tuple of nodes where to temporarily recreate disks

  """
  other_seq = ":".join([n["primary"] for n in othernodes])
  orig_seq = pnode["primary"]
  if snode:
    orig_seq = orig_seq + ":" + snode["primary"]
  # This fails beacuse the instance is running
  _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  # Disks exist: this should fail
  _AssertRecreateDisks([], instance, fail=True, destroy=False)
  # Recreate disks in place
  _AssertRecreateDisks([], instance)
  # Move disks away
  _AssertRecreateDisks(["-n", other_seq], instance)
  # Move disks back
  _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
  # This and InstanceCheck decoration check that the disks are working
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])


487
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
488
def TestInstanceExport(instance, node):
489
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
490
491
492
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
493
494


495
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
496
497
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
498
499
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
500
501


502
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
503
504
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
505
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
506
507


508
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
509
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
510
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
511
512
513
514
515
516
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
517
         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
Iustin Pop's avatar
Iustin Pop committed
518
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
519
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
520
521
522
523


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
524
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
525

526
527
528
529
530
531
532
533
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

534
535
536
537
538
539

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

540
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
541
542
543
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

544
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
545
546
547
548
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
549
550

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
551
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
552
553
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
554
    (node_name, phys, inst) = line.split("|")
555
556
557
558
559
560
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
561
        raise qa_error.Error("Unknown disk name format: %s" % phys)
562
563
564
565
566
567

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
568
569
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
570

571
572
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
573
574
575
576
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
577
    AssertCommand(" && ".join(cmds), node=node_name)
578

579
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
580
581
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
582
583
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
584
    (_, _, tmpdevpath) = line.split(":")
585
    devpath.append(tmpdevpath)
586
  print devpath
587

588
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
589
590
591
592
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
593
  drbddevs = re.findall(pattern, output, re.M)
594
  print drbddevs
595
596
597

  halted_disks = []
  try:
598
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
599
600
601
602
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
603
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
604

605
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
606
                              " the problem")
607
608
609
610
611
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
612
      AssertCommand(" && ".join(cmds), node=node)
613
614
      time.sleep(3)

615
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
616
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
617
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
618

Iustin Pop's avatar
Iustin Pop committed
619
    AssertCommand(["gnt-instance", "info", instance["name"]])
620
621

  finally:
622
    print qa_utils.FormatInfo("Activating disks again")
623
624
625
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
626
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
627

Michael Hanselmann's avatar
Michael Hanselmann committed
628
629
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
630
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
631
632
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
633
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
634

635
  # TODO
Iustin Pop's avatar
Iustin Pop committed
636
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
637

638
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
639
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
640
641

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
642
643
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
644

Iustin Pop's avatar
Iustin Pop committed
645
  AssertCommand(["gnt-cluster", "verify"])
646
647
648
649


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
650
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
651
  # due to unused args
652
653
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
654
  # The following can cause crashes, thus it's disabled until fixed
655
  #return _TestInstanceDiskFailure(instance, node, node2, True)
656
657
658
659
660


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)