qa_instance.py 21.9 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

40
41
42

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
43
44


45
def _GetGenericAddParameters(inst, force_mac=None):
46
47
48
49
50
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53
54

  # Set static MAC address if configured
55
56
57
58
  if force_mac:
    nic0_mac = force_mac
  else:
    nic0_mac = qa_config.GetInstanceNicMac(inst)
59
60
61
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

62
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
63
64


65
def _DiskTest(node, disk_template):
66
67
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
68
69
70
71
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
72
           _GetGenericAddParameters(instance))
Iustin Pop's avatar
Iustin Pop committed
73
    cmd.append(instance["name"])
74

Iustin Pop's avatar
Iustin Pop committed
75
    AssertCommand(cmd)
76
77
78

    _CheckSsconfInstanceList(instance["name"])

79
80
81
82
83
84
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def _DestroyInstanceVolumes(instance):
  """Remove all the LVM volumes of an instance.

  This is used to simulate HW errors (dead nodes, broken disks...); the
  configuration of the instance is not affected.

  """
  master = qa_config.GetMasterNode()
  infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance["name"]])
  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
  re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
  node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
  # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
  #  node1.fqdn
  #  node2.fqdn,node3.fqdn
  #  node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
  # FIXME This works with no more than 2 secondaries
  re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
  re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
  nodes = []
  vols = []
  for line in info_out.splitlines():
    m = re_node.match(line)
    if m:
      nodestr = m.group(1)
      m2 = re_nodelist.match(nodestr)
      if m2:
        nodes.extend(filter(None, m2.groups()))
      else:
        nodes.append(nodestr)
    m = re_vol.match(line)
    if m:
      vols.append(m.group(1))
  assert vols
  assert nodes
  for node in nodes:
    AssertCommand(["lvremove", "-f"] + vols, node=node)


124
@InstanceCheck(None, INST_UP, RETURN_VALUE)
125
126
def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
127
  return _DiskTest(node["primary"], "plain")
128
129


130
@InstanceCheck(None, INST_UP, RETURN_VALUE)
131
132
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
133
134
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
135
136


137
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
138
139
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
140
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
141
142
143
144

  qa_config.ReleaseInstance(instance)


145
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
146
147
def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
148
  AssertCommand(["gnt-instance", "startup", instance["name"]])
149
150


151
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
152
153
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
154
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
155
156


157
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
158
159
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
160
  options = qa_config.get("options", {})
161
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
162
  name = instance["name"]
163
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
164
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
165

166
  AssertCommand(["gnt-instance", "shutdown", name])
167
  qa_utils.RunInstanceCheck(instance, False)
168
169
170
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
171
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
172
173
174
175
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

176

177
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
178
179
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
180
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
181
182


183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


207
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
208
209
210
211
212
213
214
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
215
  _CheckSsconfInstanceList(rename_source)
216

Iustin Pop's avatar
Iustin Pop committed
217
  # first do a rename to a different actual name, expecting it to fail
218
219
220
221
222
223
224
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
225

Iustin Pop's avatar
Iustin Pop committed
226
  # and now rename instance to rename_target...
227
228
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
229
  qa_utils.RunInstanceCheck(rename_source, False)
230
  qa_utils.RunInstanceCheck(rename_target, False)
231

Iustin Pop's avatar
Iustin Pop committed
232
233
234
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
235
  qa_utils.RunInstanceCheck(rename_target, False)
236
237


238
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
239
240
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
241
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
242

Iustin Pop's avatar
Iustin Pop committed
243
244
  # failover ...
  AssertCommand(cmd)
245
246
  qa_utils.RunInstanceCheck(instance, True)

247
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
248
  AssertCommand(cmd)
249

250

251
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
252
253
254
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
255

Iustin Pop's avatar
Iustin Pop committed
256
257
  # migrate ...
  AssertCommand(cmd)
258
259
  qa_utils.RunInstanceCheck(instance, True)

260
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
261
  AssertCommand(cmd)
262
263

  # TODO: Split into multiple tests
264
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
265
  qa_utils.RunInstanceCheck(instance, False)
266
267
268
269
270
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
271
272
  qa_utils.RunInstanceCheck(instance, True)

273
274
275
276
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
277

278
  AssertCommand(cmd, fail=True)
279
  qa_utils.RunInstanceCheck(instance, True)
280
281
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
282
283

  # TODO: Verify whether the default value is restored here (not hardcoded)
284
285
286
287
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
288

289
  AssertCommand(cmd)
290
  qa_utils.RunInstanceCheck(instance, True)
291
292


293
294
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
295
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
296
297


298
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
299
300
def TestInstanceModify(instance):
  """gnt-instance modify"""
301
302
303
304
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

305
306
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
307
  #orig_bridge = qa_config.get("bridge", "xen-br0")
308
  args = [
309
310
311
312
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
313
314
315
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
316
317
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
318
319
320
321

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
322
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
323
324
325
326
327
328
329
330
331
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
332
333
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
334
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
335
336

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
337
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
338

339
340
341
342
343
  # Marking offline/online while instance is running must fail
  for arg in ["--online", "--offline"]:
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)


344
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
345
346
347
348
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
  name = instance["name"]

349
350
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
351
352
353
354
355
356
357

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

358

359
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
360
361
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
362
363
364
365
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
366
367


368
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
  name = instance["name"]
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
391
392
def TestInstanceList():
  """gnt-instance list"""
393
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
394
395


396
397
398
399
400
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


401
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
402
403
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
404
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
405
406


407
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
408
409
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
410
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
411
412
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
413
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
414
    cmd = ["gnt-instance", "replace-disks"]
415
416
417
418
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
419
420
421
422
423
424
425
426
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
427

428
429
430
431
432
433
434
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

435

436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
                         destroy=True):
  """Execute gnt-instance recreate-disks and check the result

  @param cmdargs: Arguments (instance name excluded)
  @param instance: Instance to operate on
  @param fail: True if the command is expected to fail
  @param check: If True and fail is False, check that the disks work
  @prama destroy: If True, destroy the old disks first

  """
  if destroy:
    _DestroyInstanceVolumes(instance)
  AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
                 [instance["name"]]), fail)
  if not fail and check:
    # Quick check that the disks are there
    AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
    AssertCommand(["gnt-instance", "deactivate-disks", instance["name"]])

@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
def TestRecreateDisks(instance, pnode, snode, othernodes):
  """gnt-instance recreate-disks

  @param instance: Instance to work on
  @param pnode: Primary node
  @param snode: Secondary node, or None for sigle-homed instances
  @param othernodes: list/tuple of nodes where to temporarily recreate disks

  """
  other_seq = ":".join([n["primary"] for n in othernodes])
  orig_seq = pnode["primary"]
  if snode:
    orig_seq = orig_seq + ":" + snode["primary"]
  # This fails beacuse the instance is running
  _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  # Disks exist: this should fail
  _AssertRecreateDisks([], instance, fail=True, destroy=False)
  # Recreate disks in place
  _AssertRecreateDisks([], instance)
  # Move disks away
  _AssertRecreateDisks(["-n", other_seq], instance)
  # Move disks back
  _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
  # This and InstanceCheck decoration check that the disks are working
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])


486
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
487
def TestInstanceExport(instance, node):
488
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
489
490
491
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
492
493


494
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
495
496
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
497
498
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
499
500


501
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
502
503
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
504
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
505
506


507
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
508
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
509
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
510
511
512
513
514
515
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
516
         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
Iustin Pop's avatar
Iustin Pop committed
517
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
518
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
519
520
521
522


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
523
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
524

525
526
527
528
529
530
531
532
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())

533
534
535
536
537
538

def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

539
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
540
541
542
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

543
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
544
545
546
547
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
548
549

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
550
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
551
552
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
553
    (node_name, phys, inst) = line.split("|")
554
555
556
557
558
559
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
560
        raise qa_error.Error("Unknown disk name format: %s" % phys)
561
562
563
564
565
566

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
567
568
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
569

570
571
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
572
573
574
575
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
576
    AssertCommand(" && ".join(cmds), node=node_name)
577

578
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
579
580
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
581
582
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
583
    (_, _, tmpdevpath) = line.split(":")
584
    devpath.append(tmpdevpath)
585
  print devpath
586

587
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
588
589
590
591
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
592
  drbddevs = re.findall(pattern, output, re.M)
593
  print drbddevs
594
595
596

  halted_disks = []
  try:
597
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
598
599
600
601
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
602
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
603

604
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
605
                              " the problem")
606
607
608
609
610
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
611
      AssertCommand(" && ".join(cmds), node=node)
612
613
      time.sleep(3)

614
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
615
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
616
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
617

Iustin Pop's avatar
Iustin Pop committed
618
    AssertCommand(["gnt-instance", "info", instance["name"]])
619
620

  finally:
621
    print qa_utils.FormatInfo("Activating disks again")
622
623
624
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
625
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
626

Michael Hanselmann's avatar
Michael Hanselmann committed
627
628
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
629
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
630
631
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
632
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
633

634
  # TODO
Iustin Pop's avatar
Iustin Pop committed
635
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
636

637
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
638
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
639
640

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
641
642
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
643

Iustin Pop's avatar
Iustin Pop committed
644
  AssertCommand(["gnt-cluster", "verify"])
645
646
647
648


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
649
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
650
  # due to unused args
651
652
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
653
  # The following can cause crashes, thus it's disabled until fixed
654
  #return _TestInstanceDiskFailure(instance, node, node2, True)
655
656
657
658
659


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)