qa_instance.py 34.1 KB
Newer Older
1
2
3
#
#

4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
import operator
27
import os
28
29
import re

30
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
31
from ganeti import constants
32
from ganeti import query
33
from ganeti import pathutils
34
35

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
36
import qa_utils
37
38
import qa_error

39
from qa_utils import AssertIn, AssertCommand, AssertEqual
40
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
41

42
43
44

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
45
46


47
def _GetGenericAddParameters(inst, disk_template, force_mac=None):
48
49
50
51
52
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
53
54
55
56

  if disk_template != constants.DT_DISKLESS:
    for idx, size in enumerate(qa_config.get("disk")):
      params.extend(["--disk", "%s:size=%s" % (idx, size)])
57
58

  # Set static MAC address if configured
59
60
61
  if force_mac:
    nic0_mac = force_mac
  else:
62
63
    nic0_mac = inst.GetNicMacAddr(0, None)

64
65
66
  if nic0_mac:
    params.extend(["--net", "0:mac=%s" % nic0_mac])

67
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
68
69


70
71
72
73
74
75
76
77
78
79
80
81
82
def _CreateInstanceByDiskTemplateRaw(nodes_spec, disk_template, fail=False):
  """Creates an instance with the given disk template on the given nodes(s).
     Note that this function does not check if enough nodes are given for
     the respective disk template.

  @type nodes_spec: string
  @param nodes_spec: string specification of one node (by node name) or several
                     nodes according to the requirements of the disk template
  @type disk_template: string
  @param disk_template: the disk template to be used by the instance
  @return: the created instance

  """
83
84
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
85
86
87
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
88
            "--node=%s" % nodes_spec] +
89
           _GetGenericAddParameters(instance, disk_template))
90
    cmd.append(instance.name)
91

92
    AssertCommand(cmd, fail=fail)
93

94
    if not fail:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
95
96
      _CheckSsconfInstanceList(instance.name)
      instance.SetDiskTemplate(disk_template)
97

98
      return instance
99
  except:
100
    instance.Release()
101
102
    raise

103
104
  # Handle the case where creation is expected to fail
  assert fail
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
105
  instance.Release()
106
107
  return None

108

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
def _CreateInstanceByDiskTemplateOneNode(nodes, disk_template, fail=False):
  """Creates an instance using the given disk template for disk templates
     for which one given node is sufficient. These templates are for example:
     plain, diskless, file, sharedfile, blockdev, rados.

  @type nodes: list of nodes
  @param nodes: a list of nodes, whose first element is used to create the
                instance
  @type disk_template: string
  @param disk_template: the disk template to be used by the instance
  @return: the created instance

  """
  assert len(nodes) > 0
  return _CreateInstanceByDiskTemplateRaw(nodes[0].primary, disk_template,
                                          fail=fail)


def _CreateInstanceDrbd8(nodes, fail=False):
  """Creates an instance using disk template 'drbd' on the given nodes.

  @type nodes: list of nodes
  @param nodes: nodes to be used by the instance
  @return: the created instance

  """
  assert len(nodes) > 1
  return _CreateInstanceByDiskTemplateRaw(
    ":".join(map(operator.attrgetter("primary"), nodes)),
    constants.DT_DRBD8, fail=fail)


def CreateInstanceByDiskTemplate(nodes, disk_template, fail=False):
  """Given a disk template, this function creates an instance using
     the template. It uses the required number of nodes depending on
     the disk template. This function is intended to be used by tests
     that don't care about the specifics of the instance other than
     that it uses the given disk template.

     Note: If you use this function, make sure to call
     'TestInstanceRemove' at the end of your tests to avoid orphaned
     instances hanging around and interfering with the following tests.

  @type nodes: list of nodes
  @param nodes: the list of the nodes on which the instance will be placed;
                it needs to have sufficiently many elements for the given
                disk template
  @type disk_template: string
  @param disk_template: the disk template to be used by the instance
  @return: the created instance

  """
  if disk_template == constants.DT_DRBD8:
    return _CreateInstanceDrbd8(nodes, fail=fail)
  elif disk_template in [constants.DT_DISKLESS, constants.DT_PLAIN]:
    return _CreateInstanceByDiskTemplateOneNode(nodes, disk_template, fail=fail)
  else:
    #FIXME: Implement this for the remaining disk templates
    qa_error.Error("Instance creation not implemented for disk type '%s'." %
                   disk_template)


171
172
def _GetInstanceInfo(instance):
  """Return information about the actual state of an instance.
173

174
175
  @type instance: string
  @param instance: the instance name
176
  @return: a dictionary with the following keys:
177
178
      - "nodes": instance nodes, a list of strings
      - "volumes": instance volume IDs, a list of strings
179
180
181
      - "drbd-minors": DRBD minors used by the instance, a dictionary where
        keys are nodes, and values are lists of integers (or an empty
        dictionary for non-DRBD instances)
182
183
      - "disk-template": instance disk template
      - "storage-type": storage type associated with the instance disk template
184
185
186
187
188
189
190
191
192

  """
  node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
  # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
  #  node1.fqdn
  #  node2.fqdn,node3.fqdn
  #  node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
  # FIXME This works with no more than 2 secondaries
  re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
193
194

  info = qa_utils.GetObjectInfo(["gnt-instance", "info", instance])[0]
195
  nodes = []
196
197
198
199
200
201
202
203
204
205
206
207
  for nodeinfo in info["Nodes"]:
    if "primary" in nodeinfo:
      nodes.append(nodeinfo["primary"])
    elif "secondaries" in nodeinfo:
      nodestr = nodeinfo["secondaries"]
      if nodestr:
        m = re_nodelist.match(nodestr)
        if m:
          nodes.extend(filter(None, m.groups()))
        else:
          nodes.append(nodestr)

208
209
210
211
212
  disk_template = info["Disk template"]
  if not disk_template:
    raise qa_error.Error("Can't get instance disk template")
  storage_type = constants.DISK_TEMPLATES_STORAGE_TYPE[disk_template]

213
  re_drbdnode = re.compile(r"^([^\s,]+),\s+minor=([0-9]+)$")
214
  vols = []
215
  drbd_min = {}
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
  for (count, diskinfo) in enumerate(info["Disks"]):
    (dtype, _) = diskinfo["disk/%s" % count].split(",", 1)
    if dtype == constants.LD_DRBD8:
      for child in diskinfo["child devices"]:
        vols.append(child["logical_id"])
      for key in ["nodeA", "nodeB"]:
        m = re_drbdnode.match(diskinfo[key])
        if not m:
          raise qa_error.Error("Cannot parse DRBD info: %s" % diskinfo[key])
        node = m.group(1)
        minor = int(m.group(2))
        minorlist = drbd_min.setdefault(node, [])
        minorlist.append(minor)
    elif dtype == constants.LD_LV:
      vols.append(diskinfo["logical_id"])
231

232
  assert nodes
233
  assert len(nodes) < 2 or vols
234
235
236
237
  return {
    "nodes": nodes,
    "volumes": vols,
    "drbd-minors": drbd_min,
238
239
    "disk-template": disk_template,
    "storage-type": storage_type,
240
    }
241
242


243
244
def _DestroyInstanceDisks(instance):
  """Remove all the backend disks of an instance.
245
246
247
248
249
250
251

  This is used to simulate HW errors (dead nodes, broken disks...); the
  configuration of the instance is not affected.
  @type instance: dictionary
  @param instance: the instance

  """
252
  info = _GetInstanceInfo(instance.name)
253
254
255
256
257
258
259
260
261
262
263
  # FIXME: destruction/removal should be part of the disk class
  if info["storage-type"] == constants.ST_LVM_VG:
    vols = info["volumes"]
    for node in info["nodes"]:
      AssertCommand(["lvremove", "-f"] + vols, node=node)
  elif info["storage-type"] == constants.ST_FILE:
    # FIXME: file storage dir not configurable in qa
    # Note that this works for both file and sharedfile, and this is intended.
    filestorage = pathutils.DEFAULT_FILE_STORAGE_DIR
    idir = os.path.join(filestorage, instance.name)
    for node in info["nodes"]:
264
      AssertCommand(["rm", "-rf", idir], node=node)
265
266
  elif info["storage-type"] == constants.ST_DISKLESS:
    pass
267
268


269
270
def _GetInstanceField(instance, field):
  """Get the value of a field of an instance.
271
272
273
274
275

  @type instance: string
  @param instance: Instance name
  @type field: string
  @param field: Name of the field
276
  @rtype: string
277
278
279
280

  """
  master = qa_config.GetMasterNode()
  infocmd = utils.ShellQuoteArgs(["gnt-instance", "list", "--no-headers",
281
                                  "--units", "m", "-o", field, instance])
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
282
  return qa_utils.GetCommandOutput(master.primary, infocmd).strip()
283
284
285
286
287
288
289
290
291
292
293
294
295


def _GetBoolInstanceField(instance, field):
  """Get the Boolean value of a field of an instance.

  @type instance: string
  @param instance: Instance name
  @type field: string
  @param field: Name of the field
  @rtype: bool

  """
  info_out = _GetInstanceField(instance, field)
296
297
298
299
300
301
302
303
304
  if info_out == "Y":
    return True
  elif info_out == "N":
    return False
  else:
    raise qa_error.Error("Field %s of instance %s has a non-Boolean value:"
                         " %s" % (field, instance, info_out))


305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
def _GetNumInstanceField(instance, field):
  """Get a numeric value of a field of an instance.

  @type instance: string
  @param instance: Instance name
  @type field: string
  @param field: Name of the field
  @rtype: int or float

  """
  info_out = _GetInstanceField(instance, field)
  try:
    ret = int(info_out)
  except ValueError:
    try:
      ret = float(info_out)
    except ValueError:
      raise qa_error.Error("Field %s of instance %s has a non-numeric value:"
                           " %s" % (field, instance, info_out))
  return ret


def GetInstanceSpec(instance, spec):
  """Return the current spec for the given parameter.

  @type instance: string
  @param instance: Instance name
  @type spec: string
  @param spec: one of the supported parameters: "mem-size", "cpu-count",
      "disk-count", "disk-size", "nic-count"
  @rtype: tuple
  @return: (minspec, maxspec); minspec and maxspec can be different only for
      memory and disk size

  """
  specmap = {
    "mem-size": ["be/minmem", "be/maxmem"],
    "cpu-count": ["vcpus"],
    "disk-count": ["disk.count"],
    "disk-size": ["disk.size/ "],
    "nic-count": ["nic.count"],
    }
  # For disks, first we need the number of disks
  if spec == "disk-size":
    (numdisk, _) = GetInstanceSpec(instance, "disk-count")
    fields = ["disk.size/%s" % k for k in range(0, numdisk)]
  else:
    assert spec in specmap, "%s not in %s" % (spec, specmap)
    fields = specmap[spec]
  values = [_GetNumInstanceField(instance, f) for f in fields]
  return (min(values), max(values))


358
def IsFailoverSupported(instance):
359
  return instance.disk_template in constants.DTS_MIRRORED
360
361
362


def IsMigrationSupported(instance):
363
  return instance.disk_template in constants.DTS_MIRRORED
364
365
366


def IsDiskReplacingSupported(instance):
367
  return instance.disk_template == constants.DT_DRBD8
368
369


370
def TestInstanceAddWithPlainDisk(nodes, fail=False):
371
  """gnt-instance add -t plain"""
372
373
374
375
376
377
  if constants.DT_PLAIN in qa_config.GetEnabledDiskTemplates():
    instance = _CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_PLAIN,
                                                    fail=fail)
    if not fail:
      qa_utils.RunInstanceCheck(instance, True)
    return instance
378
379


380
@InstanceCheck(None, INST_UP, RETURN_VALUE)
381
def TestInstanceAddWithDrbdDisk(nodes):
382
  """gnt-instance add -t drbd"""
383
384
  if constants.DT_DRBD8 in qa_config.GetEnabledDiskTemplates():
    return _CreateInstanceDrbd8(nodes)
385
386


387
388
389
390
@InstanceCheck(None, INST_UP, RETURN_VALUE)
def TestInstanceAddFile(nodes):
  """gnt-instance add -t file"""
  assert len(nodes) == 1
391
392
  if constants.DT_FILE in qa_config.GetEnabledDiskTemplates():
    return _CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_FILE)
393
394


395
396
397
398
@InstanceCheck(None, INST_UP, RETURN_VALUE)
def TestInstanceAddDiskless(nodes):
  """gnt-instance add -t diskless"""
  assert len(nodes) == 1
399
400
  if constants.DT_FILE in qa_config.GetEnabledDiskTemplates():
    return _CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_DISKLESS)
401
402


403
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
404
405
def TestInstanceRemove(instance):
  """gnt-instance remove"""
406
  AssertCommand(["gnt-instance", "remove", "-f", instance.name])
407
408


409
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
410
411
def TestInstanceStartup(instance):
  """gnt-instance startup"""
412
  AssertCommand(["gnt-instance", "startup", instance.name])
413
414


415
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
416
417
def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
418
  AssertCommand(["gnt-instance", "shutdown", instance.name])
419
420


421
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
422
423
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
424
  options = qa_config.get("options", {})
425
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
426
  name = instance.name
427
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
428
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
429

430
  AssertCommand(["gnt-instance", "shutdown", name])
431
  qa_utils.RunInstanceCheck(instance, False)
432
433
434
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
435
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
436
  result_output = qa_utils.GetCommandOutput(master.primary,
437
438
439
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

440

441
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
442
443
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
444
445
446
447
  if instance.disk_template == constants.DT_DISKLESS:
    print qa_utils.FormatInfo("Test not supported for diskless instances")
    return

448
  AssertCommand(["gnt-instance", "reinstall", "-f", instance.name])
Michael Hanselmann's avatar
Michael Hanselmann committed
449

450
451
452
  # Test with non-existant OS definition
  AssertCommand(["gnt-instance", "reinstall", "-f",
                 "--os-type=NonExistantOsForQa",
453
                 instance.name],
454
455
                fail=True)

Michael Hanselmann's avatar
Michael Hanselmann committed
456

457
458
459
460
461
462
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

463
464
465
466
  ssconf_path = utils.PathJoin(pathutils.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)

  cmd = ["cat", qa_utils.MakeNodePath(master, ssconf_path)]
467

468
  return qa_utils.GetCommandOutput(master.primary,
469
470
471
472
473
474
475
476
477
478
479
480
481
482
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


483
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
484
485
486
487
488
489
490
def TestInstanceRenameAndBack(rename_source, rename_target):
  """gnt-instance rename

  This must leave the instance with the original name, not the target
  name.

  """
491
  _CheckSsconfInstanceList(rename_source)
492

Iustin Pop's avatar
Iustin Pop committed
493
  # first do a rename to a different actual name, expecting it to fail
494
495
496
497
498
499
500
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
501

502
  info = _GetInstanceInfo(rename_source)
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517

  # Check instance volume tags correctly updated. Note that this check is lvm
  # specific, so we skip it for non-lvm-based instances.
  # FIXME: This will need updating when instances will be able to have
  # different disks living on storage pools with etherogeneous storage types.
  # FIXME: This check should be put inside the disk/storage class themselves,
  # rather than explicitly called here.
  if info["storage-type"] == constants.ST_LVM_VG:
    # In the lvm world we can check for tags on the logical volume
    tags_cmd = ("lvs -o tags --noheadings %s | grep " %
                (" ".join(info["volumes"]), ))
  else:
    # Other storage types don't have tags, so we use an always failing command,
    # to make sure it never gets executed
    tags_cmd = "false"
518

Iustin Pop's avatar
Iustin Pop committed
519
  # and now rename instance to rename_target...
520
521
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
522
  qa_utils.RunInstanceCheck(rename_source, False)
523
  qa_utils.RunInstanceCheck(rename_target, False)
524

525
526
  # NOTE: tags might not be the exactly as the instance name, due to
  # charset restrictions; hence the test might be flaky
527
528
  if (rename_source != rename_target and
      info["storage-type"] == constants.ST_LVM_VG):
529
530
531
532
    for node in info["nodes"]:
      AssertCommand(tags_cmd + rename_source, node=node, fail=True)
      AssertCommand(tags_cmd + rename_target, node=node, fail=False)

Iustin Pop's avatar
Iustin Pop committed
533
534
535
  # and back
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
536
  qa_utils.RunInstanceCheck(rename_target, False)
537

538
539
  if (rename_source != rename_target and
      info["storage-type"] == constants.ST_LVM_VG):
540
541
542
543
    for node in info["nodes"]:
      AssertCommand(tags_cmd + rename_source, node=node, fail=False)
      AssertCommand(tags_cmd + rename_target, node=node, fail=True)

544

545
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
546
547
def TestInstanceFailover(instance):
  """gnt-instance failover"""
548
549
550
551
552
  if not IsFailoverSupported(instance):
    print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
                              " test")
    return

553
  cmd = ["gnt-instance", "failover", "--force", instance.name]
554

Iustin Pop's avatar
Iustin Pop committed
555
556
  # failover ...
  AssertCommand(cmd)
557
558
  qa_utils.RunInstanceCheck(instance, True)

559
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
560
  AssertCommand(cmd)
561

562

563
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
564
def TestInstanceMigrate(instance, toggle_always_failover=True):
565
  """gnt-instance migrate"""
566
567
568
569
570
  if not IsMigrationSupported(instance):
    print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
                              " test")
    return

571
  cmd = ["gnt-instance", "migrate", "--force", instance.name]
572
573
  af_par = constants.BE_ALWAYS_FAILOVER
  af_field = "be/" + constants.BE_ALWAYS_FAILOVER
574
  af_init_val = _GetBoolInstanceField(instance.name, af_field)
575

Iustin Pop's avatar
Iustin Pop committed
576
577
  # migrate ...
  AssertCommand(cmd)
578
  # TODO: Verify the choice between failover and migration
579
580
  qa_utils.RunInstanceCheck(instance, True)

581
582
583
584
  # ... and back (possibly with always_failover toggled)
  if toggle_always_failover:
    AssertCommand(["gnt-instance", "modify", "-B",
                   ("%s=%s" % (af_par, not af_init_val)),
585
                   instance.name])
Iustin Pop's avatar
Iustin Pop committed
586
  AssertCommand(cmd)
587
588
589
590
  # TODO: Verify the choice between failover and migration
  qa_utils.RunInstanceCheck(instance, True)
  if toggle_always_failover:
    AssertCommand(["gnt-instance", "modify", "-B",
591
                   ("%s=%s" % (af_par, af_init_val)), instance.name])
592
593

  # TODO: Split into multiple tests
594
  AssertCommand(["gnt-instance", "shutdown", instance.name])
595
  qa_utils.RunInstanceCheck(instance, False)
596
597
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
598
599
                 instance.name])
  AssertCommand(["gnt-instance", "start", instance.name])
600
  AssertCommand(cmd)
601
  # @InstanceCheck enforces the check that the instance is running
602
603
  qa_utils.RunInstanceCheck(instance, True)

604
605
606
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
607
                 instance.name])
608

609
  AssertCommand(cmd)
610
  qa_utils.RunInstanceCheck(instance, True)
611
  # TODO: Verify that a failover has been done instead of a migration
612
613

  # TODO: Verify whether the default value is restored here (not hardcoded)
614
615
616
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
617
                 instance.name])
618

619
  AssertCommand(cmd)
620
  qa_utils.RunInstanceCheck(instance, True)
621
622


623
624
def TestInstanceInfo(instance):
  """gnt-instance info"""
625
  AssertCommand(["gnt-instance", "info", instance.name])
Michael Hanselmann's avatar
Michael Hanselmann committed
626
627


628
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
629
630
def TestInstanceModify(instance):
  """gnt-instance modify"""
631
632
  default_hv = qa_config.GetDefaultHypervisor()

633
634
635
636
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

637
638
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
639
  #orig_bridge = qa_config.get("bridge", "xen-br0")
640

641
  args = [
642
643
644
645
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
646
647
648
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
649
650
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
651
652
653
654
655
656
657

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],
658
    ]
659
660
661
662
663
664
665
666
667
668
669
670
671

  if default_hv == constants.HT_XEN_PVM:
    args.extend([
      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
      ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
      ])
  elif default_hv == constants.HT_XEN_HVM:
    args.extend([
      ["-H", "%s=acn" % constants.HV_BOOT_ORDER],
      ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
      ])

672
  for alist in args:
673
    AssertCommand(["gnt-instance", "modify"] + alist + [instance.name])
674
675

  # check no-modify
676
  AssertCommand(["gnt-instance", "modify", instance.name], fail=True)
677

678
  # Marking offline while instance is running must fail...
679
  AssertCommand(["gnt-instance", "modify", "--offline", instance.name],
680
681
682
                 fail=True)

  # ...while making it online is ok, and should work
683
  AssertCommand(["gnt-instance", "modify", "--online", instance.name])
684
685


686
687
688
689
690
691
692
693
694
695
696
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
def TestInstanceModifyPrimaryAndBack(instance, currentnode, othernode):
  """gnt-instance modify --new-primary

  This will leave the instance on its original primary node, not other node.

  """
  if instance.disk_template != constants.DT_FILE:
    print qa_utils.FormatInfo("Test only supported for the file disk template")
    return

697
698
  cluster_name = qa_config.get("name")

699
700
701
702
  name = instance.name
  current = currentnode.primary
  other = othernode.primary

Guido Trotter's avatar
Guido Trotter committed
703
704
705
  # FIXME: the qa doesn't have a customizable file storage dir parameter. As
  # such for now we use the default.
  filestorage = pathutils.DEFAULT_FILE_STORAGE_DIR
706
707
708
709
710
  disk = os.path.join(filestorage, name)

  AssertCommand(["gnt-instance", "modify", "--new-primary=%s" % other, name],
                fail=True)
  AssertCommand(["gnt-instance", "shutdown", name])
711
712
713
714
  AssertCommand(["scp", "-oGlobalKnownHostsFile=%s" %
                 pathutils.SSH_KNOWN_HOSTS_FILE,
                 "-oCheckHostIp=no", "-oStrictHostKeyChecking=yes",
                 "-oHashKnownHosts=no", "-oHostKeyAlias=%s" % cluster_name,
715
                 "-r", disk, "%s:%s" % (other, filestorage)], node=current)
716
717
718
719
720
721
722
723
724
725
  AssertCommand(["gnt-instance", "modify", "--new-primary=%s" % other, name])
  AssertCommand(["gnt-instance", "startup", name])

  # and back
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["rm", "-rf", disk], node=other)
  AssertCommand(["gnt-instance", "modify", "--new-primary=%s" % current, name])
  AssertCommand(["gnt-instance", "startup", name])


726
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
727
728
def TestInstanceStoppedModify(instance):
  """gnt-instance modify (stopped instance)"""
729
  name = instance.name
730

731
732
  # Instance was not marked offline; try marking it online once more
  AssertCommand(["gnt-instance", "modify", "--online", name])
733
734
735
736

  # Mark instance as offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

737
738
739
740
741
742
743
744
745
746
  # When the instance is offline shutdown should only work with --force,
  # while start should never work
  AssertCommand(["gnt-instance", "shutdown", name], fail=True)
  AssertCommand(["gnt-instance", "shutdown", "--force", name])
  AssertCommand(["gnt-instance", "start", name], fail=True)
  AssertCommand(["gnt-instance", "start", "--force", name], fail=True)

  # Also do offline to offline
  AssertCommand(["gnt-instance", "modify", "--offline", name])

747
748
749
  # And online again
  AssertCommand(["gnt-instance", "modify", "--online", name])

750

751
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
752
def TestInstanceConvertDiskToPlain(instance, inodes):
753
  """gnt-instance modify -t"""
754
  name = instance.name
755

756
  template = instance.disk_template
757
  if template != constants.DT_DRBD8:
758
759
760
    print qa_utils.FormatInfo("Unsupported template %s, skipping conversion"
                              " test" % template)
    return
761

762
  assert len(inodes) == 2
763
764
  AssertCommand(["gnt-instance", "modify", "-t", constants.DT_PLAIN, name])
  AssertCommand(["gnt-instance", "modify", "-t", constants.DT_DRBD8,
765
                 "-n", inodes[1].primary, name])
766
767


768
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
Iustin Pop's avatar
Iustin Pop committed
769
770
def TestInstanceGrowDisk(instance):
  """gnt-instance grow-disk"""
771
772
773
  if qa_config.GetExclusiveStorage():
    print qa_utils.FormatInfo("Test not supported with exclusive_storage")
    return
774
775
776
777
778

  if instance.disk_template == constants.DT_DISKLESS:
    print qa_utils.FormatInfo("Test not supported for diskless instances")
    return

779
  name = instance.name
Iustin Pop's avatar
Iustin Pop committed
780
781
  all_size = qa_config.get("disk")
  all_grow = qa_config.get("disk-growth")
782

Iustin Pop's avatar
Iustin Pop committed
783
784
785
786
  if not all_grow:
    # missing disk sizes but instance grow disk has been enabled,
    # let's set fixed/nomimal growth
    all_grow = ["128M" for _ in all_size]
787

Iustin Pop's avatar
Iustin Pop committed
788
789
790
791
792
793
794
795
796
797
798
799
800
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
    # succeed in grow by amount
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
    # fail in grow to the old size
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   size], fail=True)
    # succeed to grow to old size + 2 * growth
    int_size = utils.ParseUnit(size)
    int_grow = utils.ParseUnit(grow)
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
                   str(int_size + 2 * int_grow)])


Michael Hanselmann's avatar
Michael Hanselmann committed
801
802
def TestInstanceList():
  """gnt-instance list"""
803
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
804
805


806
807
808
809
810
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


811
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
812
813
def TestInstanceConsole(instance):
  """gnt-instance console"""
814
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance.name])
815
816


817
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
818
def TestReplaceDisks(instance, curr_nodes, other_nodes):
819
820
  """gnt-instance replace-disks"""
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
821
    cmd = ["gnt-instance", "replace-disks"]
822
    cmd.extend(args)
823
    cmd.append(instance.name)
824
825
    return cmd

826
827
828
829
830
  if not IsDiskReplacingSupported(instance):
    print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
                              " skipping test")
    return

831
832
833
834
835
836
  # Currently all supported templates have one primary and one secondary node
  assert len(curr_nodes) == 2
  snode = curr_nodes[1]
  assert len(other_nodes) == 1
  othernode = other_nodes[0]

837
838
  options = qa_config.get("options", {})
  use_ialloc = options.get("use-iallocators", True)
Iustin Pop's avatar
Iustin Pop committed
839
840
841
  for data in [
    ["-p"],
    ["-s"],
842
843
844
    # A placeholder; the actual command choice depends on use_ialloc
    None,
    # Restore the original secondary
845
    ["--new-secondary=%s" % snode.primary],
Iustin Pop's avatar
Iustin Pop committed
846
    ]:
847
848
849
850
    if data is None:
      if use_ialloc:
        data = ["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT]
      else:
851
        data = ["--new-secondary=%s" % othernode.primary]
Iustin Pop's avatar
Iustin Pop committed
852
    AssertCommand(buildcmd(data))
853

854
  AssertCommand(buildcmd(["-a"]))
855
  AssertCommand(["gnt-instance", "stop", instance.name])
856
  AssertCommand(buildcmd(["-a"]), fail=True)
857
  AssertCommand(["gnt-instance", "activate-disks", instance.name])
858
  AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
859
                 instance.name])
860
  AssertCommand(buildcmd(["-a"]))
861
  AssertCommand(["gnt-instance", "start", instance.name])
862

863

864
865
866
867
868
869
870
871
872
873
874
875
def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
                         destroy=True):
  """Execute gnt-instance recreate-disks and check the result

  @param cmdargs: Arguments (instance name excluded)
  @param instance: Instance to operate on
  @param fail: True if the command is expected to fail
  @param check: If True and fail is False, check that the disks work
  @prama destroy: If True, destroy the old disks first

  """
  if destroy:
876
    _DestroyInstanceDisks(instance)
877
  AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
878
                 [instance.name]), fail)
879
880
  if not fail and check:
    # Quick check that the disks are there
881
    AssertCommand(["gnt-instance", "activate-disks", instance.name])
882
    AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
883
884
                   instance.name])
    AssertCommand(["gnt-instance", "deactivate-disks", instance.name])
885

Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
886

887
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
888
def TestRecreateDisks(instance, inodes, othernodes):
889
890
891
  """gnt-instance recreate-disks

  @param instance: Instance to work on
892
  @param inodes: List of the current nodes of the instance
893
894
895
  @param othernodes: list/tuple of nodes where to temporarily recreate disks

  """
896
897
  options = qa_config.get("options", {})
  use_ialloc = options.get("use-iallocators", True)
898
899
  other_seq = ":".join([n.primary for n in othernodes])
  orig_seq = ":".join([n.primary for n in inodes])
900
  # These fail because the instance is running
901
  _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
902
903
904
905
  if use_ialloc:
    _AssertRecreateDisks(["-I", "hail"], instance, fail=True, destroy=False)
  else:
    _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
906
  AssertCommand(["gnt-instance", "stop", instance.name])
907
908
909
910
911
  # Disks exist: this should fail
  _AssertRecreateDisks([], instance, fail=True, destroy=False)
  # Recreate disks in place
  _AssertRecreateDisks([], instance)
  # Move disks away
912
913
914
915
916
917
918
  if use_ialloc:
    _AssertRecreateDisks(["-I", "hail"], instance)
    # Move disks somewhere else
    _AssertRecreateDisks(["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT],
                         instance)
  else:
    _AssertRecreateDisks(["-n", other_seq], instance)
919
920
921
  # Move disks back
  _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
  # This and InstanceCheck decoration check that the disks are working
922
923
  AssertCommand(["gnt-instance", "reinstall", "-f", instance.name])
  AssertCommand(["gnt-instance", "start", instance.name])
924
925


926
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
Michael Hanselmann's avatar
Michael Hanselmann committed
927
def TestInstanceExport(instance, node):
928
  """gnt-backup export -n ..."""
929
  name = instance.name
930
  AssertCommand(["gnt-backup", "export", "-n", node.primary, name])
Iustin Pop's avatar
Iustin Pop committed
931
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
932
933


934
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
935
936
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
937
  AssertCommand(["gnt-backup", "export", "-n", node.primary,
938
                 "--remove-instance", instance.name])
939
940


941
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
942
943
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
944
  AssertCommand(["gnt-backup", "export", instance.name], fail=True)
945
946


947
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
948
def TestInstanceImport(newinst, node, expnode, name):
Michael Hanselmann's avatar
Michael Hanselmann committed
949
  """gnt-backup import"""
950
  templ = constants.DT_PLAIN
Iustin Pop's avatar
Iustin Pop committed
951
  cmd = (["gnt-backup", "import",
952
          "--disk-template=%s" % templ,
Iustin Pop's avatar
Iustin Pop committed
953
          "--no-ip-check",
954
          "--src-node=%s" % expnode.primary,
955
          "--src-dir=%s/%s" % (pathutils.EXPORT_DIR, name),
956
          "--node=%s" % node.primary] +
957
958
         _GetGenericAddParameters(newinst, templ,
                                  force_mac=constants.VALUE_GENERATE))
959
  cmd.append(newinst.name)
Iustin Pop's avatar
Iustin Pop committed
960
  AssertCommand(cmd)
961
  newinst.SetDiskTemplate(templ)
Michael Hanselmann's avatar
Michael Hanselmann committed
962
963
964
965


def TestBackupList(expnode):
  """gnt-backup list"""
966
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode.primary])
967

968
969
970
971
972
973
974
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
                            namefield=None, test_unknown=False)


def TestBackupListFields():
  """gnt-backup list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())
975
976
977


def TestRemoveInstanceOfflineNode(instance, snode, set_offline, set_online):
978
  """gnt-instance remove with an off-line node
979
980
981
982
983
984
985

  @param instance: instance
  @param snode: secondary node, to be set offline
  @param set_offline: function to call to set the node off-line
  @param set_online: function to call to set the node on-line

  """
986
  info = _GetInstanceInfo(instance.name)
987
988
989
990
991
  set_offline(snode)
  try:
    TestInstanceRemove(instance)
  finally:
    set_online(snode)
992
993
994
995
996
997
998
999
1000
1001
1002
1003

  # Clean up the disks on the offline node, if necessary
  if instance.disk_template not in constants.DTS_EXT_MIRROR:
    # FIXME: abstract the cleanup inside the disks
    if info["storage-type"] == constants.ST_LVM_VG:
      for minor in info["drbd-minors"][snode.primary]:
        AssertCommand(["drbdsetup", str(minor), "down"], node=snode)
      AssertCommand(["lvremove", "-f"] + info["volumes"], node=snode)
    elif info["storage-type"] == constants.ST_FILE:
      filestorage = pathutils.DEFAULT_FILE_STORAGE_DIR
      disk = os.path.join(filestorage, instance.name)
      AssertCommand(["rm", "-rf", disk], node=snode)