qa_cluster.py 41.9 KB
Newer Older
1
2
3
#
#

4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Cluster related QA tests.

"""

26
import re
27
import tempfile
28
import os.path
29

30
from ganeti import constants
René Nussbaumer's avatar
René Nussbaumer committed
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34
35

import qa_config
36
import qa_daemon
37
import qa_error
38
import qa_instance
39
40
import qa_logging
import qa_utils
41

René Nussbaumer's avatar
René Nussbaumer committed
42
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
43
44


45
46
47
# Prefix for LVM volumes created by QA code during tests
_QA_LV_PREFIX = "qa-"

48
49
50
#: cluster verify command
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]

Andrea Spadaccini's avatar
Andrea Spadaccini committed
51

52
53
54
55
def _RemoveFileFromAllNodes(filename):
  """Removes a file from all nodes.

  """
Iustin Pop's avatar
Iustin Pop committed
56
57
  for node in qa_config.get("nodes"):
    AssertCommand(["rm", "-f", filename], node=node)
58
59
60
61
62
63
64


def _CheckFileOnAllNodes(filename, content):
  """Verifies the content of the given file on all nodes.

  """
  cmd = utils.ShellQuoteArgs(["cat", filename])
Iustin Pop's avatar
Iustin Pop committed
65
  for node in qa_config.get("nodes"):
66
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
67
68


69
70
def _GetClusterField(field_path):
  """Get the value of a cluster field.
71

72
73
74
75
76
  @type field_path: list of strings
  @param field_path: Names of the groups/fields to navigate to get the desired
      value, e.g. C{["Default node parameters", "oob_program"]}
  @return: The effective value of the field (the actual type depends on the
      chosen field)
77
78

  """
79
80
81
82
83
84
  assert isinstance(field_path, list)
  assert field_path
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
  for key in field_path:
    ret = ret[key]
  return ret
85
86


87
# Cluster-verify errors (date, "ERROR", then error code)
88
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
89
90
91


def _GetCVErrorCodes(cvout):
92
93
  errs = set()
  warns = set()
94
95
96
  for l in cvout.splitlines():
    m = _CVERROR_RE.match(l)
    if m:
97
98
99
100
101
102
103
      etype = m.group(1)
      ecode = m.group(2)
      if etype == "ERROR":
        errs.add(ecode)
      elif etype == "WARNING":
        warns.add(ecode)
  return (errs, warns)
104
105


106
107
108
109
110
111
112
113
114
def _CheckVerifyErrors(actual, expected, etype):
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
  if not actual.issuperset(exp_codes):
    missing = exp_codes.difference(actual)
    raise qa_error.Error("Cluster-verify didn't return these expected"
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))


def AssertClusterVerify(fail=False, errors=None, warnings=None):
115
116
117
118
119
120
121
122
  """Run cluster-verify and check the result

  @type fail: bool
  @param fail: if cluster-verify is expected to fail instead of succeeding
  @type errors: list of tuples
  @param errors: List of CV_XXX errors that are expected; if specified, all the
      errors listed must appear in cluster-verify output. A non-empty value
      implies C{fail=True}.
123
124
  @type warnings: list of tuples
  @param warnings: Same as C{errors} but for warnings.
125
126
127
128

  """
  cvcmd = "gnt-cluster verify"
  mnode = qa_config.GetMasterNode()
129
  if errors or warnings:
130
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
131
132
133
134
135
136
                             fail=(fail or errors))
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
    if errors:
      _CheckVerifyErrors(act_errs, errors, "error")
    if warnings:
      _CheckVerifyErrors(act_warns, warnings, "warning")
137
138
139
140
  else:
    AssertCommand(cvcmd, fail=fail, node=mnode)


141
142
143
144
145
146
147
148
149
150
151
152
153
154
# data for testing failures due to bad keys/values for disk parameters
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
                "drbd:nonexistent=1",
                "drbd:resync-rate=invalid",
                ]


def TestClusterInitDisk():
  """gnt-cluster init -D"""
  name = qa_config.get("name")
  for param in _FAIL_PARAMS:
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)


155
def TestClusterInit(rapi_user, rapi_secret):
156
157
158
  """gnt-cluster init"""
  master = qa_config.GetMasterNode()

159
160
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
  rapi_dir = os.path.dirname(rapi_users_path)
161

162
  # First create the RAPI credentials
163
164
165
166
167
  fh = tempfile.NamedTemporaryFile()
  try:
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
    fh.flush()

168
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
169
    try:
170
      AssertCommand(["mkdir", "-p", rapi_dir])
171
      AssertCommand(["mv", tmpru, rapi_users_path])
172
    finally:
Iustin Pop's avatar
Iustin Pop committed
173
      AssertCommand(["rm", "-f", tmpru])
174
175
176
177
  finally:
    fh.close()

  # Initialize cluster
178
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
179
180
181
  cmd = [
    "gnt-cluster", "init",
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
182
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
183
    "--enabled-disk-templates=%s" %
184
      ",".join(enabled_disk_templates),
185
    ]
186
187
188
  if constants.DT_FILE in enabled_disk_templates:
    cmd.append(
        "--file-storage-dir=%s" %
189
190
        qa_config.get("default-file-storage-dir",
                      pathutils.DEFAULT_FILE_STORAGE_DIR))
191
192
193
194
195

  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
                    "nic-count"):
    for spec_val in ("min", "max", "std"):
      spec = qa_config.get("ispec_%s_%s" %
196
                           (spec_type.replace("-", "_"), spec_val), None)
197
      if spec is not None:
198
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
Manuel Franceschini's avatar
Manuel Franceschini committed
199

200
201
  if master.secondary:
    cmd.append("--secondary-ip=%s" % master.secondary)
202

203
204
205
206
207
208
209
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    if vgname:
      cmd.append("--vg-name=%s" % vgname)
    else:
      raise qa_error.Error("Please specify a volume group if you enable"
                           " lvm-based disk templates in the QA.")
210

211
212
213
214
215
216
217
218
  master_netdev = qa_config.get("master-netdev", None)
  if master_netdev:
    cmd.append("--master-netdev=%s" % master_netdev)

  nicparams = qa_config.get("default-nicparams", None)
  if nicparams:
    cmd.append("--nic-parameters=%s" %
               ",".join(utils.FormatKeyValue(nicparams)))
219

220
221
222
223
224
225
226
227
  # Cluster value of the exclusive-storage node parameter
  e_s = qa_config.get("exclusive-storage")
  if e_s is not None:
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
  else:
    e_s = False
  qa_config.SetExclusiveStorage(e_s)

228
229
230
231
  extra_args = qa_config.get("cluster-init-args")
  if extra_args:
    cmd.extend(extra_args)

232
  cmd.append(qa_config.get("name"))
233

Iustin Pop's avatar
Iustin Pop committed
234
  AssertCommand(cmd)
235

236
  cmd = ["gnt-cluster", "modify"]
237

238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
  # hypervisor parameter modifications
  hvp = qa_config.get("hypervisor-parameters", {})
  for k, v in hvp.items():
    cmd.extend(["-H", "%s:%s" % (k, v)])
  # backend parameter modifications
  bep = qa_config.get("backend-parameters", "")
  if bep:
    cmd.extend(["-B", bep])

  if len(cmd) > 2:
    AssertCommand(cmd)

  # OS parameters
  osp = qa_config.get("os-parameters", {})
  for k, v in osp.items():
    AssertCommand(["gnt-os", "modify", "-O", v, k])

  # OS hypervisor parameters
  os_hvp = qa_config.get("os-hvp", {})
  for os_name in os_hvp:
    for hv, hvp in os_hvp[os_name].items():
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])

261

262
263
def TestClusterRename():
  """gnt-cluster rename"""
Iustin Pop's avatar
Iustin Pop committed
264
  cmd = ["gnt-cluster", "rename", "-f"]
265

Iustin Pop's avatar
Iustin Pop committed
266
267
  original_name = qa_config.get("name")
  rename_target = qa_config.get("rename", None)
268
  if rename_target is None:
269
    print qa_logging.FormatError('"rename" entry is missing')
270
271
    return

Iustin Pop's avatar
Iustin Pop committed
272
273
  for data in [
    cmd + [rename_target],
274
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
275
    cmd + [original_name],
276
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
277
278
    ]:
    AssertCommand(data)
279
280


Iustin Pop's avatar
Iustin Pop committed
281
282
def TestClusterOob():
  """out-of-band framework"""
283
284
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()

285
  AssertCommand(_CLUSTER_VERIFY)
286
287
288
289
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
                 utils.NewUUID()])

290
  AssertCommand(_CLUSTER_VERIFY, fail=True)
291

Iustin Pop's avatar
Iustin Pop committed
292
293
294
  AssertCommand(["touch", oob_path_exists])
  AssertCommand(["chmod", "0400", oob_path_exists])
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
295
296
297
298
299

  try:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program=%s" % oob_path_exists])

300
    AssertCommand(_CLUSTER_VERIFY, fail=True)
301

Iustin Pop's avatar
Iustin Pop committed
302
303
    AssertCommand(["chmod", "0500", oob_path_exists])
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
304

305
    AssertCommand(_CLUSTER_VERIFY)
306
  finally:
Iustin Pop's avatar
Iustin Pop committed
307
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
308
309
310

  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program="])
Iustin Pop's avatar
Iustin Pop committed
311
312


René Nussbaumer's avatar
René Nussbaumer committed
313
314
315
316
317
def TestClusterEpo():
  """gnt-cluster epo"""
  master = qa_config.GetMasterNode()

  # Assert that OOB is unavailable for all nodes
318
  result_output = GetCommandOutput(master.primary,
319
                                   "gnt-node list --verbose --no-headers -o"
René Nussbaumer's avatar
René Nussbaumer committed
320
321
322
323
324
325
326
327
328
329
                                   " powered")
  AssertEqual(compat.all(powered == "(unavail)"
                         for powered in result_output.splitlines()), True)

  # Conflicting
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
  # --all doesn't expect arguments
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)

  # Unless --all is given master is not allowed to be in the list
330
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
René Nussbaumer's avatar
René Nussbaumer committed
331
332
333
334
335

  # This shouldn't fail
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])

  # All instances should have been stopped now
336
  result_output = GetCommandOutput(master.primary,
337
                                   "gnt-instance list --no-headers -o status")
338
339
  # ERROR_down because the instance is stopped but not recorded as such
  AssertEqual(compat.all(status == "ERROR_down"
René Nussbaumer's avatar
René Nussbaumer committed
340
341
342
343
344
345
                         for status in result_output.splitlines()), True)

  # Now start everything again
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])

  # All instances should have been started now
346
  result_output = GetCommandOutput(master.primary,
347
                                   "gnt-instance list --no-headers -o status")
René Nussbaumer's avatar
René Nussbaumer committed
348
349
350
351
  AssertEqual(compat.all(status == "running"
                         for status in result_output.splitlines()), True)


Iustin Pop's avatar
Iustin Pop committed
352
353
def TestClusterVerify():
  """gnt-cluster verify"""
354
  AssertCommand(_CLUSTER_VERIFY)
355
  AssertCommand(["gnt-cluster", "verify-disks"])
356

357

358
359
def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):
  """gnt-cluster verify-disks with broken DRBD"""
360
  qa_daemon.TestPauseWatcher()
361

362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
  try:
    info = qa_instance.GetInstanceInfo(instance.name)
    snode = inst_nodes[1]
    for idx, minor in enumerate(info["drbd-minors"][snode.primary]):
      if idx % 2 == 0:
        break_drbd_cmd = \
          "(drbdsetup %d down >/dev/null 2>&1;" \
          " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \
          (minor, minor)
      else:
        break_drbd_cmd = \
          "(drbdsetup %d detach >/dev/null 2>&1;" \
          " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \
          (minor, minor)
      AssertCommand(break_drbd_cmd, node=snode)

    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
                                     "gnt-cluster verify-disks")
    activation_msg = "Activating disks for instance '%s'" % instance.name
    if activation_msg not in verify_output:
      raise qa_error.Error("gnt-cluster verify-disks did not activate broken"
                           " DRBD disks:\n%s" % verify_output)

    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
                                     "gnt-cluster verify-disks")
    if activation_msg in verify_output:
      raise qa_error.Error("gnt-cluster verify-disks wants to activate broken"
                           " DRBD disks on second attempt:\n%s" % verify_output)

    AssertCommand(_CLUSTER_VERIFY)
  finally:
    qa_daemon.TestResumeWatcher()
394
395


396
397
def TestJobqueue():
  """gnt-debug test-jobqueue"""
Iustin Pop's avatar
Iustin Pop committed
398
  AssertCommand(["gnt-debug", "test-jobqueue"])
399
400


401
402
403
404
405
def TestDelay(node):
  """gnt-debug delay"""
  AssertCommand(["gnt-debug", "delay", "1"])
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
  AssertCommand(["gnt-debug", "delay", "--no-master",
406
                 "-n", node.primary, "1"])
407
408


409
410
def TestClusterReservedLvs():
  """gnt-cluster reserved lvs"""
411
412
413
  # if no lvm-based templates are supported, skip the test
  if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG):
    return
414
415
416
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
  lvname = _QA_LV_PREFIX + "test"
  lvfullname = "/".join([vgname, lvname])
Iustin Pop's avatar
Iustin Pop committed
417
  for fail, cmd in [
418
    (False, _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
419
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
420
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
421
    (True, _CLUSTER_VERIFY),
422
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
423
             "%s,.*/other-test" % lvfullname]),
424
    (False, _CLUSTER_VERIFY),
425
426
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
             ".*/%s.*" % _QA_LV_PREFIX]),
427
    (False, _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
428
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
429
    (True, _CLUSTER_VERIFY),
430
    (False, ["lvremove", "-f", lvfullname]),
431
    (False, _CLUSTER_VERIFY),
432
    ]:
Iustin Pop's avatar
Iustin Pop committed
433
    AssertCommand(cmd, fail=fail)
434

435

436
437
438
439
440
def TestClusterModifyEmpty():
  """gnt-cluster modify"""
  AssertCommand(["gnt-cluster", "modify"], fail=True)


441
442
443
444
445
446
def TestClusterModifyDisk():
  """gnt-cluster modify -D"""
  for param in _FAIL_PARAMS:
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)


447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
def _GetOtherEnabledDiskTemplate(undesired_disk_templates,
                                 enabled_disk_templates):
  """Returns one template that is not in the undesired set.

  @type undesired_disk_templates: list of string
  @param undesired_disk_templates: a list of disk templates that we want to
      exclude when drawing one disk template from the list of enabled
      disk templates
  @type enabled_disk_templates: list of string
  @param enabled_disk_templates: list of enabled disk templates (in QA)

  """
  desired_templates = list(set(enabled_disk_templates)
                                - set(undesired_disk_templates))
  if desired_templates:
    template = desired_templates[0]
  else:
    # If no desired disk template is available for QA, choose 'diskless' and
    # hope for the best.
    template = constants.ST_DISKLESS

  return template


def TestClusterModifyFileBasedStorageDir(
    file_disk_template, dir_config_key, default_dir, option_name):
  """Tests gnt-cluster modify wrt to file-based directory options.

  @type file_disk_template: string
  @param file_disk_template: file-based disk template
  @type dir_config_key: string
  @param dir_config_key: key for the QA config to retrieve the default
     directory value
  @type default_dir: string
  @param default_dir: default directory, if the QA config does not specify
     it
  @type option_name: string
  @param option_name: name of the option of 'gnt-cluster modify' to
     change the directory

  """
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
  assert file_disk_template in [constants.DT_FILE, constants.DT_SHARED_FILE]
  if not qa_config.IsTemplateSupported(file_disk_template):
    return

  # Get some non-file-based disk template to disable file storage
  other_disk_template = _GetOtherEnabledDiskTemplate(
      utils.storage.GetDiskTemplatesOfStorageType(constants.ST_FILE),
      enabled_disk_templates)

  file_storage_dir = qa_config.get(dir_config_key, default_dir)
  invalid_file_storage_dir = "/boot/"

  for fail, cmd in [
    (False, ["gnt-cluster", "modify",
503
504
             "--enabled-disk-templates=%s" % file_disk_template,
             "--ipolicy-disk-templates=%s" % file_disk_template]),
505
    (False, ["gnt-cluster", "modify",
506
             "--%s=%s" % (option_name, file_storage_dir)]),
507
    (False, ["gnt-cluster", "modify",
508
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
509
510
511
512
513
514
515
516
    # file storage dir is set to an inacceptable path, thus verify
    # should fail
    (True, ["gnt-cluster", "verify"]),
    # unsetting the storage dir while file storage is enabled
    # should fail
    (True, ["gnt-cluster", "modify",
            "--%s=" % option_name]),
    (False, ["gnt-cluster", "modify",
517
             "--%s=%s" % (option_name, file_storage_dir)]),
518
    (False, ["gnt-cluster", "modify",
519
520
             "--enabled-disk-templates=%s" % other_disk_template,
             "--ipolicy-disk-templates=%s" % other_disk_template]),
521
    (False, ["gnt-cluster", "modify",
522
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
523
524
525
526
527
528
    # file storage is set to an inacceptable path, but file storage
    # is disabled, thus verify should not fail
    (False, ["gnt-cluster", "verify"]),
    # unsetting the file storage dir while file storage is not enabled
    # should be fine
    (False, ["gnt-cluster", "modify",
529
             "--%s=" % option_name]),
530
531
    # resetting everything to sane values
    (False, ["gnt-cluster", "modify",
532
533
534
             "--%s=%s" % (option_name, file_storage_dir),
             "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
             "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
    ]:
    AssertCommand(cmd, fail=fail)


def TestClusterModifyFileStorageDir():
  """gnt-cluster modify --file-storage-dir=..."""
  TestClusterModifyFileBasedStorageDir(
      constants.DT_FILE, "default-file-storage-dir",
      pathutils.DEFAULT_FILE_STORAGE_DIR,
      "file-storage-dir")


def TestClusterModifySharedFileStorageDir():
  """gnt-cluster modify --shared-file-storage-dir=..."""
  TestClusterModifyFileBasedStorageDir(
      constants.DT_SHARED_FILE, "default-shared-file-storage-dir",
      pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR,
      "shared-file-storage-dir")


555
556
def TestClusterModifyDiskTemplates():
  """gnt-cluster modify --enabled-disk-templates=..."""
557
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
558
  default_disk_template = qa_config.GetDefaultDiskTemplate()
559
560
561

  _TestClusterModifyDiskTemplatesArguments(default_disk_template,
                                           enabled_disk_templates)
562
  _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates)
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584

  _RestoreEnabledDiskTemplates()
  nodes = qa_config.AcquireManyNodes(2)

  instance_template = enabled_disk_templates[0]
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)

  _TestClusterModifyUnusedDiskTemplate(instance_template)
  _TestClusterModifyUsedDiskTemplate(instance_template,
                                     enabled_disk_templates)

  qa_instance.TestInstanceRemove(instance)
  _RestoreEnabledDiskTemplates()


def _RestoreEnabledDiskTemplates():
  """Sets the list of enabled disk templates back to the list of enabled disk
     templates from the QA configuration. This can be used to make sure that
     the tests that modify the list of disk templates do not interfere with
     other tests.

  """
585
586
587
588
589
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
  cmd = ["gnt-cluster", "modify",
         "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
         "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates),
         ]
590
591
592
593
594
595

  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    cmd.append("--vg-name=%s" % vgname)

  AssertCommand(cmd, fail=False)
596
597
598
599
600
601
602
603
604


def _TestClusterModifyDiskTemplatesArguments(default_disk_template,
                                             enabled_disk_templates):
  """Tests argument handling of 'gnt-cluster modify' with respect to
     the parameter '--enabled-disk-templates'. This test is independent
     of instances.

  """
605
  _RestoreEnabledDiskTemplates()
606

607
  # bogus templates
608
  AssertCommand(["gnt-cluster", "modify",
609
                 "--enabled-disk-templates=pinkbunny"],
610
                fail=True)
611

612
613
614
  # duplicate entries do no harm
  AssertCommand(
    ["gnt-cluster", "modify",
615
     "--enabled-disk-templates=%s,%s" %
616
617
      (default_disk_template, default_disk_template),
     "--ipolicy-disk-templates=%s" % default_disk_template],
618
    fail=False)
619

620
  if constants.DT_DRBD8 in enabled_disk_templates:
621
622
623
624
625
626
627
628
    # interaction with --drbd-usermode-helper option
    drbd_usermode_helper = qa_config.get("drbd-usermode-helper", None)
    if not drbd_usermode_helper:
      drbd_usermode_helper = "/bin/true"
    # specifying a helper when drbd gets disabled is ok. Note that drbd still
    # has to be installed on the nodes in this case
    AssertCommand(["gnt-cluster", "modify",
                   "--drbd-usermode-helper=%s" % drbd_usermode_helper,
629
630
                   "--enabled-disk-templates=%s" % constants.DT_DISKLESS,
                   "--ipolicy-disk-templates=%s" % constants.DT_DISKLESS],
631
632
                   fail=False)
    # specifying a helper when drbd is re-enabled
633
634
635
    AssertCommand(["gnt-cluster", "modify",
                   "--drbd-usermode-helper=%s" % drbd_usermode_helper,
                   "--enabled-disk-templates=%s" %
636
637
                     ",".join(enabled_disk_templates),
                   "--ipolicy-disk-templates=%s" %
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
                     ",".join(enabled_disk_templates)],
                  fail=False)


def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates):
  """Tests argument handling of 'gnt-cluster modify' with respect to
     the parameter '--enabled-disk-templates' and '--vg-name'. This test is
     independent of instances.

  """
  if not utils.IsLvmEnabled(enabled_disk_templates):
    # These tests only make sense if lvm is enabled for QA
    return

  # determine an LVM and a non-LVM disk template for the tests
653
654
  non_lvm_template = _GetOtherEnabledDiskTemplate(utils.GetLvmDiskTemplates(),
                                                  enabled_disk_templates)
655
656
657
658
659
660
661
662
663
  lvm_template = list(set(enabled_disk_templates)
                      .intersection(set(utils.GetLvmDiskTemplates())))[0]

  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)

  # Clean start: unset volume group name, disable lvm storage
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % non_lvm_template,
664
     "--ipolicy-disk-templates=%s" % non_lvm_template,
665
666
667
668
669
670
     "--vg-name="],
    fail=False)

  # Try to enable lvm, when no volume group is given
  AssertCommand(
    ["gnt-cluster", "modify",
671
672
     "--enabled-disk-templates=%s" % lvm_template,
     "--ipolicy-disk-templates=%s" % lvm_template],
673
674
675
676
677
678
679
680
681
    fail=True)

  # Set volume group, with lvm still disabled: just a warning
  AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False)

  # Try unsetting vg name and enabling lvm at the same time
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % lvm_template,
682
     "--ipolicy-disk-templates=%s" % lvm_template,
683
684
685
686
687
688
     "--vg-name="],
    fail=True)

  # Enable lvm with vg name present
  AssertCommand(
    ["gnt-cluster", "modify",
689
690
     "--enabled-disk-templates=%s" % lvm_template,
     "--ipolicy-disk-templates=%s" % lvm_template],
691
692
693
694
695
696
697
    fail=False)

  # Try unsetting vg name with lvm still enabled
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True)

  # Disable lvm with vg name still set
  AssertCommand(
698
699
700
701
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % non_lvm_template,
     "--ipolicy-disk-templates=%s" % non_lvm_template,
     ],
702
703
704
705
706
707
708
709
710
    fail=False)

  # Try unsetting vg name with lvm disabled
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False)

  # Set vg name and enable lvm at the same time
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % lvm_template,
711
     "--ipolicy-disk-templates=%s" % lvm_template,
712
713
714
715
716
717
718
     "--vg-name=%s" % vgname],
    fail=False)

  # Unset vg name and disable lvm at the same time
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % non_lvm_template,
719
     "--ipolicy-disk-templates=%s" % non_lvm_template,
720
721
722
723
724
     "--vg-name="],
    fail=False)

  _RestoreEnabledDiskTemplates()

725
726
727
728
729
730
731
732
733
734
735
736
737

def _TestClusterModifyUsedDiskTemplate(instance_template,
                                       enabled_disk_templates):
  """Tests that disk templates that are currently in use by instances cannot
     be disabled on the cluster.

  """
  # If the list of enabled disk templates contains only one template
  # we need to add some other templates, because the list of enabled disk
  # templates can only be set to a non-empty list.
  new_disk_templates = list(set(enabled_disk_templates)
                              - set([instance_template]))
  if not new_disk_templates:
738
    new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK])
739
740
741
                                - set([instance_template]))
  AssertCommand(
    ["gnt-cluster", "modify",
742
743
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
744
745
746
747
748
749
    fail=True)


def _TestClusterModifyUnusedDiskTemplate(instance_template):
  """Tests that unused disk templates can be disabled safely."""
  all_disk_templates = constants.DISK_TEMPLATES
750
751
752
753
  if not utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
    all_disk_templates = list(set(all_disk_templates) -
                              set(utils.GetLvmDiskTemplates()))

754
755
  AssertCommand(
    ["gnt-cluster", "modify",
756
757
     "--enabled-disk-templates=%s" % ",".join(all_disk_templates),
     "--ipolicy-disk-templates=%s" % ",".join(all_disk_templates)],
758
759
760
761
    fail=False)
  new_disk_templates = [instance_template]
  AssertCommand(
    ["gnt-cluster", "modify",
762
763
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
764
    fail=False)
765
766


767
768
def TestClusterModifyBe():
  """gnt-cluster modify -B"""
Iustin Pop's avatar
Iustin Pop committed
769
  for fail, cmd in [
770
771
772
773
774
775
776
777
778
779
780
781
    # max/min mem
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
782
    # vcpus
Iustin Pop's avatar
Iustin Pop committed
783
784
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
785
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
Iustin Pop's avatar
Iustin Pop committed
786
787
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
788
    # auto_balance
Iustin Pop's avatar
Iustin Pop committed
789
790
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
791
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
Iustin Pop's avatar
Iustin Pop committed
792
793
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
794
    ]:
Iustin Pop's avatar
Iustin Pop committed
795
    AssertCommand(cmd, fail=fail)
796

797
798
799
800
  # redo the original-requested BE parameters, if any
  bep = qa_config.get("backend-parameters", "")
  if bep:
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
801

Andrea Spadaccini's avatar
Andrea Spadaccini committed
802

803
804
805
806
807
808
def _GetClusterIPolicy():
  """Return the run-time values of the cluster-level instance policy.

  @rtype: tuple
  @return: (policy, specs), where:
      - policy is a dictionary of the policy values, instance specs excluded
809
810
      - specs is a dictionary containing only the specs, using the internal
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
811
812

  """
813
814
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
  policy = info["Instance policy - limits for instances"]
815
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
816

817
  # Sanity checks
818
819
  assert "minmax" in ret_specs and "std" in ret_specs
  assert len(ret_specs["minmax"]) > 0
820
821
  assert len(ret_policy) > 0
  return (ret_policy, ret_specs)
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852


def TestClusterModifyIPolicy():
  """gnt-cluster modify --ipolicy-*"""
  basecmd = ["gnt-cluster", "modify"]
  (old_policy, old_specs) = _GetClusterIPolicy()
  for par in ["vcpu-ratio", "spindle-ratio"]:
    curr_val = float(old_policy[par])
    test_values = [
      (True, 1.0),
      (True, 1.5),
      (True, 2),
      (False, "a"),
      # Restore the old value
      (True, curr_val),
      ]
    for (good, val) in test_values:
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
      AssertCommand(cmd, fail=not good)
      if good:
        curr_val = val
      # Check the affected parameter
      (eff_policy, eff_specs) = _GetClusterIPolicy()
      AssertEqual(float(eff_policy[par]), curr_val)
      # Check everything else
      AssertEqual(eff_specs, old_specs)
      for p in eff_policy.keys():
        if p == par:
          continue
        AssertEqual(eff_policy[p], old_policy[p])

853
854
855
856
857
  # Allowing disk templates via ipolicy requires them to be
  # enabled on the cluster.
  if not (qa_config.IsTemplateSupported(constants.DT_PLAIN)
          and qa_config.IsTemplateSupported(constants.DT_DRBD8)):
    return
858
859
  # Disk templates are treated slightly differently
  par = "disk-templates"
860
  disp_str = "allowed disk templates"
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
  curr_val = old_policy[disp_str]
  test_values = [
    (True, constants.DT_PLAIN),
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
    (False, "thisisnotadisktemplate"),
    (False, ""),
    # Restore the old value
    (True, curr_val.replace(" ", "")),
    ]
  for (good, val) in test_values:
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
    AssertCommand(cmd, fail=not good)
    if good:
      curr_val = val
    # Check the affected parameter
    (eff_policy, eff_specs) = _GetClusterIPolicy()
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
    # Check everything else
    AssertEqual(eff_specs, old_specs)
    for p in eff_policy.keys():
      if p == disp_str:
        continue
      AssertEqual(eff_policy[p], old_policy[p])


886
887
def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
                         old_values=None):
888
889
  """Change instance specs.

890
891
892
893
894
895
  At most one of new_specs or diff_specs can be specified.

  @type new_specs: dict
  @param new_specs: new complete specs, in the same format returned by
      L{_GetClusterIPolicy}
  @type diff_specs: dict
896
897
898
  @param diff_specs: partial specs, it can be an incomplete specifications, but
      if min/max specs are specified, their number must match the number of the
      existing specs
899
900
901
902
  @type fail: bool
  @param fail: if the change is expected to fail
  @type old_values: tuple
  @param old_values: (old_policy, old_specs), as returned by
903
      L{_GetClusterIPolicy}
904
905
906
  @return: same as L{_GetClusterIPolicy}

  """
907
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
908
909
910
911
  return qa_utils.TestSetISpecs(
    new_specs=new_specs, diff_specs=diff_specs,
    get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
    fail=fail, old_values=old_values)
912
913
914
915


def TestClusterModifyISpecs():
  """gnt-cluster modify --specs-*"""
916
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
917
  (cur_policy, cur_specs) = _GetClusterIPolicy()
918
919
  # This test assumes that there is only one min/max bound
  assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
  for par in params:
    test_values = [
      (True, 0, 4, 12),
      (True, 4, 4, 12),
      (True, 4, 12, 12),
      (True, 4, 4, 4),
      (False, 4, 0, 12),
      (False, 4, 16, 12),
      (False, 4, 4, 0),
      (False, 12, 4, 4),
      (False, 12, 4, 0),
      (False, "a", 4, 12),
      (False, 0, "a", 12),
      (False, 0, 4, "a"),
      # This is to restore the old values
      (True,
936
937
938
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
       cur_specs[constants.ISPECS_STD][par],
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
939
940
      ]
    for (good, mn, st, mx) in test_values:
941
      new_vals = {
942
943
944
945
946
        constants.ISPECS_MINMAX: [{
          constants.ISPECS_MIN: {par: mn},
          constants.ISPECS_MAX: {par: mx}
          }],
        constants.ISPECS_STD: {par: st}
947
        }
948
949
      cur_state = (cur_policy, cur_specs)
      # We update cur_specs, as we've copied the values to restore already
950
951
      (cur_policy, cur_specs) = TestClusterSetISpecs(
        diff_specs=new_vals, fail=not good, old_values=cur_state)
952

953
954
955
956
957
958
959
960
961
962
963
964
965
966
    # Get the ipolicy command
    mnode = qa_config.GetMasterNode()
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
    modcmd = ["gnt-cluster", "modify"]
    opts = initcmd.split()
    assert opts[0:2] == ["gnt-cluster", "init"]
    for k in range(2, len(opts) - 1):
      if opts[k].startswith("--ipolicy-"):
        assert k + 2 <= len(opts)
        modcmd.extend(opts[k:k + 2])
    # Re-apply the ipolicy (this should be a no-op)
    AssertCommand(modcmd)
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
    AssertEqual(initcmd, new_initcmd)
967
968


969
970
def TestClusterInfo():
  """gnt-cluster info"""
Iustin Pop's avatar
Iustin Pop committed
971
  AssertCommand(["gnt-cluster", "info"])
Michael Hanselmann's avatar
Michael Hanselmann committed
972
973


Iustin Pop's avatar
Iustin Pop committed
974
975
976
977
978
def TestClusterRedistConf():
  """gnt-cluster redist-conf"""
  AssertCommand(["gnt-cluster", "redist-conf"])


Michael Hanselmann's avatar
Michael Hanselmann committed
979
980
def TestClusterGetmaster():
  """gnt-cluster getmaster"""
Iustin Pop's avatar
Iustin Pop committed
981
  AssertCommand(["gnt-cluster", "getmaster"])
Michael Hanselmann's avatar
Michael Hanselmann committed
982
983
984
985


def TestClusterVersion():
  """gnt-cluster version"""
Iustin Pop's avatar
Iustin Pop committed
986
  AssertCommand(["gnt-cluster", "version"])
987
988


989
990
991
992
993
994
def TestClusterRenewCrypto():
  """gnt-cluster renew-crypto"""
  master = qa_config.GetMasterNode()

  # Conflicting options
  cmd = ["gnt-cluster", "renew-crypto", "--force",
Michael Hanselmann's avatar
Michael Hanselmann committed
995
996
997
998
999
1000
         "--new-cluster-certificate", "--new-confd-hmac-key"]
  conflicting = [
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
    ]
  for i in conflicting:
Andrea Spadaccini's avatar
Andrea Spadaccini committed
1001
    AssertCommand(cmd + i, fail=True)
1002
1003
1004
1005

  # Invalid RAPI certificate
  cmd = ["gnt-cluster", "renew-crypto", "--force",
         "--rapi-certificate=/dev/null"]
Iustin Pop's avatar
Iustin Pop committed
1006
  AssertCommand(cmd, fail=True)
1007

1008
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
1009
                                         pathutils.RAPI_CERT_FILE)
1010
1011
1012
  try:
    # Custom RAPI certificate
    fh = tempfile.NamedTemporaryFile()
1013

1014
1015
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
1016

Michael Hanselmann's avatar
Michael Hanselmann committed
1017
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
1018

1019
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
1020
    try:
Iustin Pop's avatar
Iustin Pop committed
1021
1022
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--rapi-certificate=%s" % tmpcert])
1023
    finally:
Iustin Pop's avatar
Iustin Pop committed
1024
      AssertCommand(["rm", "-f", tmpcert])
1025

Michael Hanselmann's avatar
Michael Hanselmann committed
1026
1027
1028
1029
1030
1031
    # Custom cluster domain secret
    cds_fh = tempfile.NamedTemporaryFile()
    cds_fh.write(utils.GenerateSecret())
    cds_fh.write("\n")
    cds_fh.flush()

1032
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
Michael Hanselmann's avatar
Michael Hanselmann committed
1033
    try:
Iustin Pop's avatar
Iustin Pop committed
1034
1035
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--cluster-domain-secret=%s" % tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
1036
    finally:
Iustin Pop's avatar
Iustin Pop committed
1037
      AssertCommand(["rm", "-f", tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
1038

1039
    # Normal case
Iustin Pop's avatar
Iustin Pop committed
1040
1041
1042
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--new-cluster-certificate", "--new-confd-hmac-key",
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
Michael Hanselmann's avatar
Michael Hanselmann committed
1043

1044
    # Restore RAPI certificate
Iustin Pop's avatar
Iustin Pop committed
1045
1046
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--rapi-certificate=%s" % rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
1047
  finally:
Iustin Pop's avatar
Iustin Pop committed
1048
    AssertCommand(["rm", "-f", rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
1049

1050

1051
1052
1053
1054
def TestClusterBurnin():
  """Burnin"""
  master = qa_config.GetMasterNode()

Iustin Pop's avatar
Iustin Pop committed
1055
  options = qa_config.get("options", {})
1056
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
Iustin Pop's avatar
Iustin Pop committed
1057
1058
1059
1060
  parallel = options.get("burnin-in-parallel", False)
  check_inst = options.get("burnin-check-instances", False)
  do_rename = options.get("burnin-rename", "")
  do_reboot = options.get("burnin-reboot", True)
1061
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
1062

1063
1064
1065
  # Get as many instances as we need
  instances = []
  try:
1066
    try:
Iustin Pop's avatar
Iustin Pop committed
1067
      num = qa_config.get("options", {}).get("burnin-instances", 1)
1068
      for _ in range(0, num):
1069
1070
1071
        instances.append(qa_config.AcquireInstance())
    except qa_error.OutOfInstancesError:
      print "Not enough instances, continuing anyway."
1072

1073
1074
    if len(instances) < 1:
      raise qa_error.Error("Burnin needs at least one instance")
1075

1076
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
1077
    try:
1078
      disks = qa_config.GetDiskOptions()
1079
      # Run burnin
1080
      cmd = [script,
Iustin Pop's avatar
Iustin Pop committed
1081
             "--os=%s" % qa_config.get("os"),
1082
1083
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
1084
1085
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
Iustin Pop's avatar
Iustin Pop committed
1086
             "--disk-template=%s" % disk_template]
1087
      if parallel:
Iustin Pop's avatar
Iustin Pop committed
1088
1089
        cmd.append("--parallel")
        cmd.append("--early-release")
1090
      if check_inst:
Iustin Pop's avatar
Iustin Pop committed
1091
        cmd.append("--http-check")
Iustin Pop's avatar
Iustin Pop committed
1092
      if do_rename:
Iustin Pop's avatar
Iustin Pop committed
1093
        cmd.append("--rename=%s" % do_rename)
1094
      if not do_reboot:
Iustin Pop's avatar
Iustin Pop committed
1095
        cmd.append("--no-reboot")
1096
      else:
Iustin Pop's avatar
Iustin Pop committed
1097
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
1098
      cmd += [inst.name for inst in instances]
Iustin Pop's avatar
Iustin Pop committed
1099
      AssertCommand(cmd)
1100
    finally:
Iustin Pop's avatar
Iustin Pop committed
1101
1102
      AssertCommand(["rm", "-f", script])

1103
1104
  finally:
    for inst in instances:
1105
      inst.Release()
1106
1107
1108


def TestClusterMasterFailover():
1109
  """gnt-cluster master-failover"""
1110
1111
1112
  master = qa_config.GetMasterNode()
  failovermaster = qa_config.AcquireNode(exclude=master)

Iustin Pop's avatar
Iustin Pop committed
1113
  cmd = ["gnt-cluster", "master-failover"]
1114
  node_list_cmd = ["gnt-node", "list"]
Iustin Pop's avatar
Iustin Pop committed
1115
1116
  try:
    AssertCommand(cmd, node=failovermaster)
1117
    AssertCommand(node_list_cmd, node=failovermaster)
1118
    # Back to original master node
Iustin Pop's avatar
Iustin Pop committed
1119
    AssertCommand(cmd, node=master)
1120
    AssertCommand(node_list_cmd, node=master)
1121
  finally:
1122
    failovermaster.Release()
1123
1124


1125
1126
1127
1128
1129
1130