qa_cluster.py 44.9 KB
Newer Older
1
2
3
#
#

4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Cluster related QA tests.

"""

26
import re
27
import tempfile
28
import os.path
29

30
from ganeti import _constants
31
from ganeti import constants
René Nussbaumer's avatar
René Nussbaumer committed
32
from ganeti import compat
33
from ganeti import utils
34
from ganeti import pathutils
35
36

import qa_config
37
import qa_daemon
38
import qa_error
39
import qa_instance
40
41
import qa_logging
import qa_utils
42

René Nussbaumer's avatar
René Nussbaumer committed
43
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
44
45


46
47
48
# Prefix for LVM volumes created by QA code during tests
_QA_LV_PREFIX = "qa-"

49
50
51
#: cluster verify command
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]

Andrea Spadaccini's avatar
Andrea Spadaccini committed
52

53
54
55
56
def _RemoveFileFromAllNodes(filename):
  """Removes a file from all nodes.

  """
Iustin Pop's avatar
Iustin Pop committed
57
58
  for node in qa_config.get("nodes"):
    AssertCommand(["rm", "-f", filename], node=node)
59
60
61
62
63
64
65


def _CheckFileOnAllNodes(filename, content):
  """Verifies the content of the given file on all nodes.

  """
  cmd = utils.ShellQuoteArgs(["cat", filename])
Iustin Pop's avatar
Iustin Pop committed
66
  for node in qa_config.get("nodes"):
67
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
68
69


70
71
def _GetClusterField(field_path):
  """Get the value of a cluster field.
72

73
74
75
76
77
  @type field_path: list of strings
  @param field_path: Names of the groups/fields to navigate to get the desired
      value, e.g. C{["Default node parameters", "oob_program"]}
  @return: The effective value of the field (the actual type depends on the
      chosen field)
78
79

  """
80
81
82
83
84
85
  assert isinstance(field_path, list)
  assert field_path
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
  for key in field_path:
    ret = ret[key]
  return ret
86
87


88
# Cluster-verify errors (date, "ERROR", then error code)
89
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
90
91
92


def _GetCVErrorCodes(cvout):
93
94
  errs = set()
  warns = set()
95
96
97
  for l in cvout.splitlines():
    m = _CVERROR_RE.match(l)
    if m:
98
99
100
101
102
103
104
      etype = m.group(1)
      ecode = m.group(2)
      if etype == "ERROR":
        errs.add(ecode)
      elif etype == "WARNING":
        warns.add(ecode)
  return (errs, warns)
105
106


107
108
109
110
111
112
113
114
def _CheckVerifyErrors(actual, expected, etype):
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
  if not actual.issuperset(exp_codes):
    missing = exp_codes.difference(actual)
    raise qa_error.Error("Cluster-verify didn't return these expected"
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))


115
116
117
118
119
120
121
122
123
124
125
def _CheckVerifyNoWarnings(actual, expected):
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
  excess = actual.intersection(exp_codes)
  if excess:
    raise qa_error.Error("Cluster-verify returned these warnings:"
                         " %s" % (utils.CommaJoin(excess)))


def AssertClusterVerify(fail=False, errors=None,
                        warnings=None, no_warnings=None):
  """Run cluster-verify and check the result, ignoring warnings by default.
126
127

  @type fail: bool
128
  @param fail: if cluster-verify is expected to fail instead of succeeding.
129
130
131
132
  @type errors: list of tuples
  @param errors: List of CV_XXX errors that are expected; if specified, all the
      errors listed must appear in cluster-verify output. A non-empty value
      implies C{fail=True}.
133
  @type warnings: list of tuples
134
135
136
137
  @param warnings: List of CV_XXX warnings that are expected to be raised; if
      specified, all the errors listed must appear in cluster-verify output.
  @type no_warnings: list of tuples
  @param no_warnings: List of CV_XXX warnings that we expect NOT to be raised.
138
139
140
  """
  cvcmd = "gnt-cluster verify"
  mnode = qa_config.GetMasterNode()
141
  if errors or warnings or no_warnings:
142
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
143
                             fail=(fail or errors))
144
    print cvout
145
146
147
148
149
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
    if errors:
      _CheckVerifyErrors(act_errs, errors, "error")
    if warnings:
      _CheckVerifyErrors(act_warns, warnings, "warning")
150
151
152
    if no_warnings:
      _CheckVerifyNoWarnings(act_warns, no_warnings)

153
154
155
156
  else:
    AssertCommand(cvcmd, fail=fail, node=mnode)


157
158
159
160
161
162
163
164
165
166
167
168
169
170
# data for testing failures due to bad keys/values for disk parameters
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
                "drbd:nonexistent=1",
                "drbd:resync-rate=invalid",
                ]


def TestClusterInitDisk():
  """gnt-cluster init -D"""
  name = qa_config.get("name")
  for param in _FAIL_PARAMS:
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)


171
def TestClusterInit(rapi_user, rapi_secret):
172
173
174
  """gnt-cluster init"""
  master = qa_config.GetMasterNode()

175
176
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
  rapi_dir = os.path.dirname(rapi_users_path)
177

178
  # First create the RAPI credentials
179
180
181
182
183
  fh = tempfile.NamedTemporaryFile()
  try:
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
    fh.flush()

184
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
185
    try:
186
      AssertCommand(["mkdir", "-p", rapi_dir])
187
      AssertCommand(["mv", tmpru, rapi_users_path])
188
    finally:
Iustin Pop's avatar
Iustin Pop committed
189
      AssertCommand(["rm", "-f", tmpru])
190
191
192
193
  finally:
    fh.close()

  # Initialize cluster
194
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
195
196
197
  cmd = [
    "gnt-cluster", "init",
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
198
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
199
    "--enabled-disk-templates=%s" %
200
      ",".join(enabled_disk_templates),
201
    ]
202
203
204
  if constants.DT_FILE in enabled_disk_templates:
    cmd.append(
        "--file-storage-dir=%s" %
205
206
        qa_config.get("default-file-storage-dir",
                      pathutils.DEFAULT_FILE_STORAGE_DIR))
207
208
209

  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
                    "nic-count"):
210
    spec_values = []
211
212
    for spec_val in ("min", "max", "std"):
      spec = qa_config.get("ispec_%s_%s" %
213
                           (spec_type.replace("-", "_"), spec_val), None)
214
      if spec is not None:
215
216
217
        spec_values.append("%s=%d" % (spec_val, spec))
    if spec_values:
      cmd.append("--specs-%s=%s" % (spec_type, ",".join(spec_values)))
Manuel Franceschini's avatar
Manuel Franceschini committed
218

219
220
  if master.secondary:
    cmd.append("--secondary-ip=%s" % master.secondary)
221

222
223
224
225
226
227
228
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    if vgname:
      cmd.append("--vg-name=%s" % vgname)
    else:
      raise qa_error.Error("Please specify a volume group if you enable"
                           " lvm-based disk templates in the QA.")
229

230
231
232
233
234
235
236
237
  master_netdev = qa_config.get("master-netdev", None)
  if master_netdev:
    cmd.append("--master-netdev=%s" % master_netdev)

  nicparams = qa_config.get("default-nicparams", None)
  if nicparams:
    cmd.append("--nic-parameters=%s" %
               ",".join(utils.FormatKeyValue(nicparams)))
238

239
240
241
242
243
244
245
246
  # Cluster value of the exclusive-storage node parameter
  e_s = qa_config.get("exclusive-storage")
  if e_s is not None:
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
  else:
    e_s = False
  qa_config.SetExclusiveStorage(e_s)

247
  extra_args = qa_config.get("cluster-init-args")
248

249
  if extra_args:
250
251
252
253
254
    # This option was removed in 2.10, but in order to not break QA of older
    # branches we remove it from the extra_args if it is in there.
    opt_drbd_storage = "--no-drbd-storage"
    if opt_drbd_storage in extra_args:
      extra_args.remove(opt_drbd_storage)
255
256
    cmd.extend(extra_args)

257
  cmd.append(qa_config.get("name"))
258

Iustin Pop's avatar
Iustin Pop committed
259
  AssertCommand(cmd)
260

261
  cmd = ["gnt-cluster", "modify"]
262

263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
  # hypervisor parameter modifications
  hvp = qa_config.get("hypervisor-parameters", {})
  for k, v in hvp.items():
    cmd.extend(["-H", "%s:%s" % (k, v)])
  # backend parameter modifications
  bep = qa_config.get("backend-parameters", "")
  if bep:
    cmd.extend(["-B", bep])

  if len(cmd) > 2:
    AssertCommand(cmd)

  # OS parameters
  osp = qa_config.get("os-parameters", {})
  for k, v in osp.items():
    AssertCommand(["gnt-os", "modify", "-O", v, k])

  # OS hypervisor parameters
  os_hvp = qa_config.get("os-hvp", {})
  for os_name in os_hvp:
    for hv, hvp in os_hvp[os_name].items():
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])

286

287
288
def TestClusterRename():
  """gnt-cluster rename"""
Iustin Pop's avatar
Iustin Pop committed
289
  cmd = ["gnt-cluster", "rename", "-f"]
290

Iustin Pop's avatar
Iustin Pop committed
291
292
  original_name = qa_config.get("name")
  rename_target = qa_config.get("rename", None)
293
  if rename_target is None:
294
    print qa_logging.FormatError('"rename" entry is missing')
295
296
    return

Iustin Pop's avatar
Iustin Pop committed
297
298
  for data in [
    cmd + [rename_target],
299
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
300
    cmd + [original_name],
301
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
302
303
    ]:
    AssertCommand(data)
304
305


Iustin Pop's avatar
Iustin Pop committed
306
307
def TestClusterOob():
  """out-of-band framework"""
308
309
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()

310
  AssertCommand(_CLUSTER_VERIFY)
311
312
313
314
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
                 utils.NewUUID()])

315
  AssertCommand(_CLUSTER_VERIFY, fail=True)
316

Iustin Pop's avatar
Iustin Pop committed
317
318
319
  AssertCommand(["touch", oob_path_exists])
  AssertCommand(["chmod", "0400", oob_path_exists])
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
320
321
322
323
324

  try:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program=%s" % oob_path_exists])

325
    AssertCommand(_CLUSTER_VERIFY, fail=True)
326

Iustin Pop's avatar
Iustin Pop committed
327
328
    AssertCommand(["chmod", "0500", oob_path_exists])
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
329

330
    AssertCommand(_CLUSTER_VERIFY)
331
  finally:
Iustin Pop's avatar
Iustin Pop committed
332
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
333
334
335

  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program="])
Iustin Pop's avatar
Iustin Pop committed
336
337


René Nussbaumer's avatar
René Nussbaumer committed
338
339
340
341
342
def TestClusterEpo():
  """gnt-cluster epo"""
  master = qa_config.GetMasterNode()

  # Assert that OOB is unavailable for all nodes
343
  result_output = GetCommandOutput(master.primary,
344
                                   "gnt-node list --verbose --no-headers -o"
René Nussbaumer's avatar
René Nussbaumer committed
345
346
347
348
349
350
351
352
353
354
                                   " powered")
  AssertEqual(compat.all(powered == "(unavail)"
                         for powered in result_output.splitlines()), True)

  # Conflicting
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
  # --all doesn't expect arguments
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)

  # Unless --all is given master is not allowed to be in the list
355
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
René Nussbaumer's avatar
René Nussbaumer committed
356
357
358
359
360

  # This shouldn't fail
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])

  # All instances should have been stopped now
361
  result_output = GetCommandOutput(master.primary,
362
                                   "gnt-instance list --no-headers -o status")
363
364
  # ERROR_down because the instance is stopped but not recorded as such
  AssertEqual(compat.all(status == "ERROR_down"
René Nussbaumer's avatar
René Nussbaumer committed
365
366
367
368
369
370
                         for status in result_output.splitlines()), True)

  # Now start everything again
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])

  # All instances should have been started now
371
  result_output = GetCommandOutput(master.primary,
372
                                   "gnt-instance list --no-headers -o status")
René Nussbaumer's avatar
René Nussbaumer committed
373
374
375
376
  AssertEqual(compat.all(status == "running"
                         for status in result_output.splitlines()), True)


Iustin Pop's avatar
Iustin Pop committed
377
378
def TestClusterVerify():
  """gnt-cluster verify"""
379
  AssertCommand(_CLUSTER_VERIFY)
380
  AssertCommand(["gnt-cluster", "verify-disks"])
381

382

383
384
def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):
  """gnt-cluster verify-disks with broken DRBD"""
385
  qa_daemon.TestPauseWatcher()
386

387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
  try:
    info = qa_instance.GetInstanceInfo(instance.name)
    snode = inst_nodes[1]
    for idx, minor in enumerate(info["drbd-minors"][snode.primary]):
      if idx % 2 == 0:
        break_drbd_cmd = \
          "(drbdsetup %d down >/dev/null 2>&1;" \
          " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \
          (minor, minor)
      else:
        break_drbd_cmd = \
          "(drbdsetup %d detach >/dev/null 2>&1;" \
          " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \
          (minor, minor)
      AssertCommand(break_drbd_cmd, node=snode)

    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
                                     "gnt-cluster verify-disks")
    activation_msg = "Activating disks for instance '%s'" % instance.name
    if activation_msg not in verify_output:
      raise qa_error.Error("gnt-cluster verify-disks did not activate broken"
                           " DRBD disks:\n%s" % verify_output)

    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
                                     "gnt-cluster verify-disks")
    if activation_msg in verify_output:
      raise qa_error.Error("gnt-cluster verify-disks wants to activate broken"
                           " DRBD disks on second attempt:\n%s" % verify_output)

    AssertCommand(_CLUSTER_VERIFY)
  finally:
    qa_daemon.TestResumeWatcher()
419
420


421
422
def TestJobqueue():
  """gnt-debug test-jobqueue"""
Iustin Pop's avatar
Iustin Pop committed
423
  AssertCommand(["gnt-debug", "test-jobqueue"])
424
425


426
427
428
429
430
def TestDelay(node):
  """gnt-debug delay"""
  AssertCommand(["gnt-debug", "delay", "1"])
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
  AssertCommand(["gnt-debug", "delay", "--no-master",
431
                 "-n", node.primary, "1"])
432
433


434
435
def TestClusterReservedLvs():
  """gnt-cluster reserved lvs"""
436
437
438
  # if no lvm-based templates are supported, skip the test
  if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG):
    return
439
440
441
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
  lvname = _QA_LV_PREFIX + "test"
  lvfullname = "/".join([vgname, lvname])
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464

  # Clean cluster
  AssertClusterVerify()

  AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ""])
  AssertCommand(["lvcreate", "-L1G", "-n", lvname, vgname])
  AssertClusterVerify(fail=False,
                      warnings=[constants.CV_ENODEORPHANLV])

  AssertCommand(["gnt-cluster", "modify", "--reserved-lvs",
                 "%s,.*/other-test" % lvfullname])
  AssertClusterVerify(no_warnings=[constants.CV_ENODEORPHANLV])

  AssertCommand(["gnt-cluster", "modify", "--reserved-lvs",
                ".*/%s.*" % _QA_LV_PREFIX])
  AssertClusterVerify(no_warnings=[constants.CV_ENODEORPHANLV])

  AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ""])
  AssertClusterVerify(fail=False,
                      warnings=[constants.CV_ENODEORPHANLV])

  AssertCommand(["lvremove", "-f", lvfullname])
  AssertClusterVerify()
465

466

467
468
469
470
471
def TestClusterModifyEmpty():
  """gnt-cluster modify"""
  AssertCommand(["gnt-cluster", "modify"], fail=True)


472
473
474
475
476
477
def TestClusterModifyDisk():
  """gnt-cluster modify -D"""
  for param in _FAIL_PARAMS:
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)


478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
def _GetOtherEnabledDiskTemplate(undesired_disk_templates,
                                 enabled_disk_templates):
  """Returns one template that is not in the undesired set.

  @type undesired_disk_templates: list of string
  @param undesired_disk_templates: a list of disk templates that we want to
      exclude when drawing one disk template from the list of enabled
      disk templates
  @type enabled_disk_templates: list of string
  @param enabled_disk_templates: list of enabled disk templates (in QA)

  """
  desired_templates = list(set(enabled_disk_templates)
                                - set(undesired_disk_templates))
  if desired_templates:
    template = desired_templates[0]
  else:
    # If no desired disk template is available for QA, choose 'diskless' and
    # hope for the best.
    template = constants.ST_DISKLESS

  return template


def TestClusterModifyFileBasedStorageDir(
    file_disk_template, dir_config_key, default_dir, option_name):
  """Tests gnt-cluster modify wrt to file-based directory options.

  @type file_disk_template: string
  @param file_disk_template: file-based disk template
  @type dir_config_key: string
  @param dir_config_key: key for the QA config to retrieve the default
     directory value
  @type default_dir: string
  @param default_dir: default directory, if the QA config does not specify
     it
  @type option_name: string
  @param option_name: name of the option of 'gnt-cluster modify' to
     change the directory

  """
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
520
  assert file_disk_template in constants.DTS_FILEBASED
521
522
523
524
525
  if not qa_config.IsTemplateSupported(file_disk_template):
    return

  # Get some non-file-based disk template to disable file storage
  other_disk_template = _GetOtherEnabledDiskTemplate(
526
527
528
529
    utils.storage.GetDiskTemplatesOfStorageTypes(constants.ST_FILE,
                                                 constants.ST_SHARED_FILE),
    enabled_disk_templates
  )
530
531
532
533
534
535

  file_storage_dir = qa_config.get(dir_config_key, default_dir)
  invalid_file_storage_dir = "/boot/"

  for fail, cmd in [
    (False, ["gnt-cluster", "modify",
536
537
             "--enabled-disk-templates=%s" % file_disk_template,
             "--ipolicy-disk-templates=%s" % file_disk_template]),
538
    (False, ["gnt-cluster", "modify",
539
             "--%s=%s" % (option_name, file_storage_dir)]),
540
    (False, ["gnt-cluster", "modify",
541
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
542
543
544
545
546
547
548
549
    # file storage dir is set to an inacceptable path, thus verify
    # should fail
    (True, ["gnt-cluster", "verify"]),
    # unsetting the storage dir while file storage is enabled
    # should fail
    (True, ["gnt-cluster", "modify",
            "--%s=" % option_name]),
    (False, ["gnt-cluster", "modify",
550
             "--%s=%s" % (option_name, file_storage_dir)]),
551
    (False, ["gnt-cluster", "modify",
552
553
             "--enabled-disk-templates=%s" % other_disk_template,
             "--ipolicy-disk-templates=%s" % other_disk_template]),
554
    (False, ["gnt-cluster", "modify",
555
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
556
557
558
559
560
561
    # file storage is set to an inacceptable path, but file storage
    # is disabled, thus verify should not fail
    (False, ["gnt-cluster", "verify"]),
    # unsetting the file storage dir while file storage is not enabled
    # should be fine
    (False, ["gnt-cluster", "modify",
562
             "--%s=" % option_name]),
563
564
    # resetting everything to sane values
    (False, ["gnt-cluster", "modify",
565
566
567
             "--%s=%s" % (option_name, file_storage_dir),
             "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
             "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
    ]:
    AssertCommand(cmd, fail=fail)


def TestClusterModifyFileStorageDir():
  """gnt-cluster modify --file-storage-dir=..."""
  TestClusterModifyFileBasedStorageDir(
      constants.DT_FILE, "default-file-storage-dir",
      pathutils.DEFAULT_FILE_STORAGE_DIR,
      "file-storage-dir")


def TestClusterModifySharedFileStorageDir():
  """gnt-cluster modify --shared-file-storage-dir=..."""
  TestClusterModifyFileBasedStorageDir(
      constants.DT_SHARED_FILE, "default-shared-file-storage-dir",
      pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR,
      "shared-file-storage-dir")


588
589
def TestClusterModifyDiskTemplates():
  """gnt-cluster modify --enabled-disk-templates=..."""
590
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
591
  default_disk_template = qa_config.GetDefaultDiskTemplate()
592

593
594
  _TestClusterModifyDiskTemplatesArguments(default_disk_template)
  _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates)
595
  _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates)
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616

  _RestoreEnabledDiskTemplates()
  nodes = qa_config.AcquireManyNodes(2)

  instance_template = enabled_disk_templates[0]
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)

  _TestClusterModifyUsedDiskTemplate(instance_template,
                                     enabled_disk_templates)

  qa_instance.TestInstanceRemove(instance)
  _RestoreEnabledDiskTemplates()


def _RestoreEnabledDiskTemplates():
  """Sets the list of enabled disk templates back to the list of enabled disk
     templates from the QA configuration. This can be used to make sure that
     the tests that modify the list of disk templates do not interfere with
     other tests.

  """
617
618
619
620
621
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
  cmd = ["gnt-cluster", "modify",
         "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
         "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates),
         ]
622
623
624
625
626
627

  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    cmd.append("--vg-name=%s" % vgname)

  AssertCommand(cmd, fail=False)
628
629


630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
def _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates):
  """Tests argument handling of 'gnt-cluster modify' with respect to
     the parameter '--drbd-usermode-helper'. This test is independent
     of instances.

  """
  _RestoreEnabledDiskTemplates()

  if constants.DT_DRBD8 not in enabled_disk_templates:
    return
  if constants.DT_PLAIN not in enabled_disk_templates:
    return

  drbd_usermode_helper = qa_config.get("drbd-usermode-helper", "/bin/true")
  bogus_usermode_helper = "/tmp/pinkbunny"
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
  for command, fail in [
    (["gnt-cluster", "modify",
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper=%s" % drbd_usermode_helper], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper=%s" % bogus_usermode_helper], True),
    # unsetting helper when DRBD is enabled should not work
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper="], True),
    (["gnt-cluster", "modify",
      "--enabled-disk-templates=%s" % constants.DT_PLAIN,
      "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper="], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper=%s" % drbd_usermode_helper], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper=%s" % drbd_usermode_helper,
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper=",
      "--enabled-disk-templates=%s" % constants.DT_PLAIN,
      "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False),
    (["gnt-cluster", "modify",
      "--drbd-usermode-helper=%s" % drbd_usermode_helper,
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
    ]:
676
677
678
679
680
    AssertCommand(command, fail=fail)
  _RestoreEnabledDiskTemplates()


def _TestClusterModifyDiskTemplatesArguments(default_disk_template):
681
682
683
684
685
  """Tests argument handling of 'gnt-cluster modify' with respect to
     the parameter '--enabled-disk-templates'. This test is independent
     of instances.

  """
686
  _RestoreEnabledDiskTemplates()
687

688
  # bogus templates
689
  AssertCommand(["gnt-cluster", "modify",
690
                 "--enabled-disk-templates=pinkbunny"],
691
                fail=True)
692

693
694
695
  # duplicate entries do no harm
  AssertCommand(
    ["gnt-cluster", "modify",
696
     "--enabled-disk-templates=%s,%s" %
697
698
      (default_disk_template, default_disk_template),
     "--ipolicy-disk-templates=%s" % default_disk_template],
699
    fail=False)
700

701
702
703
704
705
706
707
708
709
710
711
712

def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates):
  """Tests argument handling of 'gnt-cluster modify' with respect to
     the parameter '--enabled-disk-templates' and '--vg-name'. This test is
     independent of instances.

  """
  if not utils.IsLvmEnabled(enabled_disk_templates):
    # These tests only make sense if lvm is enabled for QA
    return

  # determine an LVM and a non-LVM disk template for the tests
Helga Velroyen's avatar
Helga Velroyen committed
713
  non_lvm_template = _GetOtherEnabledDiskTemplate(constants.DTS_LVM,
714
                                                  enabled_disk_templates)
Helga Velroyen's avatar
Helga Velroyen committed
715
  lvm_template = list(set(enabled_disk_templates) & constants.DTS_LVM)[0]
716
717
718
719
720
721
722

  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)

  # Clean start: unset volume group name, disable lvm storage
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % non_lvm_template,
723
     "--ipolicy-disk-templates=%s" % non_lvm_template,
724
725
726
727
728
729
     "--vg-name="],
    fail=False)

  # Try to enable lvm, when no volume group is given
  AssertCommand(
    ["gnt-cluster", "modify",
730
731
     "--enabled-disk-templates=%s" % lvm_template,
     "--ipolicy-disk-templates=%s" % lvm_template],
732
733
734
735
736
737
738
739
740
    fail=True)

  # Set volume group, with lvm still disabled: just a warning
  AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False)

  # Try unsetting vg name and enabling lvm at the same time
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % lvm_template,
741
     "--ipolicy-disk-templates=%s" % lvm_template,
742
743
744
745
746
747
     "--vg-name="],
    fail=True)

  # Enable lvm with vg name present
  AssertCommand(
    ["gnt-cluster", "modify",
748
749
     "--enabled-disk-templates=%s" % lvm_template,
     "--ipolicy-disk-templates=%s" % lvm_template],
750
751
752
753
754
755
756
    fail=False)

  # Try unsetting vg name with lvm still enabled
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True)

  # Disable lvm with vg name still set
  AssertCommand(
757
758
759
760
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % non_lvm_template,
     "--ipolicy-disk-templates=%s" % non_lvm_template,
     ],
761
762
763
764
765
766
767
768
769
    fail=False)

  # Try unsetting vg name with lvm disabled
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False)

  # Set vg name and enable lvm at the same time
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % lvm_template,
770
     "--ipolicy-disk-templates=%s" % lvm_template,
771
772
773
774
775
776
777
     "--vg-name=%s" % vgname],
    fail=False)

  # Unset vg name and disable lvm at the same time
  AssertCommand(
    ["gnt-cluster", "modify",
     "--enabled-disk-templates=%s" % non_lvm_template,
778
     "--ipolicy-disk-templates=%s" % non_lvm_template,
779
780
781
782
783
     "--vg-name="],
    fail=False)

  _RestoreEnabledDiskTemplates()

784
785
786
787
788
789
790
791
792
793
794
795
796

def _TestClusterModifyUsedDiskTemplate(instance_template,
                                       enabled_disk_templates):
  """Tests that disk templates that are currently in use by instances cannot
     be disabled on the cluster.

  """
  # If the list of enabled disk templates contains only one template
  # we need to add some other templates, because the list of enabled disk
  # templates can only be set to a non-empty list.
  new_disk_templates = list(set(enabled_disk_templates)
                              - set([instance_template]))
  if not new_disk_templates:
797
    new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK])
798
799
800
                                - set([instance_template]))
  AssertCommand(
    ["gnt-cluster", "modify",
801
802
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
803
804
805
    fail=True)


806
807
def TestClusterModifyBe():
  """gnt-cluster modify -B"""
Iustin Pop's avatar
Iustin Pop committed
808
  for fail, cmd in [
809
810
811
812
813
814
815
816
817
818
819
820
    # max/min mem
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
821
    # vcpus
Iustin Pop's avatar
Iustin Pop committed
822
823
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
824
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
Iustin Pop's avatar
Iustin Pop committed
825
826
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
827
    # auto_balance
Iustin Pop's avatar
Iustin Pop committed
828
829
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
Andrea Spadaccini's avatar
Andrea Spadaccini committed
830
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
Iustin Pop's avatar
Iustin Pop committed
831
832
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
833
    ]:
Iustin Pop's avatar
Iustin Pop committed
834
    AssertCommand(cmd, fail=fail)
835

836
837
838
839
  # redo the original-requested BE parameters, if any
  bep = qa_config.get("backend-parameters", "")
  if bep:
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
840

Andrea Spadaccini's avatar
Andrea Spadaccini committed
841

842
843
844
845
846
847
def _GetClusterIPolicy():
  """Return the run-time values of the cluster-level instance policy.

  @rtype: tuple
  @return: (policy, specs), where:
      - policy is a dictionary of the policy values, instance specs excluded
848
849
      - specs is a dictionary containing only the specs, using the internal
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
850
851

  """
852
853
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
  policy = info["Instance policy - limits for instances"]
854
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
855

856
  # Sanity checks
857
858
  assert "minmax" in ret_specs and "std" in ret_specs
  assert len(ret_specs["minmax"]) > 0
859
860
  assert len(ret_policy) > 0
  return (ret_policy, ret_specs)
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891


def TestClusterModifyIPolicy():
  """gnt-cluster modify --ipolicy-*"""
  basecmd = ["gnt-cluster", "modify"]
  (old_policy, old_specs) = _GetClusterIPolicy()
  for par in ["vcpu-ratio", "spindle-ratio"]:
    curr_val = float(old_policy[par])
    test_values = [
      (True, 1.0),
      (True, 1.5),
      (True, 2),
      (False, "a"),
      # Restore the old value
      (True, curr_val),
      ]
    for (good, val) in test_values:
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
      AssertCommand(cmd, fail=not good)
      if good:
        curr_val = val
      # Check the affected parameter
      (eff_policy, eff_specs) = _GetClusterIPolicy()
      AssertEqual(float(eff_policy[par]), curr_val)
      # Check everything else
      AssertEqual(eff_specs, old_specs)
      for p in eff_policy.keys():
        if p == par:
          continue
        AssertEqual(eff_policy[p], old_policy[p])

892
893
894
895
896
  # Allowing disk templates via ipolicy requires them to be
  # enabled on the cluster.
  if not (qa_config.IsTemplateSupported(constants.DT_PLAIN)
          and qa_config.IsTemplateSupported(constants.DT_DRBD8)):
    return
897
898
  # Disk templates are treated slightly differently
  par = "disk-templates"
899
  disp_str = "allowed disk templates"
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
  curr_val = old_policy[disp_str]
  test_values = [
    (True, constants.DT_PLAIN),
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
    (False, "thisisnotadisktemplate"),
    (False, ""),
    # Restore the old value
    (True, curr_val.replace(" ", "")),
    ]
  for (good, val) in test_values:
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
    AssertCommand(cmd, fail=not good)
    if good:
      curr_val = val
    # Check the affected parameter
    (eff_policy, eff_specs) = _GetClusterIPolicy()
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
    # Check everything else
    AssertEqual(eff_specs, old_specs)
    for p in eff_policy.keys():
      if p == disp_str:
        continue
      AssertEqual(eff_policy[p], old_policy[p])


925
926
def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
                         old_values=None):
927
928
  """Change instance specs.

929
930
931
932
933
934
  At most one of new_specs or diff_specs can be specified.

  @type new_specs: dict
  @param new_specs: new complete specs, in the same format returned by
      L{_GetClusterIPolicy}
  @type diff_specs: dict
935
936
937
  @param diff_specs: partial specs, it can be an incomplete specifications, but
      if min/max specs are specified, their number must match the number of the
      existing specs
938
939
940
941
  @type fail: bool
  @param fail: if the change is expected to fail
  @type old_values: tuple
  @param old_values: (old_policy, old_specs), as returned by
942
      L{_GetClusterIPolicy}
943
944
945
  @return: same as L{_GetClusterIPolicy}

  """
946
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
947
948
949
950
  return qa_utils.TestSetISpecs(
    new_specs=new_specs, diff_specs=diff_specs,
    get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
    fail=fail, old_values=old_values)
951
952
953
954


def TestClusterModifyISpecs():
  """gnt-cluster modify --specs-*"""
955
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
956
  (cur_policy, cur_specs) = _GetClusterIPolicy()
957
958
  # This test assumes that there is only one min/max bound
  assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
  for par in params:
    test_values = [
      (True, 0, 4, 12),
      (True, 4, 4, 12),
      (True, 4, 12, 12),
      (True, 4, 4, 4),
      (False, 4, 0, 12),
      (False, 4, 16, 12),
      (False, 4, 4, 0),
      (False, 12, 4, 4),
      (False, 12, 4, 0),
      (False, "a", 4, 12),
      (False, 0, "a", 12),
      (False, 0, 4, "a"),
      # This is to restore the old values
      (True,
975
976
977
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
       cur_specs[constants.ISPECS_STD][par],
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
978
979
      ]
    for (good, mn, st, mx) in test_values:
980
      new_vals = {
981
982
983
984
985
        constants.ISPECS_MINMAX: [{
          constants.ISPECS_MIN: {par: mn},
          constants.ISPECS_MAX: {par: mx}
          }],
        constants.ISPECS_STD: {par: st}
986
        }
987
988
      cur_state = (cur_policy, cur_specs)
      # We update cur_specs, as we've copied the values to restore already
989
990
      (cur_policy, cur_specs) = TestClusterSetISpecs(
        diff_specs=new_vals, fail=not good, old_values=cur_state)
991

992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
    # Get the ipolicy command
    mnode = qa_config.GetMasterNode()
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
    modcmd = ["gnt-cluster", "modify"]
    opts = initcmd.split()
    assert opts[0:2] == ["gnt-cluster", "init"]
    for k in range(2, len(opts) - 1):
      if opts[k].startswith("--ipolicy-"):
        assert k + 2 <= len(opts)
        modcmd.extend(opts[k:k + 2])
    # Re-apply the ipolicy (this should be a no-op)
    AssertCommand(modcmd)
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
    AssertEqual(initcmd, new_initcmd)
1006
1007


1008
1009
def TestClusterInfo():
  """gnt-cluster info"""
Iustin Pop's avatar
Iustin Pop committed
1010
  AssertCommand(["gnt-cluster", "info"])
Michael Hanselmann's avatar
Michael Hanselmann committed
1011
1012


Iustin Pop's avatar
Iustin Pop committed
1013
1014
1015
1016
1017
def TestClusterRedistConf():
  """gnt-cluster redist-conf"""
  AssertCommand(["gnt-cluster", "redist-conf"])


Michael Hanselmann's avatar
Michael Hanselmann committed
1018
1019
def TestClusterGetmaster():
  """gnt-cluster getmaster"""
Iustin Pop's avatar
Iustin Pop committed
1020
  AssertCommand(["gnt-cluster", "getmaster"])
Michael Hanselmann's avatar
Michael Hanselmann committed
1021
1022
1023
1024


def TestClusterVersion():
  """gnt-cluster version"""
Iustin Pop's avatar
Iustin Pop committed
1025
  AssertCommand(["gnt-cluster", "version"])
1026
1027


1028
1029
1030
1031
1032
1033
def TestClusterRenewCrypto():
  """gnt-cluster renew-crypto"""
  master = qa_config.GetMasterNode()

  # Conflicting options
  cmd = ["gnt-cluster", "renew-crypto", "--force",
Michael Hanselmann's avatar
Michael Hanselmann committed
1034
1035
1036
1037
1038
1039
         "--new-cluster-certificate", "--new-confd-hmac-key"]
  conflicting = [
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
    ]
  for i in conflicting:
Andrea Spadaccini's avatar
Andrea Spadaccini committed
1040
    AssertCommand(cmd + i, fail=True)
1041
1042
1043
1044

  # Invalid RAPI certificate
  cmd = ["gnt-cluster", "renew-crypto", "--force",
         "--rapi-certificate=/dev/null"]
Iustin Pop's avatar
Iustin Pop committed
1045
  AssertCommand(cmd, fail=True)
1046

1047
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
1048
                                         pathutils.RAPI_CERT_FILE)
1049
1050
1051
  try:
    # Custom RAPI certificate
    fh = tempfile.NamedTemporaryFile()
1052

1053
1054
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
1055

1056
    utils.GenerateSelfSignedSslCert(fh.name, 1, validity=validity)
1057

1058
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
1059
    try:
Iustin Pop's avatar
Iustin Pop committed
1060
1061
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--rapi-certificate=%s" % tmpcert])
1062
    finally:
Iustin Pop's avatar
Iustin Pop committed
1063
      AssertCommand(["rm", "-f", tmpcert])
1064

Michael Hanselmann's avatar
Michael Hanselmann committed
1065
1066
1067
1068
1069
1070
    # Custom cluster domain secret
    cds_fh = tempfile.NamedTemporaryFile()
    cds_fh.write(utils.GenerateSecret())
    cds_fh.write("\n")
    cds_fh.flush()

1071
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
Michael Hanselmann's avatar
Michael Hanselmann committed
1072
    try:
Iustin Pop's avatar
Iustin Pop committed
1073
1074
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--cluster-domain-secret=%s" % tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
1075
    finally:
Iustin Pop's avatar
Iustin Pop committed
1076
      AssertCommand(["rm", "-f", tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
1077

1078
    # Normal case
Iustin Pop's avatar
Iustin Pop committed
1079
1080
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--new-cluster-certificate", "--new-confd-hmac-key",
1081
1082
1083
1084
1085
1086
                   "--new-rapi-certificate", "--new-cluster-domain-secret",
                   "--new-node-certificates"])

    # Only renew node certificates
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--new-node-certificates"])
Michael Hanselmann's avatar
Michael Hanselmann committed
1087

1088
    # Restore RAPI certificate
Iustin Pop's avatar
Iustin Pop committed
1089
1090
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--rapi-certificate=%s" % rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
1091
  finally:
Iustin Pop's avatar
Iustin Pop committed
1092
    AssertCommand(["rm", "-f", rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
1093

1094

1095
1096
1097
1098
def TestClusterBurnin():
  """Burnin"""
  master = qa_config.GetMasterNode()

Iustin Pop's avatar
Iustin Pop committed
1099
  options = qa_config.get("options", {})
1100
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
Iustin Pop's avatar
Iustin Pop committed
1101
1102
1103
1104
  parallel = options.get("burnin-in-parallel", False)
  check_inst = options.get("burnin-check-instances", False)
  do_rename = options.get("burnin-rename", "")
  do_reboot = options.get("burnin-reboot", True)
1105
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
1106

1107
1108
1109
  # Get as many instances as we need
  instances = []
  try:
1110
    try:
Iustin Pop's avatar
Iustin Pop committed
1111
      num = qa_config.get("options", {}).get("burnin-instances", 1)
1112
      for _ in range(0, num):
1113
1114
1115
        instances.append(qa_config.AcquireInstance())
    except qa_error.OutOfInstancesError:
      print "Not enough instances, continuing anyway."
1116

1117
1118
    if len(instances) < 1:
      raise qa_error.Error("Burnin needs at least one instance")
1119

1120
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
1121
    try:
1122
      disks = qa_config.GetDiskOptions()
1123
      # Run burnin
Klaus Aehlig's avatar
Klaus Aehlig committed
1124
      cmd = ["env",
1125
             "PYTHONPATH=%s" % _constants.VERSIONEDSHAREDIR,
Klaus Aehlig's avatar
Klaus Aehlig committed
1126
             script,
Iustin Pop's avatar
Iustin Pop committed
1127
             "--os=%s" % qa_config.get("os"),
1128
1129
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
1130
1131
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
Iustin Pop's avatar
Iustin Pop committed
1132
             "--disk-template=%s" % disk_template]
1133
      if parallel:
Iustin Pop's avatar
Iustin Pop committed
1134
1135
        cmd.append("--parallel")
        cmd.append("--early-release")
1136
      if check_inst:
Iustin Pop's avatar
Iustin Pop committed
1137
        cmd.append("--http-check")
Iustin Pop's avatar
Iustin Pop committed
1138
      if do_rename:
Iustin Pop's avatar
Iustin Pop committed
1139
        cmd.append("--rename=%s" % do_rename)
1140
      if not do_reboot:
Iustin Pop's avatar
Iustin Pop committed
1141
        cmd.append("--no-reboot")
1142
      else:
Iustin Pop's avatar
Iustin Pop committed
1143
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
1144
      cmd += [inst.name for inst in instances]
Iustin Pop's avatar
Iustin Pop committed
1145
      AssertCommand(cmd)
1146
    finally:
Iustin Pop's avatar
Iustin Pop committed
1147
1148
      AssertCommand(["rm", "-f", script])

1149
1150
  finally:
    for inst in instances:
1151
      inst.Release()
1152
1153
1154


def TestClusterMasterFailover():
1155
  """gnt-cluster master-failover"""
1156
1157
1158
  master = qa_config.GetMasterNode()
  failovermaster = qa_config.AcquireNode(exclude=master)

Iustin Pop's avatar
Iustin Pop committed
1159
  cmd = ["gnt-cluster", "master-failover"]
1160
  node_list_cmd = ["gnt-node", "list"]
Iustin Pop's avatar
Iustin Pop committed
1161
1162
  try:
    AssertCommand(cmd, node=failovermaster)
1163
    AssertCommand(node_list_cmd, node=failovermaster)
1164
    # Back to original master node
Iustin Pop's avatar
Iustin Pop committed
1165
    AssertCommand(cmd, node=master)
1166
    AssertCommand(node_list_cmd, node=master)
1167
  finally:
1168
    failovermaster.Release()
1169
1170


1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
def TestUpgrade():
  """Test gnt-cluster upgrade.

  This tests the 'gnt-cluster upgrade' command by flipping
  between the current and a different version of Ganeti.
  To also recover subtile points in the configuration up/down
  grades, instances are left over both upgrades.

  """
  this_version = qa_config.get("dir-version")
  other_version = qa_config.get("other-dir-version")
  if this_version is None or other_version is None:
    print qa_utils.FormatInfo("Test not run, as versions not specified")
    return

  inst_creates = []
  upgrade_instances = qa_config.get("upgrade-instances", [])
  live_instances = []
  for (test_name, templ, cf, n) in qa_instance.available_instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ) and
        templ in upgrade_instances):
      inst_creates.append((cf, n))
Klaus Aehlig's avatar
Klaus Aehlig committed
1194

1195
1196
1197
1198
1199
1200
1201
1202
1203