ganeti-qa.py 38.4 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2 3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Klaus Aehlig's avatar
Klaus Aehlig committed
5
# All rights reserved.
Iustin Pop's avatar
Iustin Pop committed
6
#
Klaus Aehlig's avatar
Klaus Aehlig committed
7 8 9
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
Iustin Pop's avatar
Iustin Pop committed
10
#
Klaus Aehlig's avatar
Klaus Aehlig committed
11 12
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
Iustin Pop's avatar
Iustin Pop committed
13
#
Klaus Aehlig's avatar
Klaus Aehlig committed
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Iustin Pop's avatar
Iustin Pop committed
29 30


31
"""Script for doing QA on Ganeti.
32 33

"""
Iustin Pop's avatar
Iustin Pop committed
34

35
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
36 37
# due to invalid name

38
import copy
39 40
import datetime
import optparse
41
import sys
Iustin Pop's avatar
Iustin Pop committed
42

43
import colors
44 45 46 47
import qa_cluster
import qa_config
import qa_daemon
import qa_env
48
import qa_error
Niklas Hambuechen's avatar
Niklas Hambuechen committed
49
import qa_filters
50
import qa_group
51
import qa_instance
52
import qa_iptables
53
import qa_monitoring
Helga Velroyen's avatar
Helga Velroyen committed
54
import qa_network
55
import qa_node
56
import qa_os
57
import qa_performance
58
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
59
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
60
import qa_tags
61
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
62

63
from ganeti import utils
64
from ganeti import rapi # pylint: disable=W0611
65
from ganeti import constants
66
from ganeti import netutils
67

68
import ganeti.rapi.client # pylint: disable=W0611
69
from ganeti.rapi.client import UsesRapiClient
70

Iustin Pop's avatar
Iustin Pop committed
71

72
def _FormatHeader(line, end=72, mark="-", color=None):
Iustin Pop's avatar
Iustin Pop committed
73 74 75
  """Fill a line up to the end column.

  """
76
  line = (mark * 4) + " " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
77
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
78
  line = line.rstrip()
Petr Pudlak's avatar
Petr Pudlak committed
79
  line = colors.colorize(line, color=color)
Iustin Pop's avatar
Iustin Pop committed
80 81 82
  return line


Iustin Pop's avatar
Iustin Pop committed
83 84
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
85 86

  """
87 88
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
89 90 91
    desc = desc.rstrip(".")
    if fn.__name__:
      desc = "[" + fn.__name__ + "] " + desc
Iustin Pop's avatar
Iustin Pop committed
92
  else:
Iustin Pop's avatar
Iustin Pop committed
93
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
94

95
  return desc
Iustin Pop's avatar
Iustin Pop committed
96

97

98
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
99 100 101
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
102

Iustin Pop's avatar
Iustin Pop committed
103
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
104

Iustin Pop's avatar
Iustin Pop committed
105 106
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
107
  print
108 109
  print _FormatHeader("%s start %s" % (tstart, desc),
                      color=colors.YELLOW, mark="<")
Iustin Pop's avatar
Iustin Pop committed
110 111

  try:
112
    retval = fn(*args, **kwargs)
113
    print _FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)
Iustin Pop's avatar
Iustin Pop committed
114
    return retval
115 116 117
  except Exception, e:
    print _FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)
    raise
Iustin Pop's avatar
Iustin Pop committed
118 119 120
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
121 122
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
                        color=colors.MAGENTA, mark=">")
Iustin Pop's avatar
Iustin Pop committed
123 124


125 126 127 128 129 130 131 132 133 134 135 136 137
def ReportTestSkip(desc, testnames):
  """Reports that tests have been skipped.

  @type desc: string
  @param desc: string
  @type testnames: string or list of string
  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  tstart = datetime.datetime.now()
  # TODO: Formatting test names when non-string names are involved
  print _FormatHeader("%s skipping %s, test(s) %s disabled" %
138 139
                      (tstart, desc, testnames),
                      color=colors.BLUE, mark="*")
140 141


142
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
143 144 145 146 147 148 149
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
150
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
151 152
  else:
    desc = _DescriptionOf(fn)
153
    ReportTestSkip(desc, testnames)
Iustin Pop's avatar
Iustin Pop committed
154 155


156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
def RunTestBlock(fn, *args, **kwargs):
  """Runs a block of tests after printing a header.

  """
  tstart = datetime.datetime.now()

  desc = _DescriptionOf(fn)

  print
  print _FormatHeader("BLOCK %s start %s" % (tstart, desc),
                      color=[colors.YELLOW, colors.BOLD], mark="v")

  try:
    return fn(*args, **kwargs)
  except Exception, e:
    print _FormatHeader("BLOCK FAILED %s: %s" % (desc, e),
                        color=[colors.RED, colors.BOLD])
    raise
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("BLOCK %s time=%s %s" % (tstop, tdelta, desc),
                        color=[colors.MAGENTA, colors.BOLD], mark="^")


Michael Hanselmann's avatar
Michael Hanselmann committed
181 182
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
183 184

  """
Iustin Pop's avatar
Iustin Pop committed
185 186 187
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
188

189

190
def SetupCluster(rapi_user):
Michael Hanselmann's avatar
Michael Hanselmann committed
191
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
192

193
  @param rapi_user: Login user for RAPI
194
  @return: Login secret for RAPI
195

Michael Hanselmann's avatar
Michael Hanselmann committed
196
  """
197
  rapi_secret = utils.GenerateSecret()
Iustin Pop's avatar
Iustin Pop committed
198 199
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
200 201 202 203
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
204 205
    if qa_rapi.Enabled():
      # To support RAPI on an existing cluster we have to find out the secret
206
      rapi_secret = qa_rapi.LookupRapiSecret(rapi_user)
207

208 209
  qa_group.ConfigureGroups()

210 211 212
  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
213
  RunTestIf("job-list", qa_job.TestJobList)
214

Iustin Pop's avatar
Iustin Pop committed
215 216
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
217 218
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
219

Iustin Pop's avatar
Iustin Pop committed
220
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
221
  RunTestIf("test-jobqueue", qa_job.TestJobCancellation)
222

223 224 225
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

226 227
  RunTestIf("node-list", qa_node.TestNodeList)

228 229 230
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
231
  RunTestIf("job-list", qa_job.TestJobListFields)
232
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
233

Iustin Pop's avatar
Iustin Pop committed
234
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
235

236 237
  return rapi_secret

Michael Hanselmann's avatar
Michael Hanselmann committed
238

239
def RunClusterTests(rapi_user=None, rapi_secret=None):
Michael Hanselmann's avatar
Michael Hanselmann committed
240
  """Runs tests related to gnt-cluster.
241

242 243 244 245 246
  @type rapi_user: string
  @param rapi_user: name of the rapi user
  @type rapi_secret: string
  @param rapi_secret: the rapi secret

Michael Hanselmann's avatar
Michael Hanselmann committed
247
  """
Iustin Pop's avatar
Iustin Pop committed
248
  for test, fn in [
249
    ("create-cluster", qa_cluster.TestClusterInitDisk),
250 251 252 253 254 255 256 257 258 259 260 261 262
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto)
    ]:
    RunTestIf(test, fn)

  # Since renew-crypto replaces the RAPI cert, reload it.
  if qa_rapi.Enabled():
    if not rapi_user:
      raise qa_error.Error("No RAPI user given.")
    if not rapi_secret:
      raise qa_error.Error("No RAPI secret given.")
    qa_rapi.Setup(rapi_user, rapi_secret)

  for test, fn in [
Iustin Pop's avatar
Iustin Pop committed
263 264
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
265
    # TODO: add more cluster modify tests
266
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
267 268
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
Iustin Pop's avatar
Iustin Pop committed
269
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
270
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
271
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
272 273
    ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
    ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
274
    ("cluster-modify", qa_cluster.TestClusterModifyInstallImage),
275
    ("cluster-modify", qa_cluster.TestClusterModifyUserShutdown),
Iustin Pop's avatar
Iustin Pop committed
276 277 278 279
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
280
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
281 282
    (["cluster-copyfile", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterCopyfile),
Iustin Pop's avatar
Iustin Pop committed
283 284 285
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
286 287
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
288 289
    (["cluster-oob", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterOob),
290
    ("cluster-instance-communication", qa_cluster.TestInstanceCommunication),
291 292 293
    (qa_rapi.Enabled, qa_rapi.TestVersion),
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
294 295
    ]:
    RunTestIf(test, fn)
296

297

298 299 300 301 302 303 304
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
305 306
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
307

Michael Hanselmann's avatar
Michael Hanselmann committed
308
  """
309 310
  os_enabled = ["os", qa_config.NoVirtualCluster]

311
  if qa_config.TestEnabled(qa_rapi.Enabled):
312 313 314 315
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
316 317 318
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
319
    ]:
320
    RunTestIf(os_enabled, fn)
321 322

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
323 324 325
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
326
    ]:
327
    RunTestIf(os_enabled, fn, rapi_getos)
328 329

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
330 331
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
332
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
333
    ]:
334
    RunTestIf(os_enabled, fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
335 336


337
def RunCommonInstanceTests(instance, inst_nodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
338 339 340
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
341
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
342
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
343
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
344 345
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
346
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
347

348
  # Test shutdown/start via RAPI
349
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
350
            qa_rapi.TestRapiInstanceShutdown, instance)
351
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
352 353
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
354
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
355

Iustin Pop's avatar
Iustin Pop committed
356
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
357

Iustin Pop's avatar
Iustin Pop committed
358
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
359
  RunTestIf(["instance-modify", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
360
            qa_rapi.TestRapiInstanceModify, instance)
361

Iustin Pop's avatar
Iustin Pop committed
362
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
363
  RunTestIf(["instance-console", qa_rapi.Enabled],
364
            qa_rapi.TestRapiInstanceConsole, instance)
365

366 367
  RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
            instance)
368 369 370 371 372 373
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
374 375 376 377
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
378
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
379
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
380
            qa_rapi.TestRapiInstanceReinstall, instance)
381

Iustin Pop's avatar
Iustin Pop committed
382
  if qa_config.TestEnabled("instance-rename"):
383 384
    tgt_instance = qa_config.AcquireInstance()
    try:
385 386
      rename_source = instance.name
      rename_target = tgt_instance.name
387
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
388
      RunTest(qa_instance.TestInstanceRenameAndBack,
389
              rename_source, rename_source)
390
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
391 392 393 394
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
395
                rename_source, rename_target)
396
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
397 398
                  rename_source, rename_target)
    finally:
399
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
400

Iustin Pop's avatar
Iustin Pop committed
401 402
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
403 404 405 406
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
407

Iustin Pop's avatar
Iustin Pop committed
408
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
409

410 411 412
  if instance.disk_template == constants.DT_DRBD8:
    RunTestIf("cluster-verify",
              qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
Michael Hanselmann's avatar
Michael Hanselmann committed
413
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
414

415
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
416

417 418 419
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

420 421 422
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

423 424 425 426 427

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
428 429
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
430
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
431

432

433 434 435 436
def RunGroupListTests():
  """Run tests for listing node groups.

  """
437 438
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
439 440


Helga Velroyen's avatar
Helga Velroyen committed
441 442 443 444 445 446
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)
Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
447
  RunTestIf(["network", "tags"], qa_network.TestNetworkTags)
Helga Velroyen's avatar
Helga Velroyen committed
448 449


Niklas Hambuechen's avatar
Niklas Hambuechen committed
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
def RunFilterTests():
  """Run tests for job filter management.

  """
  RunTestIf("filters", qa_filters.TestFilterList)
  RunTestIf("filters", qa_filters.TestFilterListFields)
  RunTestIf("filters", qa_filters.TestFilterAddRemove)
  RunTestIf("filters", qa_filters.TestFilterReject)
  RunTestIf("filters", qa_filters.TestFilterOpCode)
  RunTestIf("filters", qa_filters.TestFilterReasonChain)
  RunTestIf("filters", qa_filters.TestFilterContinue)
  RunTestIf("filters", qa_filters.TestFilterAcceptPause)
  RunTestIf("filters", qa_filters.TestFilterWatermark)
  RunTestIf("filters", qa_filters.TestFilterRateLimit)
  RunTestIf("filters", qa_filters.TestAdHocReasonRateLimit)


467 468 469 470 471
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
472 473
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
474
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
475 476
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
477

478

479
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
480
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
481

482 483
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
484

Michael Hanselmann's avatar
Michael Hanselmann committed
485
  """
486 487 488
  # FIXME: export explicitly bails out on file based storage. other non-lvm
  # based storage types are untested, though. Also note that import could still
  # work, but is deeply embedded into the "export" case.
489
  if qa_config.TestEnabled("instance-export"):
490 491
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

492
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
493 494 495 496 497 498
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
499
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
500
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
501
        try:
502
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
503
                  expnode, name)
504 505
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
506
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
507
        finally:
508
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
509
    finally:
510
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
511

512 513
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
  # See Issue 414.
Aaron Karper's avatar
Aaron Karper committed
514
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])):
515 516
    newinst = qa_config.AcquireInstance()
    try:
517
      tnode = qa_config.AcquireNode(exclude=inodes)
518 519
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
520
                inodes, tnode)
521
      finally:
522
        tnode.Release()
523
    finally:
524
      newinst.Release()
525

Michael Hanselmann's avatar
Michael Hanselmann committed
526

527
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
528
  """Test the ganeti-watcher script.
529

Michael Hanselmann's avatar
Michael Hanselmann committed
530
  """
531
  RunTest(qa_daemon.TestPauseWatcher)
532

Iustin Pop's avatar
Iustin Pop committed
533
  RunTestIf("instance-automatic-restart",
534
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
535
  RunTestIf("instance-consecutive-failures",
536
            qa_daemon.TestInstanceConsecutiveFailures, instance)
537

538 539
  RunTest(qa_daemon.TestResumeWatcher)

540

541
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
542
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
543

Michael Hanselmann's avatar
Michael Hanselmann committed
544
  """
Iustin Pop's avatar
Iustin Pop committed
545
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
546
  RunTestIf(["instance-failover", qa_rapi.Enabled],
547
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
548

Iustin Pop's avatar
Iustin Pop committed
549
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
550
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
551
            qa_rapi.TestRapiInstanceMigrate, instance)
552

Iustin Pop's avatar
Iustin Pop committed
553
  if qa_config.TestEnabled("instance-replace-disks"):
554 555
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
556
    try:
557
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
558
      RunTest(qa_instance.TestReplaceDisks,
559
              instance, inodes, othernodes)
560
    finally:
561 562
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
563

564 565
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
566 567
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
568
    except qa_error.OutOfNodesError:
569 570 571 572 573 574 575 576
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
577 578
    try:
      RunTest(qa_instance.TestRecreateDisks,
579
              instance, inodes, othernodes)
580
    finally:
581
      qa_config.ReleaseManyNodes(acquirednodes)
582

583 584 585
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Thomas Thrainer's avatar
Thomas Thrainer committed
586
    RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
587 588


589 590 591 592 593 594 595
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
596
    old_es = qa_cluster.TestSetExclStorCluster(False)
597
    qa_node.TestExclStorSingleNode(node)
598 599

    qa_cluster.TestSetExclStorCluster(True)
600 601
    qa_cluster.TestExclStorSharedPv(node)

602 603 604
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
605 606

      # Create and allocate instances
607
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
608 609 610 611 612 613 614 615 616 617
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
618
          instance2.Release()
619
      finally:
620
        instance1.Release()
621

622 623 624 625
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
626
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
627 628 629 630 631 632
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
633
          instance.Release()
634
      finally:
635
        snode.Release()
636 637
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
638
    node.Release()
639 640


641 642 643 644 645 646 647 648 649
def RunCustomSshPortTests():
  """Test accessing nodes with custom SSH ports.

  This requires removing nodes, adding them to a new group, and then undoing
  the change.
  """
  if not qa_config.TestEnabled("group-custom-ssh-port"):
    return

650
  std_port = netutils.GetDaemonPort(constants.SSH)
651 652 653
  port = 211
  master = qa_config.GetMasterNode()
  with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
654 655 656 657 658 659 660 661 662
    # Checks if the node(s) could be contacted through IPv6.
    # If yes, better skip the whole test.

    for node in nodes:
      if qa_utils.UsesIPv6Connection(node.primary, std_port):
        print ("Node %s is likely to be reached using IPv6,"
               "skipping the test" % (node.primary, ))
        return

663 664
    for node in nodes:
      qa_node.NodeRemove(node)
Niklas Hambuechen's avatar
Niklas Hambuechen committed
665
    with qa_iptables.RulesContext() as r:
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
      with qa_group.NewGroupCtx() as group:
        qa_group.ModifyGroupSshPort(r, group, nodes, port)

        for node in nodes:
          qa_node.NodeAdd(node, group=group)

        # Make sure that the cluster doesn't have any pre-existing problem
        qa_cluster.AssertClusterVerify()

        # Create and allocate instances
        instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
        try:
          instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
          try:
            # cluster-verify checks that disks are allocated correctly
            qa_cluster.AssertClusterVerify()

            # Remove instances
            qa_instance.TestInstanceRemove(instance2)
            qa_instance.TestInstanceRemove(instance1)
          finally:
            instance2.Release()
        finally:
          instance1.Release()

        for node in nodes:
          qa_node.NodeRemove(node)

    for node in nodes:
      qa_node.NodeAdd(node)

    qa_cluster.AssertClusterVerify()


700
def _BuildSpecDict(par, mn, st, mx):
701
  return {
702 703 704 705 706
    constants.ISPECS_MINMAX: [{
      constants.ISPECS_MIN: {par: mn},
      constants.ISPECS_MAX: {par: mx},
      }],
    constants.ISPECS_STD: {par: st},
707
    }
708 709


710 711 712 713 714 715 716 717 718 719 720 721 722
def _BuildDoubleSpecDict(index, par, mn, st, mx):
  new_spec = {
    constants.ISPECS_MINMAX: [{}, {}],
    }
  if st is not None:
    new_spec[constants.ISPECS_STD] = {par: st}
  new_spec[constants.ISPECS_MINMAX][index] = {
    constants.ISPECS_MIN: {par: mn},
    constants.ISPECS_MAX: {par: mx},
    }
  return new_spec


723 724
def TestIPolicyPlainInstance():
  """Test instance policy interaction with instances"""
725
  params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
726 727 728 729 730
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
    print "Template %s not supported" % constants.DT_PLAIN
    return

  # This test assumes that the group policy is empty
731
  (_, old_specs) = qa_cluster.TestClusterSetISpecs()
732 733
  # We also assume to have only one min/max bound
  assert len(old_specs[constants.ISPECS_MINMAX]) == 1
734 735
  node = qa_config.AcquireNode()
  try:
736 737
    # Log of policy changes, list of tuples:
    # (full_change, incremental_change, policy_violated)
738
    history = []
739 740 741 742
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
    try:
      policyerror = [constants.CV_EINSTANCEPOLICY]
      for par in params:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
743
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
744 745
        # Some specs must be multiple of 4
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
746
        history.append((None, new_spec, True))
747 748 749 750 751 752 753
        if iminval > 0:
          # Some specs must be multiple of 4
          if iminval >= 4:
            upper = iminval - 4
          else:
            upper = iminval - 1
          new_spec = _BuildSpecDict(par, 0, upper, upper)
754 755
          history.append((None, new_spec, True))
        history.append((old_specs, None, False))
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795

      # Test with two instance specs
      double_specs = copy.deepcopy(old_specs)
      double_specs[constants.ISPECS_MINMAX] = \
          double_specs[constants.ISPECS_MINMAX] * 2
      (par1, par2) = params[0:2]
      (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
      (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
      old_minmax = old_specs[constants.ISPECS_MINMAX][0]
      history.extend([
        (double_specs, None, False),
        # The first min/max limit is being violated
        (None,
         _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
                              imaxval1 + 4),
         False),
        # Both min/max limits are being violated
        (None,
         _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
         True),
        # The second min/max limit is being violated
        (None,
         _BuildDoubleSpecDict(0, par1,
                              old_minmax[constants.ISPECS_MIN][par1],
                              old_specs[constants.ISPECS_STD][par1],
                              old_minmax[constants.ISPECS_MAX][par1]),
         False),
        (old_specs, None, False),
        ])

      # Apply the changes, and check policy violations after each change
      qa_cluster.AssertClusterVerify()
      for (new_specs, diff_specs, failed) in history:
        qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                        diff_specs=diff_specs)
        if failed:
          qa_cluster.AssertClusterVerify(warnings=policyerror)
        else:
          qa_cluster.AssertClusterVerify()

796 797
      qa_instance.TestInstanceRemove(instance)
    finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
798
      instance.Release()
799 800 801

    # Now we replay the same policy changes, and we expect that the instance
    # cannot be created for the cases where we had a policy violation above
802 803 804
    for (new_specs, diff_specs, failed) in history:
      qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                      diff_specs=diff_specs)
805 806 807
      if failed:
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
      # Instance creation with no policy violation has been tested already
808
  finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
809
    node.Release()
810 811


812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
def IsExclusiveStorageInstanceTestEnabled():
  test_name = "exclusive-storage-instance-tests"
  if qa_config.TestEnabled(test_name):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    vgscmd = utils.ShellQuoteArgs([
      "vgs", "--noheadings", "-o", "pv_count", vgname,
      ])
    nodes = qa_config.GetConfig()["nodes"]
    for node in nodes:
      try:
        pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
      except Exception, e:
        msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
               (node.primary, test_name, e))
        raise qa_error.Error(msg)
      if pvnum < 2:
        raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
                             (node.primary, pvnum, test_name))
    res = True
  else:
    res = False
  return res


836 837
def RunInstanceTests():
  """Create and exercise instances."""
838

839 840 841
  requested_conversions = qa_config.get("convert-disk-templates", [])
  supported_conversions = \
      set(requested_conversions).difference(constants.DTS_NOT_CONVERTIBLE_TO)
842 843
  for (test_name, templ, create_fun, num_nodes) in \
      qa_instance.available_instance_tests:
844 845 846 847 848
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
849
        try:
850
          RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
851
                    instance)
852 853 854 855
          RunTestIf("instance-communication",
                    qa_instance.TestInstanceCommunication,
                    instance,
                    qa_config.GetMasterNode())
856 857 858
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
859 860
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node,
                      templ == constants.DT_DRBD8)
861 862 863
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
864
                      inodes[0].primary, inodes[1].primary)
865 866
          # This test will run once but it will cover all the supported
          # user-provided disk template conversions
867
          if qa_config.TestEnabled("instance-convert-disk"):
868 869 870 871 872 873 874 875 876 877 878 879
            if (len(supported_conversions) > 1 and
                instance.disk_template in supported_conversions):
              RunTest(qa_instance.TestInstanceShutdown, instance)
              RunTest(qa_instance.TestInstanceConvertDiskTemplate, instance,
                      supported_conversions)
              RunTest(qa_instance.TestInstanceStartup, instance)
              # At this point we clear the set because the requested conversions
              # has been tested
              supported_conversions.clear()
            else:
              test_desc = "Converting instance of template %s" % templ
              ReportTestSkip(test_desc, "conversion feature")
880 881
          RunTestIf("instance-modify-disks",
                    qa_instance.TestInstanceModifyDisks, instance)
882
          RunCommonInstanceTests(instance, inodes)
883 884 885 886 887
          if qa_config.TestEnabled("instance-modify-primary"):
            othernode = qa_config.AcquireNode()
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
                    instance, inodes[0], othernode)
            othernode.Release()
888 889 890 891 892 893
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
894
          instance.Release()
895 896 897 898
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
899 900 901 902 903 904
    else:
      test_desc = "Creating instances of template %s" % templ
      if not qa_config.TestEnabled(test_name):
        ReportTestSkip(test_desc, test_name)
      else:
        ReportTestSkip(test_desc, "disk template %s" % templ)
Michael Hanselmann's avatar
Michael Hanselmann committed
905 906


907
def RunMonitoringTests():
908
  RunTestIf("mon-collector", qa_monitoring.TestInstStatusCollector)
909 910


911 912 913 914 915 916 917 918
PARALLEL_TEST_DICT = {
  "parallel-failover": qa_performance.TestParallelInstanceFailover,
  "parallel-migration": qa_performance.TestParallelInstanceMigration,
  "parallel-replace-disks": qa_performance.TestParallelInstanceReplaceDisks,
  "parallel-reboot": qa_performance.TestParallelInstanceReboot,
  "parallel-reinstall": qa_performance.TestParallelInstanceReinstall,
  "parallel-rename": qa_performance.TestParallelInstanceRename,
  }
919 920


921
def RunPerformanceTests():
922 923 924 925
  if not qa_config.TestEnabled("performance"):
    ReportTestSkip("performance related tests", "performance")
    return

926
  if qa_config.TestEnabled("jobqueue-performance"):
927
    RunTest(qa_performance.TestParallelMaxInstanceCreationPerformance)
928
    RunTest(qa_performance.TestParallelNodeCountInstanceCreationPerformance)
929

930 931 932
    instances = qa_performance.CreateAllInstances()

    RunTest(qa_performance.TestParallelModify, instances)
933
    RunTest(qa_performance.TestParallelInstanceOSOperations, instances)
934
    RunTest(qa_performance.TestParallelInstanceQueries, instances)
935 936 937

    qa_performance.RemoveAllInstances(instances)

938 939
    RunTest(qa_performance.TestJobQueueSubmissionPerformance)

940
  if qa_config.TestEnabled("parallel-performance"):
941 942 943 944
    if qa_config.IsTemplateSupported(constants.DT_DRBD8):
      RunTest(qa_performance.TestParallelDRBDInstanceCreationPerformance)
    if qa_config.IsTemplateSupported(constants.DT_PLAIN):
      RunTest(qa_performance.TestParallelPlainInstanceCreationPerformance)
945

946 947 948 949 950 951
  # Preparations need to be made only if some of these tests are enabled
  if qa_config.IsTemplateSupported(constants.DT_DRBD8) and \
     qa_config.TestEnabled(qa_config.Either(PARALLEL_TEST_DICT.keys())):
    inodes = qa_config.AcquireManyNodes(2)
    try:
      instance = qa_instance.TestInstanceAddWithDrbdDisk(inodes)
952
      try:
953 954
        for (test_name, test_fn) in PARALLEL_TEST_DICT.items():
          RunTestIf(test_name, test_fn, instance)
955
      finally:
956 957 958 959
        instance.Release()
      qa_instance.TestInstanceRemove(instance)
    finally:
      qa_config.ReleaseManyNodes(inodes)
960

961

962 963
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
964 965

  """
966 967
  rapi_user = "ganeti-qa"

968
  RunTestBlock(RunEnvTests)
969
  rapi_secret = SetupCluster(rapi_user)
Michael Hanselmann's avatar
Michael Hanselmann committed
970

971 972 973
  if qa_rapi.Enabled():
    # Load RAPI certificate
    qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
974

975
  RunTestBlock(RunClusterTests, rapi_user=rapi_user, rapi_secret=rapi_secret)
976
  RunTestBlock(RunOsTests)
977

Iustin Pop's avatar
Iustin Pop committed
978
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
979

980 981 982 983
  RunTestBlock(RunCommonNodeTests)
  RunTestBlock(RunGroupListTests)
  RunTestBlock(RunGroupRwTests)
  RunTestBlock(RunNetworkTests)
Niklas Hambuechen's avatar
Niklas Hambuechen committed
984
  RunTestBlock(RunFilterTests)
985

986 987
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
988 989
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
990 991
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
992
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
993
  finally:
994
    pnode.Release()
995

996 997 998
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
999 1000
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
1001
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
1002

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
1003 1004 1005
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

1006 1007
      if (qa_config.TestEnabled("instance-add-plain-disk")
          and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
1008
        # Normal instance allocation via RAPI
1009 1010 1011
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)