ganeti-qa.py 21.2 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
Helga Velroyen's avatar
Helga Velroyen committed
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
44
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import qa_tags
46
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
47

48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

Iustin Pop's avatar
Iustin Pop committed
55

Iustin Pop's avatar
Iustin Pop committed
56
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
61
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
62
63
64
65
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
66
67
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
70
71
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
72
  else:
Iustin Pop's avatar
Iustin Pop committed
73
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
74

Iustin Pop's avatar
Iustin Pop committed
75
76
  return desc.rstrip(".")

77

78
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
79
80
81
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
82

Iustin Pop's avatar
Iustin Pop committed
83
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
84

Iustin Pop's avatar
Iustin Pop committed
85
86
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
87
88
89
90
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
91
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
92
93
94
95
96
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
97
98


99
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
104
105
106
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
108
109
110
111
112
113
114
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
115
116
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
117
118

  """
Iustin Pop's avatar
Iustin Pop committed
119
120
121
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
122

123

124
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
125
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
126

127
128
129
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
130
  """
Iustin Pop's avatar
Iustin Pop committed
131
132
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
133
134
135
136
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
137
138
139
140

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
141
  RunTestIf("job-list", qa_job.TestJobList)
142

Iustin Pop's avatar
Iustin Pop committed
143
144
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
145
146
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
147

Iustin Pop's avatar
Iustin Pop committed
148
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
149

150
151
152
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

153
154
  RunTestIf("node-list", qa_node.TestNodeList)

155
156
157
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
158
  RunTestIf("job-list", qa_job.TestJobListFields)
159
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
160

Iustin Pop's avatar
Iustin Pop committed
161
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
162
163
164
165


def RunClusterTests():
  """Runs tests related to gnt-cluster.
166

Michael Hanselmann's avatar
Michael Hanselmann committed
167
  """
Iustin Pop's avatar
Iustin Pop committed
168
  for test, fn in [
169
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
170
171
172
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
173
    # TODO: add more cluster modify tests
174
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
175
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
176
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
177
178
179
180
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
181
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
182
183
184
185
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
186
187
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
188
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
189
190
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
191
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
192
193
    ]:
    RunTestIf(test, fn)
194

195

196
197
198
199
200
201
202
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
203
204
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
205

Michael Hanselmann's avatar
Michael Hanselmann committed
206
  """
207
208
209
210
211
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
212
213
214
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
215
216
217
218
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
219
220
221
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
222
223
224
225
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
226
227
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
228
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
229
230
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
231
232
233
234
235
236


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
237
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
238
239
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
240
241
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
242
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
243

244
245
246
247
248
249
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
250
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
251

Iustin Pop's avatar
Iustin Pop committed
252
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
253

Iustin Pop's avatar
Iustin Pop committed
254
255
256
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
257

Iustin Pop's avatar
Iustin Pop committed
258
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
259
260
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
261

262
263
264
265
266
267
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
268
269
270
271
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
272
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
273
274
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
275

Iustin Pop's avatar
Iustin Pop committed
276
  if qa_config.TestEnabled("instance-rename"):
277
278
279
280
281
    tgt_instance = qa_config.AcquireInstance()
    try:
      rename_source = instance["name"]
      rename_target = tgt_instance["name"]
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
282
      RunTest(qa_instance.TestInstanceRenameAndBack,
283
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
284
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
285
286
287
288
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
289
                rename_source, rename_target)
290
291
292
293
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
      qa_config.ReleaseInstance(tgt_instance)
Iustin Pop's avatar
Iustin Pop committed
294

Iustin Pop's avatar
Iustin Pop committed
295
296
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
297
298
299
300
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
301

Iustin Pop's avatar
Iustin Pop committed
302
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
303

Michael Hanselmann's avatar
Michael Hanselmann committed
304
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
305

Iustin Pop's avatar
Iustin Pop committed
306
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
307

308
309
310
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

311
312
313
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

314
315
316
317
318

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
319
320
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
321
  RunTestIf("node-oob", qa_node.TestOutOfBand)
322

323

324
325
326
327
def RunGroupListTests():
  """Run tests for listing node groups.

  """
328
329
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
330
331


Helga Velroyen's avatar
Helga Velroyen committed
332
333
334
335
336
337
338
339
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


340
341
342
343
344
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
345
346
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
347
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
348
349
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
350

351

352
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
353
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
354

355
356
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
357

Michael Hanselmann's avatar
Michael Hanselmann committed
358
  """
Iustin Pop's avatar
Iustin Pop committed
359
  if qa_config.TestEnabled("instance-export"):
360
361
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

362
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
363
364
365
366
367
368
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
369
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
370
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
371
        try:
372
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
373
                  expnode, name)
374
375
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
376
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
377
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
378
379
380
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
381

Iustin Pop's avatar
Iustin Pop committed
382
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
383
384
    newinst = qa_config.AcquireInstance()
    try:
385
      tnode = qa_config.AcquireNode(exclude=inodes)
386
387
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
388
                inodes, tnode)
389
      finally:
390
        qa_config.ReleaseNode(tnode)
391
392
393
    finally:
      qa_config.ReleaseInstance(newinst)

Michael Hanselmann's avatar
Michael Hanselmann committed
394

395
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
396
  """Test the ganeti-watcher script.
397

Michael Hanselmann's avatar
Michael Hanselmann committed
398
  """
399
  RunTest(qa_daemon.TestPauseWatcher)
400

Iustin Pop's avatar
Iustin Pop committed
401
  RunTestIf("instance-automatic-restart",
402
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
403
  RunTestIf("instance-consecutive-failures",
404
            qa_daemon.TestInstanceConsecutiveFailures, instance)
405

406
407
  RunTest(qa_daemon.TestResumeWatcher)

408

409
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
410
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
411

Michael Hanselmann's avatar
Michael Hanselmann committed
412
  """
Iustin Pop's avatar
Iustin Pop committed
413
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
414
415
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
416

Iustin Pop's avatar
Iustin Pop committed
417
418
419
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
420

Iustin Pop's avatar
Iustin Pop committed
421
  if qa_config.TestEnabled("instance-replace-disks"):
422
423
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
424
    try:
425
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
426
      RunTest(qa_instance.TestReplaceDisks,
427
              instance, inodes, othernodes)
428
    finally:
429
430
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
431

432
433
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
434
435
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
436
    except qa_error.OutOfNodesError:
437
438
439
440
441
442
443
444
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
445
446
    try:
      RunTest(qa_instance.TestRecreateDisks,
447
              instance, inodes, othernodes)
448
    finally:
449
      qa_config.ReleaseManyNodes(acquirednodes)
450

451
452
453
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
454
455


456
457
458
459
460
461
462
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
463
464
465
466
    old_es = qa_cluster.TestSetExclStorCluster(False)
    qa_cluster.TestExclStorSingleNode(node)

    qa_cluster.TestSetExclStorCluster(True)
467
468
    qa_cluster.TestExclStorSharedPv(node)

469
470
471
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
472
473
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
      instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
474
475
476
477
      # cluster-verify checks that disks are allocated correctly
      qa_cluster.AssertClusterVerify()
      qa_instance.TestInstanceRemove(instance1)
      qa_instance.TestInstanceRemove(instance2)
478
479
480
481
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
482
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
483
484
485
486
487
488
        qa_cluster.TestSetExclStorCluster(True)
        exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
        qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
        qa_instance.TestInstanceRemove(instance)
      finally:
        qa_config.ReleaseNode(snode)
489
490
491
492
493
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
    qa_config.ReleaseNode(node)


494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
def RunInstanceTests():
  """Create and exercise instances."""
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)

        RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
        RunDaemonTests(instance)
        for node in inodes:
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
        if len(inodes) > 1:
          RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                    constants.INITIAL_NODE_GROUP_NAME,
                    inodes[0]["primary"], inodes[1]["primary"])
        if qa_config.TestEnabled("instance-convert-disk"):
          RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceConvertDiskToPlain, instance, inodes)
          RunTest(qa_instance.TestInstanceStartup, instance)
        RunCommonInstanceTests(instance)
        RunGroupListTests()
        RunExportImportTests(instance, inodes)
        RunHardwareFailureTests(instance, inodes)
        RunRepairDiskSizes()
        RunTest(qa_instance.TestInstanceRemove, instance)
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
532
533


534
535
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
536
537

  """
538
539
540
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
541
  RunEnvTests()
542
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
543
544

  # Load RAPI certificate
545
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
546

Michael Hanselmann's avatar
Michael Hanselmann committed
547
548
  RunClusterTests()
  RunOsTests()
549

Iustin Pop's avatar
Iustin Pop committed
550
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
551

552
  RunCommonNodeTests()
553
  RunGroupListTests()
554
  RunGroupRwTests()
Helga Velroyen's avatar
Helga Velroyen committed
555
  RunNetworkTests()
556

557
558
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
559
560
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
561
562
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
563
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
564
565
  finally:
    qa_config.ReleaseNode(pnode)
566

567
568
569
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
570
571
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
572
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
573

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
574
575
576
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

577
      if qa_config.TestEnabled("instance-add-plain-disk"):
578
579
580
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
581
582
          if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
            RunCommonInstanceTests(rapi_instance)
583
584
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          del rapi_instance
585

586
587
588
  finally:
    qa_config.ReleaseNode(pnode)

589
590
591
592
593
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
594
  ]
595
596
597
598
599
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
600

601
602
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
603
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
604
      for shutdown in [False, True]:
605
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
606
        expnode = qa_config.AcquireNode(exclude=pnode)
607
        try:
608
609
          if shutdown:
            # Stop instance before exporting and removing it
610
            RunTest(qa_instance.TestInstanceShutdown, instance)
611
612
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
          RunTest(qa_instance.TestBackupList, expnode)
613
        finally:
614
615
616
          qa_config.ReleaseNode(expnode)
        del expnode
        del instance
617
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
618

619
620
621
  finally:
    qa_config.ReleaseNode(pnode)

622
623
  RunExclusiveStorageTests()

624
625
626
627
628
629
  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
630
      try:
631
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
632
        qa_node.MakeNodeOffline(snode, "yes")
633
634
635
636
        try:
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
          qa_node.MakeNodeOffline(snode, "no")
637
      finally:
638
639
640
        qa_config.ReleaseNode(pnode)
    finally:
      qa_config.ReleaseNode(snode)
641
642
    # FIXME: This test leaves a DRBD device and two LVs behind
    # Cluster-verify would fail
Iustin Pop's avatar
Iustin Pop committed
643

Iustin Pop's avatar
Iustin Pop committed
644
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
645

Iustin Pop's avatar
Iustin Pop committed
646
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
647

648

649
@UsesRapiClient
650
651
652
653
654
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
655
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
656
657
                    action="store_true",
                    help="Really execute the tests")
658
  (opts, args) = parser.parse_args()
659
660
661
662
663
664

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

665
  if not opts.yes_do_it:
666
667
668
669
670
671
672
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

673
674
675
676
677
678
  primary = qa_config.GetMasterNode()["primary"]
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
679
680
681
682
683
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
684
if __name__ == "__main__":
685
  main()