ganeti-qa.py 21.5 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
Helga Velroyen's avatar
Helga Velroyen committed
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
44
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import qa_tags
46
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
47

48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

Iustin Pop's avatar
Iustin Pop committed
55

Iustin Pop's avatar
Iustin Pop committed
56
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
61
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
62
63
64
65
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
66
67
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
70
71
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
72
  else:
Iustin Pop's avatar
Iustin Pop committed
73
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
74

Iustin Pop's avatar
Iustin Pop committed
75
76
  return desc.rstrip(".")

77

78
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
79
80
81
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
82

Iustin Pop's avatar
Iustin Pop committed
83
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
84

Iustin Pop's avatar
Iustin Pop committed
85
86
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
87
88
89
90
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
91
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
92
93
94
95
96
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
97
98


99
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
104
105
106
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
108
109
110
111
112
113
114
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
115
116
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
117
118

  """
Iustin Pop's avatar
Iustin Pop committed
119
120
121
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
122

123

124
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
125
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
126

127
128
129
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
130
  """
Iustin Pop's avatar
Iustin Pop committed
131
132
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
133
134
135
136
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
137
138
139
140

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
141
  RunTestIf("job-list", qa_job.TestJobList)
142

Iustin Pop's avatar
Iustin Pop committed
143
144
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
145
146
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
147

Iustin Pop's avatar
Iustin Pop committed
148
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
149

150
151
152
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

153
154
  RunTestIf("node-list", qa_node.TestNodeList)

155
156
157
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
158
  RunTestIf("job-list", qa_job.TestJobListFields)
159
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
160

Iustin Pop's avatar
Iustin Pop committed
161
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
162
163
164
165


def RunClusterTests():
  """Runs tests related to gnt-cluster.
166

Michael Hanselmann's avatar
Michael Hanselmann committed
167
  """
Iustin Pop's avatar
Iustin Pop committed
168
  for test, fn in [
169
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
170
171
172
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
173
    # TODO: add more cluster modify tests
174
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
175
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
176
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
177
178
179
180
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
181
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
182
183
184
185
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
186
187
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
188
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
189
190
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
191
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
192
193
    ]:
    RunTestIf(test, fn)
194

195

196
197
198
199
200
201
202
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
203
204
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
205

Michael Hanselmann's avatar
Michael Hanselmann committed
206
  """
207
208
209
210
211
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
212
213
214
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
215
216
217
218
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
219
220
221
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
222
223
224
225
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
226
227
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
228
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
229
230
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
231
232
233
234
235
236


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
237
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
238
239
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
240
241
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
242
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
243

244
245
246
247
248
249
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
250
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
251

Iustin Pop's avatar
Iustin Pop committed
252
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
253

Iustin Pop's avatar
Iustin Pop committed
254
255
256
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
257

Iustin Pop's avatar
Iustin Pop committed
258
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
259
260
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
261

262
263
264
265
266
267
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
268
269
270
271
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
272
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
273
274
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
275

Iustin Pop's avatar
Iustin Pop committed
276
  if qa_config.TestEnabled("instance-rename"):
277
278
    tgt_instance = qa_config.AcquireInstance()
    try:
279
280
      rename_source = instance.name
      rename_target = tgt_instance.name
281
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
282
      RunTest(qa_instance.TestInstanceRenameAndBack,
283
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
284
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
285
286
287
288
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
289
                rename_source, rename_target)
290
291
292
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
293
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
294

Iustin Pop's avatar
Iustin Pop committed
295
296
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
297
298
299
300
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
301

Iustin Pop's avatar
Iustin Pop committed
302
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
303

Michael Hanselmann's avatar
Michael Hanselmann committed
304
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
305

Iustin Pop's avatar
Iustin Pop committed
306
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
307

308
309
310
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

311
312
313
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

314
315
316
317
318

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
319
320
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
321
  RunTestIf("node-oob", qa_node.TestOutOfBand)
322

323

324
325
326
327
def RunGroupListTests():
  """Run tests for listing node groups.

  """
328
329
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
330
331


Helga Velroyen's avatar
Helga Velroyen committed
332
333
334
335
336
337
338
339
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


340
341
342
343
344
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
345
346
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
347
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
348
349
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
350

351

352
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
353
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
354

355
356
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
357

Michael Hanselmann's avatar
Michael Hanselmann committed
358
  """
Iustin Pop's avatar
Iustin Pop committed
359
  if qa_config.TestEnabled("instance-export"):
360
361
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

362
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
363
364
365
366
367
368
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
369
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
370
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
371
        try:
372
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
373
                  expnode, name)
374
375
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
376
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
377
        finally:
378
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
379
    finally:
380
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
381

Iustin Pop's avatar
Iustin Pop committed
382
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
383
384
    newinst = qa_config.AcquireInstance()
    try:
385
      tnode = qa_config.AcquireNode(exclude=inodes)
386
387
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
388
                inodes, tnode)
389
      finally:
390
        tnode.Release()
391
    finally:
392
      newinst.Release()
393

Michael Hanselmann's avatar
Michael Hanselmann committed
394

395
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
396
  """Test the ganeti-watcher script.
397

Michael Hanselmann's avatar
Michael Hanselmann committed
398
  """
399
  RunTest(qa_daemon.TestPauseWatcher)
400

Iustin Pop's avatar
Iustin Pop committed
401
  RunTestIf("instance-automatic-restart",
402
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
403
  RunTestIf("instance-consecutive-failures",
404
            qa_daemon.TestInstanceConsecutiveFailures, instance)
405

406
407
  RunTest(qa_daemon.TestResumeWatcher)

408

409
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
410
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
411

Michael Hanselmann's avatar
Michael Hanselmann committed
412
  """
Iustin Pop's avatar
Iustin Pop committed
413
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
414
415
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
416

Iustin Pop's avatar
Iustin Pop committed
417
418
419
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
420

Iustin Pop's avatar
Iustin Pop committed
421
  if qa_config.TestEnabled("instance-replace-disks"):
422
423
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
424
    try:
425
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
426
      RunTest(qa_instance.TestReplaceDisks,
427
              instance, inodes, othernodes)
428
    finally:
429
430
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
431

432
433
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
434
435
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
436
    except qa_error.OutOfNodesError:
437
438
439
440
441
442
443
444
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
445
446
    try:
      RunTest(qa_instance.TestRecreateDisks,
447
              instance, inodes, othernodes)
448
    finally:
449
      qa_config.ReleaseManyNodes(acquirednodes)
450

451
452
453
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
454
455


456
457
458
459
460
461
462
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
463
    old_es = qa_cluster.TestSetExclStorCluster(False)
464
    qa_node.TestExclStorSingleNode(node)
465
466

    qa_cluster.TestSetExclStorCluster(True)
467
468
    qa_cluster.TestExclStorSharedPv(node)

469
470
471
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
472
473

      # Create and allocate instances
474
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
475
476
477
478
479
480
481
482
483
484
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
485
          instance2.Release()
486
      finally:
487
        instance1.Release()
488

489
490
491
492
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
493
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
494
495
496
497
498
499
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
500
          instance.Release()
501
      finally:
502
        snode.Release()
503
504
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
505
    node.Release()
506
507


508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
def RunInstanceTests():
  """Create and exercise instances."""
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
523
524
525
526
527
528
529
530
        try:
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
531
                      inodes[0].primary, inodes[1].primary)
532
533
534
535
536
537
538
539
540
541
542
543
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
          RunCommonInstanceTests(instance)
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
544
          instance.Release()
545
546
547
548
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
549
550


551
552
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
553
554

  """
555
556
557
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
558
  RunEnvTests()
559
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
560
561

  # Load RAPI certificate
562
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
563

Michael Hanselmann's avatar
Michael Hanselmann committed
564
565
  RunClusterTests()
  RunOsTests()
566

Iustin Pop's avatar
Iustin Pop committed
567
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
568

569
  RunCommonNodeTests()
570
  RunGroupListTests()
571
  RunGroupRwTests()
Helga Velroyen's avatar
Helga Velroyen committed
572
  RunNetworkTests()
573

574
575
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
576
577
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
578
579
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
580
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
581
  finally:
582
    pnode.Release()
583

584
585
586
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
587
588
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
589
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
590

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
591
592
593
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

594
      if qa_config.TestEnabled("instance-add-plain-disk"):
595
596
597
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
598
599
600
601
602
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
              RunCommonInstanceTests(rapi_instance)
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
603
            rapi_instance.Release()
604
          del rapi_instance
605

606
  finally:
607
    pnode.Release()
608

609
610
611
612
613
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
614
  ]
615
616
617
618
619
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
620

621
622
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
623
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
624
      for shutdown in [False, True]:
625
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
626
        try:
627
628
629
630
631
632
633
634
          expnode = qa_config.AcquireNode(exclude=pnode)
          try:
            if shutdown:
              # Stop instance before exporting and removing it
              RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
            RunTest(qa_instance.TestBackupList, expnode)
          finally:
635
            expnode.Release()
636
        finally:
637
          instance.Release()
638
639
        del expnode
        del instance
640
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
641

642
  finally:
643
    pnode.Release()
644

645
646
  RunExclusiveStorageTests()

647
648
649
650
651
652
  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
653
      try:
654
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
655
656
657
658
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
659
      finally:
660
        pnode.Release()
661
    finally:
662
      snode.Release()
663
    qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
664

Iustin Pop's avatar
Iustin Pop committed
665
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
666

Iustin Pop's avatar
Iustin Pop committed
667
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
668

669

670
@UsesRapiClient
671
672
673
674
675
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
676
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
677
678
                    action="store_true",
                    help="Really execute the tests")
679
  (opts, args) = parser.parse_args()
680
681
682
683
684
685

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

686
  if not opts.yes_do_it:
687
688
689
690
691
692
693
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

694
  primary = qa_config.GetMasterNode().primary
695
696
697
698
699
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
700
701
702
703
704
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
705
if __name__ == "__main__":
706
  main()