ganeti-qa.py 21.1 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
40
import qa_instance
import qa_node
41
import qa_os
42
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
43
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
44
import qa_tags
45
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
46

47
from ganeti import utils
48
from ganeti import rapi # pylint: disable=W0611
49
from ganeti import constants
50

51
import ganeti.rapi.client # pylint: disable=W0611
52
from ganeti.rapi.client import UsesRapiClient
53

Iustin Pop's avatar
Iustin Pop committed
54

Iustin Pop's avatar
Iustin Pop committed
55
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
56
57
58
59
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
60
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
61
62
63
64
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
65
66
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
67
68

  """
69
70
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
71
  else:
Iustin Pop's avatar
Iustin Pop committed
72
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
73

Iustin Pop's avatar
Iustin Pop committed
74
75
  return desc.rstrip(".")

76

77
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
78
79
80
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
81

Iustin Pop's avatar
Iustin Pop committed
82
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
83

Iustin Pop's avatar
Iustin Pop committed
84
85
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
86
87
88
89
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
90
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
91
92
93
94
95
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
96
97


98
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
99
100
101
102
103
104
105
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
106
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
107
108
109
110
111
112
113
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
114
115
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
116
117

  """
Iustin Pop's avatar
Iustin Pop committed
118
119
120
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
121

122

123
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
124
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
125

126
127
128
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
129
  """
Iustin Pop's avatar
Iustin Pop committed
130
131
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
132
133
134
135
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
136
137
138
139

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
140
  RunTestIf("job-list", qa_job.TestJobList)
141

Iustin Pop's avatar
Iustin Pop committed
142
143
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
144
145
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
146

Iustin Pop's avatar
Iustin Pop committed
147
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
148

149
150
151
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

152
153
  RunTestIf("node-list", qa_node.TestNodeList)

154
155
156
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
157
  RunTestIf("job-list", qa_job.TestJobListFields)
158
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
159

Iustin Pop's avatar
Iustin Pop committed
160
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
161
162
163
164


def RunClusterTests():
  """Runs tests related to gnt-cluster.
165

Michael Hanselmann's avatar
Michael Hanselmann committed
166
  """
Iustin Pop's avatar
Iustin Pop committed
167
  for test, fn in [
168
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
169
170
171
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
172
    # TODO: add more cluster modify tests
173
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
174
175
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
Iustin Pop's avatar
Iustin Pop committed
176
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
177
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
178
179
180
181
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
182
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
183
184
185
186
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
187
188
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
189
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
190
191
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
192
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
193
194
    ]:
    RunTestIf(test, fn)
195

196

197
198
199
200
201
202
203
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
204
205
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
206

Michael Hanselmann's avatar
Michael Hanselmann committed
207
  """
208
209
210
211
212
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
213
214
215
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
216
217
218
219
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
220
221
222
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
223
224
225
226
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
227
228
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
229
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
230
231
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
232
233
234
235
236
237


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
238
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
239
240
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
241
242
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
243
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
244

245
246
247
248
249
250
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
251
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
252

Iustin Pop's avatar
Iustin Pop committed
253
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
254

Iustin Pop's avatar
Iustin Pop committed
255
256
257
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
258

Iustin Pop's avatar
Iustin Pop committed
259
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
260
261
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
262

263
264
265
266
267
268
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
269
270
271
272
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
273
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
274
275
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
276

Iustin Pop's avatar
Iustin Pop committed
277
  if qa_config.TestEnabled("instance-rename"):
278
279
280
281
282
    tgt_instance = qa_config.AcquireInstance()
    try:
      rename_source = instance["name"]
      rename_target = tgt_instance["name"]
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
283
      RunTest(qa_instance.TestInstanceRenameAndBack,
284
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
285
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
286
287
288
289
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
290
                rename_source, rename_target)
291
292
293
294
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
      qa_config.ReleaseInstance(tgt_instance)
Iustin Pop's avatar
Iustin Pop committed
295

Iustin Pop's avatar
Iustin Pop committed
296
297
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
298
299
300
301
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
302

Iustin Pop's avatar
Iustin Pop committed
303
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
304

Michael Hanselmann's avatar
Michael Hanselmann committed
305
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
306

Iustin Pop's avatar
Iustin Pop committed
307
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
308

309
310
311
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

312
313
314
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

315
316
317
318
319

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
320
321
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
322
  RunTestIf("node-oob", qa_node.TestOutOfBand)
323

324

325
326
327
328
def RunGroupListTests():
  """Run tests for listing node groups.

  """
329
330
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
331
332


333
334
335
336
337
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
338
339
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
340
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
341
342
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
343

344

345
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
346
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
347

348
349
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
350

Michael Hanselmann's avatar
Michael Hanselmann committed
351
  """
Iustin Pop's avatar
Iustin Pop committed
352
  if qa_config.TestEnabled("instance-export"):
353
354
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

355
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
356
357
358
359
360
361
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
362
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
363
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
364
        try:
365
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
366
                  expnode, name)
367
368
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
369
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
370
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
371
372
373
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
374

Iustin Pop's avatar
Iustin Pop committed
375
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
376
377
    newinst = qa_config.AcquireInstance()
    try:
378
      tnode = qa_config.AcquireNode(exclude=inodes)
379
380
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
381
                inodes, tnode)
382
      finally:
383
        qa_config.ReleaseNode(tnode)
384
385
386
    finally:
      qa_config.ReleaseInstance(newinst)

Michael Hanselmann's avatar
Michael Hanselmann committed
387

388
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
389
  """Test the ganeti-watcher script.
390

Michael Hanselmann's avatar
Michael Hanselmann committed
391
  """
392
  RunTest(qa_daemon.TestPauseWatcher)
393

Iustin Pop's avatar
Iustin Pop committed
394
  RunTestIf("instance-automatic-restart",
395
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
396
  RunTestIf("instance-consecutive-failures",
397
            qa_daemon.TestInstanceConsecutiveFailures, instance)
398

399
400
  RunTest(qa_daemon.TestResumeWatcher)

401

402
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
403
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
404

Michael Hanselmann's avatar
Michael Hanselmann committed
405
  """
Iustin Pop's avatar
Iustin Pop committed
406
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
407
408
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
409

Iustin Pop's avatar
Iustin Pop committed
410
411
412
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
413

Iustin Pop's avatar
Iustin Pop committed
414
  if qa_config.TestEnabled("instance-replace-disks"):
415
416
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
417
    try:
418
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
419
      RunTest(qa_instance.TestReplaceDisks,
420
              instance, inodes, othernodes)
421
    finally:
422
423
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
424

425
426
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
427
428
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
429
    except qa_error.OutOfNodesError:
430
431
432
433
434
435
436
437
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
438
439
    try:
      RunTest(qa_instance.TestRecreateDisks,
440
              instance, inodes, othernodes)
441
    finally:
442
      qa_config.ReleaseManyNodes(acquirednodes)
Michael Hanselmann's avatar
Michael Hanselmann committed
443

444
445
446
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
447
448


449
450
451
452
453
454
455
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
456
    old_es = qa_cluster.TestSetExclStorCluster(False)
457
    qa_node.TestExclStorSingleNode(node)
458
459

    qa_cluster.TestSetExclStorCluster(True)
460
461
    qa_cluster.TestExclStorSharedPv(node)

462
463
464
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
465
466
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
      instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
467
468
469
470
      # cluster-verify checks that disks are allocated correctly
      qa_cluster.AssertClusterVerify()
      qa_instance.TestInstanceRemove(instance1)
      qa_instance.TestInstanceRemove(instance2)
471
472
473
474
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
475
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
476
477
478
479
480
481
        qa_cluster.TestSetExclStorCluster(True)
        exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
        qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
        qa_instance.TestInstanceRemove(instance)
      finally:
        qa_config.ReleaseNode(snode)
482
483
484
485
486
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
    qa_config.ReleaseNode(node)


487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
def RunInstanceTests():
  """Create and exercise instances."""
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)

        RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
        RunDaemonTests(instance)
        for node in inodes:
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
        if len(inodes) > 1:
          RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                    constants.INITIAL_NODE_GROUP_NAME,
                    inodes[0]["primary"], inodes[1]["primary"])
        if qa_config.TestEnabled("instance-convert-disk"):
          RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceConvertDiskToPlain, instance, inodes)
          RunTest(qa_instance.TestInstanceStartup, instance)
        RunCommonInstanceTests(instance)
        RunGroupListTests()
        RunExportImportTests(instance, inodes)
        RunHardwareFailureTests(instance, inodes)
        RunRepairDiskSizes()
        RunTest(qa_instance.TestInstanceRemove, instance)
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()


527
528
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
529
530

  """
531
532
533
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
534
  RunEnvTests()
535
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
536
537

  # Load RAPI certificate
538
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
539

Michael Hanselmann's avatar
Michael Hanselmann committed
540
541
  RunClusterTests()
  RunOsTests()
542

Iustin Pop's avatar
Iustin Pop committed
543
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
544

545
  RunCommonNodeTests()
546
  RunGroupListTests()
547
  RunGroupRwTests()
548

549
550
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
551
552
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
553
554
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
555
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
556
557
  finally:
    qa_config.ReleaseNode(pnode)
558

559
560
561
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
562
563
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
564
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
565

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
566
567
568
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

569
      if qa_config.TestEnabled("instance-add-plain-disk"):
570
571
572
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
573
574
          if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
            RunCommonInstanceTests(rapi_instance)
575
576
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          del rapi_instance
577

578
579
580
  finally:
    qa_config.ReleaseNode(pnode)

581
582
583
584
585
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
586
  ]
587
588
589
590
591
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
592

593
594
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
595
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
596
      for shutdown in [False, True]:
597
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
598
599
600
601
602
603
604
605
606
607
608
        expnode = qa_config.AcquireNode(exclude=pnode)
        try:
          if shutdown:
            # Stop instance before exporting and removing it
            RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
          RunTest(qa_instance.TestBackupList, expnode)
        finally:
          qa_config.ReleaseNode(expnode)
        del expnode
        del instance
609
      qa_cluster.AssertClusterVerify()
610

Iustin Pop's avatar
Iustin Pop committed
611
  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
612
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
613

614
615
  RunExclusiveStorageTests()

616
617
618
619
620
621
622
623
  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
      try:
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
624
625
626
627
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
628
629
630
631
      finally:
        qa_config.ReleaseNode(pnode)
    finally:
      qa_config.ReleaseNode(snode)
632
    qa_cluster.AssertClusterVerify()
633

Iustin Pop's avatar
Iustin Pop committed
634
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
635

Iustin Pop's avatar
Iustin Pop committed
636
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
637

638

639
@UsesRapiClient
640
641
642
643
644
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
645
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
646
647
                    action="store_true",
                    help="Really execute the tests")
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
  (qa_config.options, args) = parser.parse_args()

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

663
664
665
666
667
668
  primary = qa_config.GetMasterNode()["primary"]
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
669
670
671
672
673
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
674
if __name__ == "__main__":
675
  main()