ganeti-qa.py 21.6 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
Helga Velroyen's avatar
Helga Velroyen committed
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
44
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import qa_tags
46
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
47

48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

Iustin Pop's avatar
Iustin Pop committed
55

Iustin Pop's avatar
Iustin Pop committed
56
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
61
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
62
63
64
65
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
66
67
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
70
71
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
72
  else:
Iustin Pop's avatar
Iustin Pop committed
73
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
74

Iustin Pop's avatar
Iustin Pop committed
75
76
  return desc.rstrip(".")

77

78
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
79
80
81
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
82

Iustin Pop's avatar
Iustin Pop committed
83
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
84

Iustin Pop's avatar
Iustin Pop committed
85
86
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
87
88
89
90
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
91
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
92
93
94
95
96
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
97
98


99
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
104
105
106
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
108
109
110
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
111
    # TODO: Formatting test names when non-string names are involved
Iustin Pop's avatar
Iustin Pop committed
112
113
114
115
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
116
117
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
118
119

  """
Iustin Pop's avatar
Iustin Pop committed
120
121
122
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
123

124

125
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
126
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
127

128
129
130
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
131
  """
Iustin Pop's avatar
Iustin Pop committed
132
133
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
134
135
136
137
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
138
139
140
141

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
142
  RunTestIf("job-list", qa_job.TestJobList)
143

Iustin Pop's avatar
Iustin Pop committed
144
145
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
146
147
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
148

Iustin Pop's avatar
Iustin Pop committed
149
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
150

151
152
153
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

154
155
  RunTestIf("node-list", qa_node.TestNodeList)

156
157
158
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
159
  RunTestIf("job-list", qa_job.TestJobListFields)
160
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
161

Iustin Pop's avatar
Iustin Pop committed
162
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
163
164
165
166


def RunClusterTests():
  """Runs tests related to gnt-cluster.
167

Michael Hanselmann's avatar
Michael Hanselmann committed
168
  """
Iustin Pop's avatar
Iustin Pop committed
169
  for test, fn in [
170
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
171
172
173
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
174
    # TODO: add more cluster modify tests
175
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
176
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
177
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
178
179
180
181
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
182
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
183
184
185
186
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
187
188
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
189
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
190
191
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
192
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
193
194
    ]:
    RunTestIf(test, fn)
195

196

197
198
199
200
201
202
203
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
204
205
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
206

Michael Hanselmann's avatar
Michael Hanselmann committed
207
  """
208
209
210
211
212
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
213
214
215
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
216
217
218
219
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
220
221
222
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
223
224
225
226
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
227
228
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
229
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
230
231
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
232
233
234
235
236
237


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
238
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
239
240
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
241
242
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
243
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
244

245
246
247
248
249
250
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
251
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
252

Iustin Pop's avatar
Iustin Pop committed
253
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
254

Iustin Pop's avatar
Iustin Pop committed
255
256
257
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
258

Iustin Pop's avatar
Iustin Pop committed
259
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
260
261
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
262

263
264
265
266
267
268
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
269
270
271
272
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
273
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
274
275
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
276

Iustin Pop's avatar
Iustin Pop committed
277
  if qa_config.TestEnabled("instance-rename"):
278
279
    tgt_instance = qa_config.AcquireInstance()
    try:
280
281
      rename_source = instance.name
      rename_target = tgt_instance.name
282
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
283
      RunTest(qa_instance.TestInstanceRenameAndBack,
284
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
285
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
286
287
288
289
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
290
                rename_source, rename_target)
291
292
293
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
294
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
295

Iustin Pop's avatar
Iustin Pop committed
296
297
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
298
299
300
301
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
302

Iustin Pop's avatar
Iustin Pop committed
303
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
304

Michael Hanselmann's avatar
Michael Hanselmann committed
305
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
306

Iustin Pop's avatar
Iustin Pop committed
307
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
308

309
310
311
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

312
313
314
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

315
316
317
318
319

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
320
321
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
322
  RunTestIf("node-oob", qa_node.TestOutOfBand)
323

324

325
326
327
328
def RunGroupListTests():
  """Run tests for listing node groups.

  """
329
330
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
331
332


Helga Velroyen's avatar
Helga Velroyen committed
333
334
335
336
337
338
339
340
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


341
342
343
344
345
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
346
347
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
348
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
349
350
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
351

352

353
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
354
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
355

356
357
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
358

Michael Hanselmann's avatar
Michael Hanselmann committed
359
  """
Iustin Pop's avatar
Iustin Pop committed
360
  if qa_config.TestEnabled("instance-export"):
361
362
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

363
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
364
365
366
367
368
369
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
370
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
371
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
372
        try:
373
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
374
                  expnode, name)
375
376
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
377
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
378
        finally:
379
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
380
    finally:
381
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
382

Iustin Pop's avatar
Iustin Pop committed
383
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
384
385
    newinst = qa_config.AcquireInstance()
    try:
386
      tnode = qa_config.AcquireNode(exclude=inodes)
387
388
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
389
                inodes, tnode)
390
      finally:
391
        tnode.Release()
392
    finally:
393
      newinst.Release()
394

Michael Hanselmann's avatar
Michael Hanselmann committed
395

396
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
397
  """Test the ganeti-watcher script.
398

Michael Hanselmann's avatar
Michael Hanselmann committed
399
  """
400
  RunTest(qa_daemon.TestPauseWatcher)
401

Iustin Pop's avatar
Iustin Pop committed
402
  RunTestIf("instance-automatic-restart",
403
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
404
  RunTestIf("instance-consecutive-failures",
405
            qa_daemon.TestInstanceConsecutiveFailures, instance)
406

407
408
  RunTest(qa_daemon.TestResumeWatcher)

409

410
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
411
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
412

Michael Hanselmann's avatar
Michael Hanselmann committed
413
  """
Iustin Pop's avatar
Iustin Pop committed
414
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
415
416
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
417

Iustin Pop's avatar
Iustin Pop committed
418
419
420
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
421

Iustin Pop's avatar
Iustin Pop committed
422
  if qa_config.TestEnabled("instance-replace-disks"):
423
424
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
425
    try:
426
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
427
      RunTest(qa_instance.TestReplaceDisks,
428
              instance, inodes, othernodes)
429
    finally:
430
431
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
432

433
434
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
435
436
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
437
    except qa_error.OutOfNodesError:
438
439
440
441
442
443
444
445
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
446
447
    try:
      RunTest(qa_instance.TestRecreateDisks,
448
              instance, inodes, othernodes)
449
    finally:
450
      qa_config.ReleaseManyNodes(acquirednodes)
451

452
453
454
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
455
456


457
458
459
460
461
462
463
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
464
    old_es = qa_cluster.TestSetExclStorCluster(False)
465
    qa_node.TestExclStorSingleNode(node)
466
467

    qa_cluster.TestSetExclStorCluster(True)
468
469
    qa_cluster.TestExclStorSharedPv(node)

470
471
472
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
473
474

      # Create and allocate instances
475
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
476
477
478
479
480
481
482
483
484
485
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
486
          instance2.Release()
487
      finally:
488
        instance1.Release()
489

490
491
492
493
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
494
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
495
496
497
498
499
500
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
501
          instance.Release()
502
      finally:
503
        snode.Release()
504
505
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
506
    node.Release()
507
508


509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
def RunInstanceTests():
  """Create and exercise instances."""
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
524
525
526
527
528
529
530
531
        try:
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
532
                      inodes[0].primary, inodes[1].primary)
533
534
535
536
537
538
539
540
541
542
543
544
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
          RunCommonInstanceTests(instance)
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
545
          instance.Release()
546
547
548
549
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
550
551


552
553
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
554
555

  """
556
557
558
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
559
  RunEnvTests()
560
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
561
562

  # Load RAPI certificate
563
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
564

Michael Hanselmann's avatar
Michael Hanselmann committed
565
566
  RunClusterTests()
  RunOsTests()
567

Iustin Pop's avatar
Iustin Pop committed
568
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
569

570
  RunCommonNodeTests()
571
  RunGroupListTests()
572
  RunGroupRwTests()
Helga Velroyen's avatar
Helga Velroyen committed
573
  RunNetworkTests()
574

575
576
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
577
578
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
579
580
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
581
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
582
  finally:
583
    pnode.Release()
584

585
586
587
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
588
589
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
590
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
591

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
592
593
594
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

595
      if qa_config.TestEnabled("instance-add-plain-disk"):
596
597
598
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
599
600
601
602
603
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
              RunCommonInstanceTests(rapi_instance)
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
604
            rapi_instance.Release()
605
          del rapi_instance
606

607
  finally:
608
    pnode.Release()
609

610
611
612
613
614
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
615
  ]
616
617
618
619
620
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
621

622
623
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
624
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
625
      for shutdown in [False, True]:
626
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
627
        try:
628
629
630
631
632
633
634
635
          expnode = qa_config.AcquireNode(exclude=pnode)
          try:
            if shutdown:
              # Stop instance before exporting and removing it
              RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
            RunTest(qa_instance.TestBackupList, expnode)
          finally:
636
            expnode.Release()
637
        finally:
638
          instance.Release()
639
640
        del expnode
        del instance
641
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
642

643
  finally:
644
    pnode.Release()
645

646
647
  RunExclusiveStorageTests()

648
649
650
651
652
653
  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
654
      try:
655
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
656
657
658
659
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
660
      finally:
661
        pnode.Release()
662
    finally:
663
      snode.Release()
664
    qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
665

Iustin Pop's avatar
Iustin Pop committed
666
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
667

Iustin Pop's avatar
Iustin Pop committed
668
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
669

670

671
@UsesRapiClient
672
673
674
675
676
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
677
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
678
679
                    action="store_true",
                    help="Really execute the tests")
680
  (opts, args) = parser.parse_args()
681
682
683
684
685
686

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

687
  if not opts.yes_do_it:
688
689
690
691
692
693
694
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

695
  primary = qa_config.GetMasterNode().primary
696
697
698
699
700
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
701
702
703
704
705
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
706
if __name__ == "__main__":
707
  main()