ganeti-qa.py 21.6 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
Helga Velroyen's avatar
Helga Velroyen committed
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
44
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import qa_tags
46
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
47

48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

Iustin Pop's avatar
Iustin Pop committed
55

Iustin Pop's avatar
Iustin Pop committed
56
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
61
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
62
63
64
65
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
66
67
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
70
71
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
72
  else:
Iustin Pop's avatar
Iustin Pop committed
73
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
74

Iustin Pop's avatar
Iustin Pop committed
75
76
  return desc.rstrip(".")

77

78
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
79
80
81
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
82

Iustin Pop's avatar
Iustin Pop committed
83
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
84

Iustin Pop's avatar
Iustin Pop committed
85
86
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
87
88
89
90
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
91
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
92
93
94
95
96
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
97
98


99
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
104
105
106
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
108
109
110
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
111
    # TODO: Formatting test names when non-string names are involved
Iustin Pop's avatar
Iustin Pop committed
112
113
114
115
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
116
117
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
118
119

  """
Iustin Pop's avatar
Iustin Pop committed
120
121
122
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
123

124

125
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
126
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
127

128
129
130
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
131
  """
Iustin Pop's avatar
Iustin Pop committed
132
133
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
134
135
136
137
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
138
139
140
141

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
142
  RunTestIf("job-list", qa_job.TestJobList)
143

Iustin Pop's avatar
Iustin Pop committed
144
145
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
146
147
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
148

Iustin Pop's avatar
Iustin Pop committed
149
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
150

151
152
153
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

154
155
  RunTestIf("node-list", qa_node.TestNodeList)

156
157
158
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
159
  RunTestIf("job-list", qa_job.TestJobListFields)
160
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
161

Iustin Pop's avatar
Iustin Pop committed
162
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
163
164
165
166


def RunClusterTests():
  """Runs tests related to gnt-cluster.
167

Michael Hanselmann's avatar
Michael Hanselmann committed
168
  """
Iustin Pop's avatar
Iustin Pop committed
169
  for test, fn in [
170
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
171
172
173
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
174
    # TODO: add more cluster modify tests
175
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
176
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
177
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
178
179
180
181
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
182
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
183
184
185
186
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
187
188
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
189
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
190
191
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
192
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
193
194
    ]:
    RunTestIf(test, fn)
195

196

197
198
199
200
201
202
203
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
204
205
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
206

Michael Hanselmann's avatar
Michael Hanselmann committed
207
  """
208
209
  os_enabled = ["os", qa_config.NoVirtualCluster]

210
211
212
213
214
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
215
216
217
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
218
    ]:
219
    RunTestIf(os_enabled, fn)
220
221

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
222
223
224
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
225
    ]:
226
    RunTestIf(os_enabled, fn, rapi_getos)
227
228

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
229
230
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
231
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
232
    ]:
233
    RunTestIf(os_enabled, fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
234
235
236
237
238
239


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
240
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
241
242
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
243
244
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
245
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
246

247
248
249
250
251
252
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
253
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
254

Iustin Pop's avatar
Iustin Pop committed
255
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
256

Iustin Pop's avatar
Iustin Pop committed
257
258
259
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
260

Iustin Pop's avatar
Iustin Pop committed
261
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
262
263
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
264

265
266
267
268
269
270
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
271
272
273
274
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
275
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
276
277
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
278

Iustin Pop's avatar
Iustin Pop committed
279
  if qa_config.TestEnabled("instance-rename"):
280
281
    tgt_instance = qa_config.AcquireInstance()
    try:
282
283
      rename_source = instance.name
      rename_target = tgt_instance.name
284
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
285
      RunTest(qa_instance.TestInstanceRenameAndBack,
286
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
287
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
288
289
290
291
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
292
                rename_source, rename_target)
293
294
295
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
296
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
297

Iustin Pop's avatar
Iustin Pop committed
298
299
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
300
301
302
303
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
304

Iustin Pop's avatar
Iustin Pop committed
305
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
306

Michael Hanselmann's avatar
Michael Hanselmann committed
307
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
308

Iustin Pop's avatar
Iustin Pop committed
309
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
310

311
312
313
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

314
315
316
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

317
318
319
320
321

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
322
323
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
324
  RunTestIf("node-oob", qa_node.TestOutOfBand)
325

326

327
328
329
330
def RunGroupListTests():
  """Run tests for listing node groups.

  """
331
332
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
333
334


Helga Velroyen's avatar
Helga Velroyen committed
335
336
337
338
339
340
341
342
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


343
344
345
346
347
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
348
349
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
350
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
351
352
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
353

354

355
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
356
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
357

358
359
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
360

Michael Hanselmann's avatar
Michael Hanselmann committed
361
  """
Iustin Pop's avatar
Iustin Pop committed
362
  if qa_config.TestEnabled("instance-export"):
363
364
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

365
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
366
367
368
369
370
371
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
372
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
373
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
374
        try:
375
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
376
                  expnode, name)
377
378
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
379
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
380
        finally:
381
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
382
    finally:
383
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
384

Iustin Pop's avatar
Iustin Pop committed
385
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
386
387
    newinst = qa_config.AcquireInstance()
    try:
388
      tnode = qa_config.AcquireNode(exclude=inodes)
389
390
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
391
                inodes, tnode)
392
      finally:
393
        tnode.Release()
394
    finally:
395
      newinst.Release()
396

Michael Hanselmann's avatar
Michael Hanselmann committed
397

398
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
399
  """Test the ganeti-watcher script.
400

Michael Hanselmann's avatar
Michael Hanselmann committed
401
  """
402
  RunTest(qa_daemon.TestPauseWatcher)
403

Iustin Pop's avatar
Iustin Pop committed
404
  RunTestIf("instance-automatic-restart",
405
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
406
  RunTestIf("instance-consecutive-failures",
407
            qa_daemon.TestInstanceConsecutiveFailures, instance)
408

409
410
  RunTest(qa_daemon.TestResumeWatcher)

411

412
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
413
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
414

Michael Hanselmann's avatar
Michael Hanselmann committed
415
  """
Iustin Pop's avatar
Iustin Pop committed
416
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
417
418
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
419

Iustin Pop's avatar
Iustin Pop committed
420
421
422
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
423

Iustin Pop's avatar
Iustin Pop committed
424
  if qa_config.TestEnabled("instance-replace-disks"):
425
426
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
427
    try:
428
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
429
      RunTest(qa_instance.TestReplaceDisks,
430
              instance, inodes, othernodes)
431
    finally:
432
433
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
434

435
436
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
437
438
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
439
    except qa_error.OutOfNodesError:
440
441
442
443
444
445
446
447
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
448
449
    try:
      RunTest(qa_instance.TestRecreateDisks,
450
              instance, inodes, othernodes)
451
    finally:
452
      qa_config.ReleaseManyNodes(acquirednodes)
453

454
455
456
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
457
458


459
460
461
462
463
464
465
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
466
    old_es = qa_cluster.TestSetExclStorCluster(False)
467
    qa_node.TestExclStorSingleNode(node)
468
469

    qa_cluster.TestSetExclStorCluster(True)
470
471
    qa_cluster.TestExclStorSharedPv(node)

472
473
474
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
475
476

      # Create and allocate instances
477
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
478
479
480
481
482
483
484
485
486
487
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
488
          instance2.Release()
489
      finally:
490
        instance1.Release()
491

492
493
494
495
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
496
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
497
498
499
500
501
502
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
503
          instance.Release()
504
      finally:
505
        snode.Release()
506
507
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
508
    node.Release()
509
510


511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
def RunInstanceTests():
  """Create and exercise instances."""
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
526
527
528
529
530
531
532
533
        try:
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
534
                      inodes[0].primary, inodes[1].primary)
535
536
537
538
539
540
541
542
543
544
545
546
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
          RunCommonInstanceTests(instance)
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
547
          instance.Release()
548
549
550
551
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
552
553


554
555
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
556
557

  """
558
559
560
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
561
  RunEnvTests()
562
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
563
564

  # Load RAPI certificate
565
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
566

Michael Hanselmann's avatar
Michael Hanselmann committed
567
568
  RunClusterTests()
  RunOsTests()
569

Iustin Pop's avatar
Iustin Pop committed
570
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
571

572
  RunCommonNodeTests()
573
  RunGroupListTests()
574
  RunGroupRwTests()
Helga Velroyen's avatar
Helga Velroyen committed
575
  RunNetworkTests()
576

577
578
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
579
580
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
581
582
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
583
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
584
  finally:
585
    pnode.Release()
586

587
588
589
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
590
591
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
592
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
593

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
594
595
596
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

597
      if qa_config.TestEnabled("instance-add-plain-disk"):
598
599
600
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
601
602
603
604
605
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
              RunCommonInstanceTests(rapi_instance)
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
606
            rapi_instance.Release()
607
          del rapi_instance
608

609
  finally:
610
    pnode.Release()
611

612
613
614
615
616
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
617
  ]
618
619
620
621
622
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
623

624
625
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
626
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
627
      for shutdown in [False, True]:
628
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
629
        try:
630
631
632
633
634
635
636
637
          expnode = qa_config.AcquireNode(exclude=pnode)
          try:
            if shutdown:
              # Stop instance before exporting and removing it
              RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
            RunTest(qa_instance.TestBackupList, expnode)
          finally:
638
            expnode.Release()
639
        finally:
640
          instance.Release()
641
642
        del expnode
        del instance
643
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
644

645
  finally:
646
    pnode.Release()
647

648
649
  RunExclusiveStorageTests()

650
651
652
653
654
655
  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
656
      try:
657
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
658
659
660
661
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
662
      finally:
663
        pnode.Release()
664
    finally:
665
      snode.Release()
666
    qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
667

Iustin Pop's avatar
Iustin Pop committed
668
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
669

Iustin Pop's avatar
Iustin Pop committed
670
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
671

672

673
@UsesRapiClient
674
675
676
677
678
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
679
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
680
681
                    action="store_true",
                    help="Really execute the tests")
682
  (opts, args) = parser.parse_args()
683
684
685
686
687
688

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

689
  if not opts.yes_do_it:
690
691
692
693
694
695
696
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

697
  primary = qa_config.GetMasterNode().primary
698
699
700
701
702
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
703
704
705
706
707
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
708
if __name__ == "__main__":
709
  main()