ganeti-qa.py 20.5 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
40
import qa_instance
import qa_node
41
import qa_os
42
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
43
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
44
import qa_tags
45
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
46

47
from ganeti import utils
48
from ganeti import rapi # pylint: disable=W0611
49
from ganeti import constants
50

51
import ganeti.rapi.client # pylint: disable=W0611
52
from ganeti.rapi.client import UsesRapiClient
53

Iustin Pop's avatar
Iustin Pop committed
54

Iustin Pop's avatar
Iustin Pop committed
55
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
56
57
58
59
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
60
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
61
62
63
64
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
65
66
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
67
68

  """
69
70
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
71
  else:
Iustin Pop's avatar
Iustin Pop committed
72
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
73

Iustin Pop's avatar
Iustin Pop committed
74
75
  return desc.rstrip(".")

76

77
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
78
79
80
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
81

Iustin Pop's avatar
Iustin Pop committed
82
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
83

Iustin Pop's avatar
Iustin Pop committed
84
85
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
86
87
88
89
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
90
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
91
92
93
94
95
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
96
97


98
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
99
100
101
102
103
104
105
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
106
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
107
108
109
110
111
112
113
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
114
115
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
116
117

  """
Iustin Pop's avatar
Iustin Pop committed
118
119
120
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
121

122

123
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
124
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
125

126
127
128
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
129
  """
Iustin Pop's avatar
Iustin Pop committed
130
131
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
132
133
134
135
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
136
137
138
139

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
140
  RunTestIf("job-list", qa_job.TestJobList)
141

Iustin Pop's avatar
Iustin Pop committed
142
143
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
144
145
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
146

Iustin Pop's avatar
Iustin Pop committed
147
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
148

149
150
151
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

152
153
  RunTestIf("node-list", qa_node.TestNodeList)

154
155
156
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
157
  RunTestIf("job-list", qa_job.TestJobListFields)
158
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
159

Iustin Pop's avatar
Iustin Pop committed
160
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
161
162
163
164


def RunClusterTests():
  """Runs tests related to gnt-cluster.
165

Michael Hanselmann's avatar
Michael Hanselmann committed
166
  """
Iustin Pop's avatar
Iustin Pop committed
167
  for test, fn in [
168
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
169
170
171
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
172
    # TODO: add more cluster modify tests
173
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
174
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
175
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
176
177
178
179
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
180
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
181
182
183
184
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
185
186
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
187
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
188
189
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
190
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
191
192
    ]:
    RunTestIf(test, fn)
193

194

195
196
197
198
199
200
201
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
202
203
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
204

Michael Hanselmann's avatar
Michael Hanselmann committed
205
  """
206
207
208
209
210
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
211
212
213
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
214
215
216
217
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
218
219
220
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
221
222
223
224
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
225
226
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
227
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
228
229
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
230
231
232
233
234
235


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
236
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
237
238
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
239
240
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
241
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
242

243
244
245
246
247
248
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
249
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
250

Iustin Pop's avatar
Iustin Pop committed
251
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
252

Iustin Pop's avatar
Iustin Pop committed
253
254
255
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
256

Iustin Pop's avatar
Iustin Pop committed
257
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
258
259
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
260

261
262
263
264
265
266
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
267
268
269
270
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
271
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
272
273
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
274

Iustin Pop's avatar
Iustin Pop committed
275
  if qa_config.TestEnabled("instance-rename"):
276
277
278
279
280
    tgt_instance = qa_config.AcquireInstance()
    try:
      rename_source = instance["name"]
      rename_target = tgt_instance["name"]
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
281
      RunTest(qa_instance.TestInstanceRenameAndBack,
282
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
283
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
284
285
286
287
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
288
                rename_source, rename_target)
289
290
291
292
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
      qa_config.ReleaseInstance(tgt_instance)
Iustin Pop's avatar
Iustin Pop committed
293

Iustin Pop's avatar
Iustin Pop committed
294
295
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
296
297
298
299
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
300

Iustin Pop's avatar
Iustin Pop committed
301
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
302

Michael Hanselmann's avatar
Michael Hanselmann committed
303
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
304

Iustin Pop's avatar
Iustin Pop committed
305
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
306

307
308
309
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

310
311
312
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

313
314
315
316
317

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
318
319
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
320
  RunTestIf("node-oob", qa_node.TestOutOfBand)
321

322

323
324
325
326
def RunGroupListTests():
  """Run tests for listing node groups.

  """
327
328
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
329
330


331
332
333
334
335
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
336
337
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
338
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
339
340
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
341

342

343
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
344
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
345

346
347
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
348

Michael Hanselmann's avatar
Michael Hanselmann committed
349
  """
Iustin Pop's avatar
Iustin Pop committed
350
  if qa_config.TestEnabled("instance-export"):
351
352
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

353
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
354
355
356
357
358
359
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
360
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
361
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
362
        try:
363
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
364
                  expnode, name)
365
366
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
367
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
368
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
369
370
371
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
372

Iustin Pop's avatar
Iustin Pop committed
373
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
374
375
    newinst = qa_config.AcquireInstance()
    try:
376
      tnode = qa_config.AcquireNode(exclude=inodes)
377
378
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
379
                inodes, tnode)
380
      finally:
381
        qa_config.ReleaseNode(tnode)
382
383
384
    finally:
      qa_config.ReleaseInstance(newinst)

Michael Hanselmann's avatar
Michael Hanselmann committed
385

386
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
387
  """Test the ganeti-watcher script.
388

Michael Hanselmann's avatar
Michael Hanselmann committed
389
  """
390
  RunTest(qa_daemon.TestPauseWatcher)
391

Iustin Pop's avatar
Iustin Pop committed
392
  RunTestIf("instance-automatic-restart",
393
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
394
  RunTestIf("instance-consecutive-failures",
395
            qa_daemon.TestInstanceConsecutiveFailures, instance)
396

397
398
  RunTest(qa_daemon.TestResumeWatcher)

399

400
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
401
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
402

Michael Hanselmann's avatar
Michael Hanselmann committed
403
  """
Iustin Pop's avatar
Iustin Pop committed
404
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
405
406
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
407

Iustin Pop's avatar
Iustin Pop committed
408
409
410
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
411

Iustin Pop's avatar
Iustin Pop committed
412
  if qa_config.TestEnabled("instance-replace-disks"):
413
414
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
415
    try:
416
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
417
      RunTest(qa_instance.TestReplaceDisks,
418
              instance, inodes, othernodes)
419
    finally:
420
421
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
422

423
424
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
425
426
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
427
    except qa_error.OutOfNodesError:
428
429
430
431
432
433
434
435
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
436
437
    try:
      RunTest(qa_instance.TestRecreateDisks,
438
              instance, inodes, othernodes)
439
    finally:
440
      qa_config.ReleaseManyNodes(acquirednodes)
Michael Hanselmann's avatar
Michael Hanselmann committed
441

442
443
444
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
445
446


447
448
449
450
451
452
453
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
454
455
456
457
    old_es = qa_cluster.TestSetExclStorCluster(False)
    qa_cluster.TestExclStorSingleNode(node)

    qa_cluster.TestSetExclStorCluster(True)
458
459
    qa_cluster.TestExclStorSharedPv(node)

460
461
462
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
463
464
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
      instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
465
466
467
468
      # cluster-verify checks that disks are allocated correctly
      qa_cluster.AssertClusterVerify()
      qa_instance.TestInstanceRemove(instance1)
      qa_instance.TestInstanceRemove(instance2)
469
470
471
472
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
473
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
474
475
476
477
478
479
        qa_cluster.TestSetExclStorCluster(True)
        exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
        qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
        qa_instance.TestInstanceRemove(instance)
      finally:
        qa_config.ReleaseNode(snode)
480
481
482
483
484
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
    qa_config.ReleaseNode(node)


485
486
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
487
488

  """
489
490
491
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
492
  RunEnvTests()
493
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
494
495

  # Load RAPI certificate
496
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
497

Michael Hanselmann's avatar
Michael Hanselmann committed
498
499
  RunClusterTests()
  RunOsTests()
500

Iustin Pop's avatar
Iustin Pop committed
501
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
502

503
  RunCommonNodeTests()
504
  RunGroupListTests()
505
  RunGroupRwTests()
506

507
508
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
509
510
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
511
512
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
513
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
514
515
  finally:
    qa_config.ReleaseNode(pnode)
516

517
518
519
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
520
521
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
522
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
523

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
524
525
526
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

527
      if qa_config.TestEnabled("instance-add-plain-disk"):
528
529
530
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
531
532
          if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
            RunCommonInstanceTests(rapi_instance)
533
534
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          del rapi_instance
535

536
537
538
  finally:
    qa_config.ReleaseNode(pnode)

539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)

        RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
        RunDaemonTests(instance)
        for node in inodes:
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
        if len(inodes) > 1:
          RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                    constants.INITIAL_NODE_GROUP_NAME,
                    inodes[0]["primary"], inodes[1]["primary"])
        if qa_config.TestEnabled("instance-convert-disk"):
          RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceConvertDiskToPlain, instance, inodes)
          RunTest(qa_instance.TestInstanceStartup, instance)
        RunCommonInstanceTests(instance)
        RunGroupListTests()
        RunExportImportTests(instance, inodes)
        RunHardwareFailureTests(instance, inodes)
        RunRepairDiskSizes()
        RunTest(qa_instance.TestInstanceRemove, instance)
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
574
      qa_cluster.AssertClusterVerify()
575

576
577
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
578
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
579
      for shutdown in [False, True]:
580
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
581
582
583
584
585
586
587
588
589
590
591
        expnode = qa_config.AcquireNode(exclude=pnode)
        try:
          if shutdown:
            # Stop instance before exporting and removing it
            RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
          RunTest(qa_instance.TestBackupList, expnode)
        finally:
          qa_config.ReleaseNode(expnode)
        del expnode
        del instance
592
      qa_cluster.AssertClusterVerify()
593

Iustin Pop's avatar
Iustin Pop committed
594
  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
595
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
596

597
598
  RunExclusiveStorageTests()

599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
      try:
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
        qa_node.MakeNodeOffline(snode, "yes")
        try:
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
          qa_node.MakeNodeOffline(snode, "no")
      finally:
        qa_config.ReleaseNode(pnode)
    finally:
      qa_config.ReleaseNode(snode)
    # FIXME: This test leaves a DRBD device and two LVs behind
    # Cluster-verify would fail

Iustin Pop's avatar
Iustin Pop committed
619
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
620

Iustin Pop's avatar
Iustin Pop committed
621
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
622

623

624
@UsesRapiClient
625
626
627
628
629
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
630
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
631
632
                    action="store_true",
                    help="Really execute the tests")
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
  (qa_config.options, args) = parser.parse_args()

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

648
649
650
651
652
653
  primary = qa_config.GetMasterNode()["primary"]
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
654
655
656
657
658
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
659
if __name__ == "__main__":
660
  main()