ganeti-qa.py 20.7 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
40
import qa_instance
import qa_node
41
import qa_os
42
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
43
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
44
import qa_tags
45
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
46

47
from ganeti import utils
48
from ganeti import rapi # pylint: disable=W0611
49
from ganeti import constants
50

51
import ganeti.rapi.client # pylint: disable=W0611
52
from ganeti.rapi.client import UsesRapiClient
53

Iustin Pop's avatar
Iustin Pop committed
54

Iustin Pop's avatar
Iustin Pop committed
55
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
56
57
58
59
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
60
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
61
62
63
64
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
65
66
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
67
68

  """
69
70
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
71
  else:
Iustin Pop's avatar
Iustin Pop committed
72
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
73

Iustin Pop's avatar
Iustin Pop committed
74
75
  return desc.rstrip(".")

76

77
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
78
79
80
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
81

Iustin Pop's avatar
Iustin Pop committed
82
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
83

Iustin Pop's avatar
Iustin Pop committed
84
85
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
86
87
88
89
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
90
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
91
92
93
94
95
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
96
97


98
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
99
100
101
102
103
104
105
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
106
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
107
108
109
110
111
112
113
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
114
115
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
116
117

  """
Iustin Pop's avatar
Iustin Pop committed
118
119
120
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
121

122

123
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
124
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
125

126
127
128
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
129
  """
Iustin Pop's avatar
Iustin Pop committed
130
131
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
132
133
134
135

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
136
  RunTestIf("job-list", qa_job.TestJobList)
137

Iustin Pop's avatar
Iustin Pop committed
138
139
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
140
141
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
142

Iustin Pop's avatar
Iustin Pop committed
143
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
144

145
146
147
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

148
149
  RunTestIf("node-list", qa_node.TestNodeList)

150
151
152
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
153
  RunTestIf("job-list", qa_job.TestJobListFields)
154
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
155

Iustin Pop's avatar
Iustin Pop committed
156
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
157
158
159
160


def RunClusterTests():
  """Runs tests related to gnt-cluster.
161

Michael Hanselmann's avatar
Michael Hanselmann committed
162
  """
Iustin Pop's avatar
Iustin Pop committed
163
  for test, fn in [
164
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
165
166
167
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
168
    # TODO: add more cluster modify tests
169
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
170
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
171
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
172
173
174
175
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
176
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
177
178
179
180
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
181
182
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
183
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
184
185
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
186
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
187
188
    ]:
    RunTestIf(test, fn)
189

190

191
192
193
194
195
196
197
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
198
199
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
200

Michael Hanselmann's avatar
Michael Hanselmann committed
201
  """
202
203
204
205
206
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
207
208
209
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
210
211
212
213
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
214
215
216
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
217
218
219
220
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
221
222
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
223
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
224
225
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
226
227
228
229
230
231


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
232
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
233
234
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
235
236
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
237
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
238

239
240
241
242
243
244
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
245
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
246

Iustin Pop's avatar
Iustin Pop committed
247
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
248

Iustin Pop's avatar
Iustin Pop committed
249
250
251
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
252

Iustin Pop's avatar
Iustin Pop committed
253
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
254
255
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
256

257
258
259
260
261
262
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
263
264
265
266
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
267
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
268
269
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
270

Iustin Pop's avatar
Iustin Pop committed
271
  if qa_config.TestEnabled("instance-rename"):
272
273
274
275
276
    tgt_instance = qa_config.AcquireInstance()
    try:
      rename_source = instance["name"]
      rename_target = tgt_instance["name"]
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
277
      RunTest(qa_instance.TestInstanceRenameAndBack,
278
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
279
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
280
281
282
283
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
284
                rename_source, rename_target)
285
286
287
288
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
      qa_config.ReleaseInstance(tgt_instance)
Iustin Pop's avatar
Iustin Pop committed
289

Iustin Pop's avatar
Iustin Pop committed
290
291
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
292
293
294
295
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
296

Iustin Pop's avatar
Iustin Pop committed
297
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
298

Michael Hanselmann's avatar
Michael Hanselmann committed
299
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
300

Iustin Pop's avatar
Iustin Pop committed
301
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
302

303
304
305
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

306
307
308
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

309
310
311
312
313

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
314
315
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
316
  RunTestIf("node-oob", qa_node.TestOutOfBand)
317

318

319
320
321
322
def RunGroupListTests():
  """Run tests for listing node groups.

  """
323
324
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
325
326


327
328
329
330
331
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
332
333
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
334
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
335
336
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
337

338

339
def RunExportImportTests(instance, pnode, snode):
Michael Hanselmann's avatar
Michael Hanselmann committed
340
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
341

342
343
344
345
  @param pnode: current primary node of the instance
  @param snode: current secondary node of the instance, if any,
      otherwise None

Michael Hanselmann's avatar
Michael Hanselmann committed
346
  """
Iustin Pop's avatar
Iustin Pop committed
347
  if qa_config.TestEnabled("instance-export"):
348
349
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
350
351
352
353
354
355
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
356
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
357
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
358
        try:
359
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
360
                  expnode, name)
361
362
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
363
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
364
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
365
366
367
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
368

Iustin Pop's avatar
Iustin Pop committed
369
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
370
371
    newinst = qa_config.AcquireInstance()
    try:
372
373
374
375
376
      if snode is None:
        excl = [pnode]
      else:
        excl = [pnode, snode]
      tnode = qa_config.AcquireNode(exclude=excl)
377
378
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
379
                pnode, snode, tnode)
380
      finally:
381
        qa_config.ReleaseNode(tnode)
382
383
384
    finally:
      qa_config.ReleaseInstance(newinst)

Michael Hanselmann's avatar
Michael Hanselmann committed
385

386
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
387
  """Test the ganeti-watcher script.
388

Michael Hanselmann's avatar
Michael Hanselmann committed
389
  """
390
  RunTest(qa_daemon.TestPauseWatcher)
391

Iustin Pop's avatar
Iustin Pop committed
392
  RunTestIf("instance-automatic-restart",
393
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
394
  RunTestIf("instance-consecutive-failures",
395
            qa_daemon.TestInstanceConsecutiveFailures, instance)
396

397
398
  RunTest(qa_daemon.TestResumeWatcher)

399

400
401
402
403
404
405
406
407
408
409
410
411
412
def RunSingleHomedHardwareFailureTests(instance, pnode):
  """Test hardware failure recovery for single-homed instances.

  """
  if qa_config.TestEnabled("instance-recreate-disks"):
    othernode = qa_config.AcquireNode(exclude=[pnode])
    try:
      RunTest(qa_instance.TestRecreateDisks,
              instance, pnode, None, [othernode])
    finally:
      qa_config.ReleaseNode(othernode)


Michael Hanselmann's avatar
Michael Hanselmann committed
413
414
def RunHardwareFailureTests(instance, pnode, snode):
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
415

Michael Hanselmann's avatar
Michael Hanselmann committed
416
  """
Iustin Pop's avatar
Iustin Pop committed
417
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
418
419
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
420

Iustin Pop's avatar
Iustin Pop committed
421
422
423
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
424

Iustin Pop's avatar
Iustin Pop committed
425
  if qa_config.TestEnabled("instance-replace-disks"):
426
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
427
    try:
428
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
429
430
431
432
433
      RunTest(qa_instance.TestReplaceDisks,
              instance, pnode, snode, othernode)
    finally:
      qa_config.ReleaseNode(othernode)

434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
  if qa_config.TestEnabled("instance-recreate-disks"):
    othernode1 = qa_config.AcquireNode(exclude=[pnode, snode])
    try:
      othernode2 = qa_config.AcquireNode(exclude=[pnode, snode, othernode1])
    except qa_error.OutOfNodesError:
      # Let's reuse one of the nodes if the cluster is not big enough
      othernode2 = pnode
    try:
      RunTest(qa_instance.TestRecreateDisks,
              instance, pnode, snode, [othernode1, othernode2])
    finally:
      qa_config.ReleaseNode(othernode1)
      if othernode2 != pnode:
        qa_config.ReleaseNode(othernode2)

Iustin Pop's avatar
Iustin Pop committed
449
  RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode)
Michael Hanselmann's avatar
Michael Hanselmann committed
450

Iustin Pop's avatar
Iustin Pop committed
451
  RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode)
Michael Hanselmann's avatar
Michael Hanselmann committed
452

Iustin Pop's avatar
Iustin Pop committed
453
  RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure,
Michael Hanselmann's avatar
Michael Hanselmann committed
454
            instance, pnode, snode)
Iustin Pop's avatar
Iustin Pop committed
455
456
457
  RunTestIf("instance-disk-failure",
            qa_instance.TestInstanceSecondaryDiskFailure, instance,
            pnode, snode)
Michael Hanselmann's avatar
Michael Hanselmann committed
458
459


460
461
462
463
464
465
466
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
467
468
469
470
    old_es = qa_cluster.TestSetExclStorCluster(False)
    qa_cluster.TestExclStorSingleNode(node)

    qa_cluster.TestSetExclStorCluster(True)
471
472
    qa_cluster.TestExclStorSharedPv(node)

473
474
475
476
477
478
479
480
481
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
      instance1 = qa_instance.TestInstanceAddWithPlainDisk(node)
      instance2 = qa_instance.TestInstanceAddWithPlainDisk(node)
      # cluster-verify checks that disks are allocated correctly
      qa_cluster.AssertClusterVerify()
      qa_instance.TestInstanceRemove(instance1)
      qa_instance.TestInstanceRemove(instance2)
482
483
484
485
486
487
488
489
490
491
492
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
        instance = qa_instance.TestInstanceAddWithDrbdDisk(node, snode)
        qa_cluster.TestSetExclStorCluster(True)
        exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
        qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
        qa_instance.TestInstanceRemove(instance)
      finally:
        qa_config.ReleaseNode(snode)
493
494
495
496
497
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
    qa_config.ReleaseNode(node)


498
499
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
500
501

  """
502
503
504
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
505
  RunEnvTests()
506
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
507
508

  # Load RAPI certificate
509
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
510

Michael Hanselmann's avatar
Michael Hanselmann committed
511
512
  RunClusterTests()
  RunOsTests()
513

Iustin Pop's avatar
Iustin Pop committed
514
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
515

516
  RunCommonNodeTests()
517
  RunGroupListTests()
518
  RunGroupRwTests()
519

520
521
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
522
523
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
524
525
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
526
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
527
528
  finally:
    qa_config.ReleaseNode(pnode)
529

Michael Hanselmann's avatar
Michael Hanselmann committed
530
531
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
532
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
533

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
534
535
536
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

537
      if qa_config.TestEnabled("instance-add-plain-disk"):
538
539
540
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
541
542
          if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
            RunCommonInstanceTests(rapi_instance)
543
544
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          del rapi_instance
545

546
    if qa_config.TestEnabled("instance-add-plain-disk"):
Michael Hanselmann's avatar
Michael Hanselmann committed
547
548
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
      RunCommonInstanceTests(instance)
549
      RunGroupListTests()
550
      RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
551
      RunExportImportTests(instance, pnode, None)
552
      RunDaemonTests(instance)
553
      RunRepairDiskSizes()
554
      RunSingleHomedHardwareFailureTests(instance, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
555
556
      RunTest(qa_instance.TestInstanceRemove, instance)
      del instance
557

558
    multinode_tests = [
Iustin Pop's avatar
Iustin Pop committed
559
      ("instance-add-drbd-disk",
560
561
562
563
564
565
566
567
       qa_instance.TestInstanceAddWithDrbdDisk),
    ]

    for name, func in multinode_tests:
      if qa_config.TestEnabled(name):
        snode = qa_config.AcquireNode(exclude=pnode)
        try:
          instance = RunTest(func, pnode, snode)
568
569
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, pnode)
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, snode)
570
          RunCommonInstanceTests(instance)
571
          RunGroupListTests()
572
573
574
          RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                    constants.INITIAL_NODE_GROUP_NAME,
                    pnode["primary"], snode["primary"])
Iustin Pop's avatar
Iustin Pop committed
575
          if qa_config.TestEnabled("instance-convert-disk"):
576
            RunTest(qa_instance.TestInstanceShutdown, instance)
577
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
578
            RunTest(qa_instance.TestInstanceStartup, instance)
579
          RunExportImportTests(instance, pnode, snode)
580
          RunHardwareFailureTests(instance, pnode, snode)
581
          RunRepairDiskSizes()
582
583
584
585
          RunTest(qa_instance.TestInstanceRemove, instance)
          del instance
        finally:
          qa_config.ReleaseNode(snode)
Iustin Pop's avatar
Iustin Pop committed
586

587
588
589
590
591
592
593
594
595
  finally:
    qa_config.ReleaseNode(pnode)

  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
596
      try:
597
        instance = qa_instance.TestInstanceAddWithDrbdDisk(pnode, snode)
598
        qa_node.MakeNodeOffline(snode, "yes")
599
600
601
602
        try:
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
          qa_node.MakeNodeOffline(snode, "no")
603
      finally:
604
605
606
        qa_config.ReleaseNode(pnode)
    finally:
      qa_config.ReleaseNode(snode)
607

608
609
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
610
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
611
612
613
614
615
616
617
618
619
620
621
622
623
      for shutdown in [False, True]:
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
        expnode = qa_config.AcquireNode(exclude=pnode)
        try:
          if shutdown:
            # Stop instance before exporting and removing it
            RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
          RunTest(qa_instance.TestBackupList, expnode)
        finally:
          qa_config.ReleaseNode(expnode)
        del expnode
        del instance
624

Iustin Pop's avatar
Iustin Pop committed
625
  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
626
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
627

628
629
  RunExclusiveStorageTests()

Iustin Pop's avatar
Iustin Pop committed
630
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
631

Iustin Pop's avatar
Iustin Pop committed
632
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
633

634

635
@UsesRapiClient
636
637
638
639
640
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
641
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
642
643
                    action="store_true",
                    help="Really execute the tests")
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
  (qa_config.options, args) = parser.parse_args()

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

659
660
661
662
663
664
  primary = qa_config.GetMasterNode()["primary"]
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
665
666
667
668
669
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
670
if __name__ == "__main__":
671
  main()