ganeti-qa.py 20.7 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
40
import qa_instance
import qa_node
41
import qa_os
42
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
43
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
44
import qa_tags
45
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
46

47
from ganeti import utils
48
from ganeti import rapi # pylint: disable=W0611
49
from ganeti import constants
50

51
import ganeti.rapi.client # pylint: disable=W0611
52
from ganeti.rapi.client import UsesRapiClient
53

Iustin Pop's avatar
Iustin Pop committed
54

Iustin Pop's avatar
Iustin Pop committed
55
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
56
57
58
59
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
60
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
61
62
63
64
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
65
66
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
67
68

  """
69
70
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
71
  else:
Iustin Pop's avatar
Iustin Pop committed
72
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
73

Iustin Pop's avatar
Iustin Pop committed
74
75
  return desc.rstrip(".")

76

77
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
78
79
80
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
81

Iustin Pop's avatar
Iustin Pop committed
82
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
83

Iustin Pop's avatar
Iustin Pop committed
84
85
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
86
87
88
89
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
90
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
91
92
93
94
95
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
96
97


98
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
99
100
101
102
103
104
105
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
106
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
107
108
109
110
111
112
113
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
114
115
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
116
117

  """
Iustin Pop's avatar
Iustin Pop committed
118
119
120
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
121

122

123
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
124
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
125

126
127
128
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
129
  """
Iustin Pop's avatar
Iustin Pop committed
130
131
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
132
133
134
135

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
136
  RunTestIf("job-list", qa_job.TestJobList)
137

Iustin Pop's avatar
Iustin Pop committed
138
139
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
140
141
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
142

Iustin Pop's avatar
Iustin Pop committed
143
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
144

145
146
147
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

148
149
  RunTestIf("node-list", qa_node.TestNodeList)

150
151
152
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
153
  RunTestIf("job-list", qa_job.TestJobListFields)
154
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
155

Iustin Pop's avatar
Iustin Pop committed
156
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
157
158
159
160


def RunClusterTests():
  """Runs tests related to gnt-cluster.
161

Michael Hanselmann's avatar
Michael Hanselmann committed
162
  """
Iustin Pop's avatar
Iustin Pop committed
163
  for test, fn in [
164
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
165
166
167
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
168
    # TODO: add more cluster modify tests
169
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
170
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
171
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
172
173
174
175
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
176
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
Iustin Pop's avatar
Iustin Pop committed
177
178
179
180
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
181
182
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
Iustin Pop's avatar
Iustin Pop committed
183
    ("cluster-oob", qa_cluster.TestClusterOob),
Iustin Pop's avatar
Iustin Pop committed
184
185
    ("rapi", qa_rapi.TestVersion),
    ("rapi", qa_rapi.TestEmptyCluster),
186
    ("rapi", qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
187
188
    ]:
    RunTestIf(test, fn)
189

190

191
192
193
194
195
196
197
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
198
199
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
200

Michael Hanselmann's avatar
Michael Hanselmann committed
201
  """
202
203
204
205
206
  if qa_config.TestEnabled("rapi"):
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
207
208
209
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
210
211
212
213
    ]:
    RunTestIf("os", fn)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
214
215
216
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
217
218
219
220
    ]:
    RunTestIf("os", fn, rapi_getos)

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
221
222
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
223
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
224
225
    ]:
    RunTestIf("os", fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
226
227
228
229
230
231


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
232
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
233
234
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
235
236
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
237
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
238

239
240
241
242
243
244
  # Test shutdown/start via RAPI
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceShutdown, instance)
  RunTestIf(["instance-shutdown", "rapi"],
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
245
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
246

Iustin Pop's avatar
Iustin Pop committed
247
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
248

Iustin Pop's avatar
Iustin Pop committed
249
250
251
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
  RunTestIf(["instance-modify", "rapi"],
            qa_rapi.TestRapiInstanceModify, instance)
252

Iustin Pop's avatar
Iustin Pop committed
253
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
254
255
  RunTestIf(["instance-console", "rapi"],
            qa_rapi.TestRapiInstanceConsole, instance)
256

257
258
259
260
261
262
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
263
264
265
266
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
267
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
268
269
  RunTestIf(["instance-reinstall", "rapi"],
            qa_rapi.TestRapiInstanceReinstall, instance)
270

Iustin Pop's avatar
Iustin Pop committed
271
  if qa_config.TestEnabled("instance-rename"):
272
273
274
275
276
    tgt_instance = qa_config.AcquireInstance()
    try:
      rename_source = instance["name"]
      rename_target = tgt_instance["name"]
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
277
      RunTest(qa_instance.TestInstanceRenameAndBack,
278
              rename_source, rename_source)
Iustin Pop's avatar
Iustin Pop committed
279
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
280
281
282
283
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
284
                rename_source, rename_target)
285
286
287
288
        RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_target)
    finally:
      qa_config.ReleaseInstance(tgt_instance)
Iustin Pop's avatar
Iustin Pop committed
289

Iustin Pop's avatar
Iustin Pop committed
290
291
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
292
293
294
295
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
296

Iustin Pop's avatar
Iustin Pop committed
297
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
298

Michael Hanselmann's avatar
Michael Hanselmann committed
299
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
300

Iustin Pop's avatar
Iustin Pop committed
301
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
302

303
304
305
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

306
307
308
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

309
310
311
312
313

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
314
315
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
316
  RunTestIf("node-oob", qa_node.TestOutOfBand)
317

318

319
320
321
322
def RunGroupListTests():
  """Run tests for listing node groups.

  """
323
324
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
325
326


327
328
329
330
331
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
332
333
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
334
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
335
336
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
337

338

339
def RunExportImportTests(instance, pnode, snode):
Michael Hanselmann's avatar
Michael Hanselmann committed
340
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
341

342
343
344
345
  @param pnode: current primary node of the instance
  @param snode: current secondary node of the instance, if any,
      otherwise None

Michael Hanselmann's avatar
Michael Hanselmann committed
346
  """
Iustin Pop's avatar
Iustin Pop committed
347
  if qa_config.TestEnabled("instance-export"):
348
349
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
350
351
352
353
354
355
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
356
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
357
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
358
        try:
359
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
360
                  expnode, name)
361
362
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
363
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
364
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
365
366
367
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
368

Iustin Pop's avatar
Iustin Pop committed
369
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
370
371
    newinst = qa_config.AcquireInstance()
    try:
372
373
374
375
376
      if snode is None:
        excl = [pnode]
      else:
        excl = [pnode, snode]
      tnode = qa_config.AcquireNode(exclude=excl)
377
378
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
379
                pnode, snode, tnode)
380
      finally:
381
        qa_config.ReleaseNode(tnode)
382
383
384
    finally:
      qa_config.ReleaseInstance(newinst)

Michael Hanselmann's avatar
Michael Hanselmann committed
385

386
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
387
  """Test the ganeti-watcher script.
388

Michael Hanselmann's avatar
Michael Hanselmann committed
389
  """
390
  RunTest(qa_daemon.TestPauseWatcher)
391

Iustin Pop's avatar
Iustin Pop committed
392
  RunTestIf("instance-automatic-restart",
393
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
394
  RunTestIf("instance-consecutive-failures",
395
            qa_daemon.TestInstanceConsecutiveFailures, instance)
396

397
398
  RunTest(qa_daemon.TestResumeWatcher)

399

400
401
402
403
404
405
406
407
408
409
410
411
412
def RunSingleHomedHardwareFailureTests(instance, pnode):
  """Test hardware failure recovery for single-homed instances.

  """
  if qa_config.TestEnabled("instance-recreate-disks"):
    othernode = qa_config.AcquireNode(exclude=[pnode])
    try:
      RunTest(qa_instance.TestRecreateDisks,
              instance, pnode, None, [othernode])
    finally:
      qa_config.ReleaseNode(othernode)


Michael Hanselmann's avatar
Michael Hanselmann committed
413
414
def RunHardwareFailureTests(instance, pnode, snode):
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
415

Michael Hanselmann's avatar
Michael Hanselmann committed
416
  """
Iustin Pop's avatar
Iustin Pop committed
417
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
418
419
  RunTestIf(["instance-failover", "rapi"],
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
420

Iustin Pop's avatar
Iustin Pop committed
421
422
423
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
  RunTestIf(["instance-migrate", "rapi"],
            qa_rapi.TestRapiInstanceMigrate, instance)
424

Iustin Pop's avatar
Iustin Pop committed
425
  if qa_config.TestEnabled("instance-replace-disks"):
426
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
427
    try:
428
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
429
430
431
432
433
      RunTest(qa_instance.TestReplaceDisks,
              instance, pnode, snode, othernode)
    finally:
      qa_config.ReleaseNode(othernode)

434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
  if qa_config.TestEnabled("instance-recreate-disks"):
    othernode1 = qa_config.AcquireNode(exclude=[pnode, snode])
    try:
      othernode2 = qa_config.AcquireNode(exclude=[pnode, snode, othernode1])
    except qa_error.OutOfNodesError:
      # Let's reuse one of the nodes if the cluster is not big enough
      othernode2 = pnode
    try:
      RunTest(qa_instance.TestRecreateDisks,
              instance, pnode, snode, [othernode1, othernode2])
    finally:
      qa_config.ReleaseNode(othernode1)
      if othernode2 != pnode:
        qa_config.ReleaseNode(othernode2)

Iustin Pop's avatar
Iustin Pop committed
449
  RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode)
Michael Hanselmann's avatar
Michael Hanselmann committed
450

Iustin Pop's avatar
Iustin Pop committed
451
  RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode)
Michael Hanselmann's avatar
Michael Hanselmann committed
452

Iustin Pop's avatar
Iustin Pop committed
453
  RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure,
Michael Hanselmann's avatar
Michael Hanselmann committed
454
            instance, pnode, snode)
Iustin Pop's avatar
Iustin Pop committed
455
456
457
  RunTestIf("instance-disk-failure",
            qa_instance.TestInstanceSecondaryDiskFailure, instance,
            pnode, snode)
Michael Hanselmann's avatar
Michael Hanselmann committed
458
459


460
461
462
463
464
465
466
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
467
468
469
470
    old_es = qa_cluster.TestSetExclStorCluster(False)
    qa_cluster.TestExclStorSingleNode(node)

    qa_cluster.TestSetExclStorCluster(True)
471
472
473
474
475
476
477
478
479
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
      instance1 = qa_instance.TestInstanceAddWithPlainDisk(node)
      instance2 = qa_instance.TestInstanceAddWithPlainDisk(node)
      # cluster-verify checks that disks are allocated correctly
      qa_cluster.AssertClusterVerify()
      qa_instance.TestInstanceRemove(instance1)
      qa_instance.TestInstanceRemove(instance2)
480
481
482
483
484
485
486
487
488
489
490
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
        instance = qa_instance.TestInstanceAddWithDrbdDisk(node, snode)
        qa_cluster.TestSetExclStorCluster(True)
        exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
        qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
        qa_instance.TestInstanceRemove(instance)
      finally:
        qa_config.ReleaseNode(snode)
491
492
493
494
495
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
    qa_config.ReleaseNode(node)


496
497
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
498
499

  """
500
501
502
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
503
  RunEnvTests()
504
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
505
506

  # Load RAPI certificate
507
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
508

Michael Hanselmann's avatar
Michael Hanselmann committed
509
510
  RunClusterTests()
  RunOsTests()
511

Iustin Pop's avatar
Iustin Pop committed
512
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
513

514
  RunCommonNodeTests()
515
  RunGroupListTests()
516
  RunGroupRwTests()
517

518
519
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
520
521
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
522
523
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
524
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
525
526
  finally:
    qa_config.ReleaseNode(pnode)
527

Michael Hanselmann's avatar
Michael Hanselmann committed
528
529
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
530
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
531

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
532
533
534
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

535
      if qa_config.TestEnabled("instance-add-plain-disk"):
536
537
538
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
539
540
          if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
            RunCommonInstanceTests(rapi_instance)
541
542
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          del rapi_instance
543

544
    if qa_config.TestEnabled("instance-add-plain-disk"):
Michael Hanselmann's avatar
Michael Hanselmann committed
545
546
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
      RunCommonInstanceTests(instance)
547
      RunGroupListTests()
548
      RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
549
      RunExportImportTests(instance, pnode, None)
550
      RunDaemonTests(instance)
551
      RunRepairDiskSizes()
552
      RunSingleHomedHardwareFailureTests(instance, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
553
554
      RunTest(qa_instance.TestInstanceRemove, instance)
      del instance
555

556
    multinode_tests = [
Iustin Pop's avatar
Iustin Pop committed
557
      ("instance-add-drbd-disk",
558
559
560
561
562
563
564
565
       qa_instance.TestInstanceAddWithDrbdDisk),
    ]

    for name, func in multinode_tests:
      if qa_config.TestEnabled(name):
        snode = qa_config.AcquireNode(exclude=pnode)
        try:
          instance = RunTest(func, pnode, snode)
566
567
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, pnode)
          RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, snode)
568
          RunCommonInstanceTests(instance)
569
          RunGroupListTests()
570
571
572
          RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                    constants.INITIAL_NODE_GROUP_NAME,
                    pnode["primary"], snode["primary"])
Iustin Pop's avatar
Iustin Pop committed
573
          if qa_config.TestEnabled("instance-convert-disk"):
574
            RunTest(qa_instance.TestInstanceShutdown, instance)
575
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
576
            RunTest(qa_instance.TestInstanceStartup, instance)
577
          RunExportImportTests(instance, pnode, snode)
578
          RunHardwareFailureTests(instance, pnode, snode)
579
          RunRepairDiskSizes()
580
581
582
583
          RunTest(qa_instance.TestInstanceRemove, instance)
          del instance
        finally:
          qa_config.ReleaseNode(snode)
Iustin Pop's avatar
Iustin Pop committed
584

585
586
587
588
589
590
591
592
593
  finally:
    qa_config.ReleaseNode(pnode)

  # Test removing instance with offline drbd secondary
  if qa_config.TestEnabled("instance-remove-drbd-offline"):
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
594
      try:
595
        instance = qa_instance.TestInstanceAddWithDrbdDisk(pnode, snode)
596
        qa_node.MakeNodeOffline(snode, "yes")
597
598
599
600
        try:
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
          qa_node.MakeNodeOffline(snode, "no")
601
      finally:
602
603
604
        qa_config.ReleaseNode(pnode)
    finally:
      qa_config.ReleaseNode(snode)
605

606
607
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
608
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
609
610
611
612
613
614
615
616
617
618
619
620
621
      for shutdown in [False, True]:
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
        expnode = qa_config.AcquireNode(exclude=pnode)
        try:
          if shutdown:
            # Stop instance before exporting and removing it
            RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
          RunTest(qa_instance.TestBackupList, expnode)
        finally:
          qa_config.ReleaseNode(expnode)
        del expnode
        del instance
622

Iustin Pop's avatar
Iustin Pop committed
623
  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
624
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
625

626
627
  RunExclusiveStorageTests()

Iustin Pop's avatar
Iustin Pop committed
628
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
629

Iustin Pop's avatar
Iustin Pop committed
630
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
631

632

633
@UsesRapiClient
634
635
636
637
638
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
639
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
640
641
                    action="store_true",
                    help="Really execute the tests")
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
  (qa_config.options, args) = parser.parse_args()

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

657
658
659
660
661
662
  primary = qa_config.GetMasterNode()["primary"]
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
663
664
665
666
667
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
668
if __name__ == "__main__":
669
  main()