ganeti-qa.py 22 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
Helga Velroyen's avatar
Helga Velroyen committed
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
44
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import qa_tags
46
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
47

48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

Iustin Pop's avatar
Iustin Pop committed
55

Iustin Pop's avatar
Iustin Pop committed
56
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
61
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
62
63
64
65
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
66
67
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
70
71
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
72
  else:
Iustin Pop's avatar
Iustin Pop committed
73
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
74

Iustin Pop's avatar
Iustin Pop committed
75
76
  return desc.rstrip(".")

77

78
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
79
80
81
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
82

Iustin Pop's avatar
Iustin Pop committed
83
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
84

Iustin Pop's avatar
Iustin Pop committed
85
86
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
87
88
89
90
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
91
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
92
93
94
95
96
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
97
98


99
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
104
105
106
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
108
109
110
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
111
    # TODO: Formatting test names when non-string names are involved
Iustin Pop's avatar
Iustin Pop committed
112
113
114
115
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
116
117
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
118
119

  """
Iustin Pop's avatar
Iustin Pop committed
120
121
122
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
123

124

125
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
126
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
127

128
129
130
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
131
  """
Iustin Pop's avatar
Iustin Pop committed
132
133
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
134
135
136
137
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
138
139
140
141

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
142
  RunTestIf("job-list", qa_job.TestJobList)
143

Iustin Pop's avatar
Iustin Pop committed
144
145
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
146
147
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
148

Iustin Pop's avatar
Iustin Pop committed
149
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
150

151
152
153
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

154
155
  RunTestIf("node-list", qa_node.TestNodeList)

156
157
158
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
159
  RunTestIf("job-list", qa_job.TestJobListFields)
160
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
161

Iustin Pop's avatar
Iustin Pop committed
162
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
163
164
165
166


def RunClusterTests():
  """Runs tests related to gnt-cluster.
167

Michael Hanselmann's avatar
Michael Hanselmann committed
168
  """
Iustin Pop's avatar
Iustin Pop committed
169
  for test, fn in [
170
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
171
172
173
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
174
    # TODO: add more cluster modify tests
175
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
Iustin Pop's avatar
Iustin Pop committed
176
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
177
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
Iustin Pop's avatar
Iustin Pop committed
178
179
180
181
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
182
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
183
184
    (["cluster-copyfile", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterCopyfile),
Iustin Pop's avatar
Iustin Pop committed
185
186
187
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
188
189
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
190
191
    (["cluster-oob", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterOob),
192
193
194
    (qa_rapi.Enabled, qa_rapi.TestVersion),
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
195
196
    ]:
    RunTestIf(test, fn)
197

198

199
200
201
202
203
204
205
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
206
207
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
208

Michael Hanselmann's avatar
Michael Hanselmann committed
209
  """
210
211
  os_enabled = ["os", qa_config.NoVirtualCluster]

212
  if qa_config.TestEnabled(qa_rapi.Enabled):
213
214
215
216
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
217
218
219
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
220
    ]:
221
    RunTestIf(os_enabled, fn)
222
223

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
224
225
226
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
227
    ]:
228
    RunTestIf(os_enabled, fn, rapi_getos)
229
230

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
231
232
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
233
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
234
    ]:
235
    RunTestIf(os_enabled, fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
236
237
238
239
240
241


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
242
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
243
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
244
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
245
246
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
247
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
248

249
  # Test shutdown/start via RAPI
250
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
251
            qa_rapi.TestRapiInstanceShutdown, instance)
252
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
253
254
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
255
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
256

Iustin Pop's avatar
Iustin Pop committed
257
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
258

Iustin Pop's avatar
Iustin Pop committed
259
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
260
  RunTestIf(["instance-modify", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
261
            qa_rapi.TestRapiInstanceModify, instance)
262

Iustin Pop's avatar
Iustin Pop committed
263
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
264
  RunTestIf(["instance-console", qa_rapi.Enabled],
265
            qa_rapi.TestRapiInstanceConsole, instance)
266

267
268
269
270
271
272
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
273
274
275
276
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
277
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
278
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
279
            qa_rapi.TestRapiInstanceReinstall, instance)
280

Iustin Pop's avatar
Iustin Pop committed
281
  if qa_config.TestEnabled("instance-rename"):
282
283
    tgt_instance = qa_config.AcquireInstance()
    try:
284
285
      rename_source = instance.name
      rename_target = tgt_instance.name
286
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
287
      RunTest(qa_instance.TestInstanceRenameAndBack,
288
              rename_source, rename_source)
289
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
290
291
292
293
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
294
                rename_source, rename_target)
295
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
296
297
                  rename_source, rename_target)
    finally:
298
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
299

Iustin Pop's avatar
Iustin Pop committed
300
301
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
302
303
304
305
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
306

Iustin Pop's avatar
Iustin Pop committed
307
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
308

Michael Hanselmann's avatar
Michael Hanselmann committed
309
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
310

311
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
312

313
314
315
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

316
317
318
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

319
320
321
322
323

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
324
325
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
326
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
327

328

329
330
331
332
def RunGroupListTests():
  """Run tests for listing node groups.

  """
333
334
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
335
336


Helga Velroyen's avatar
Helga Velroyen committed
337
338
339
340
341
342
343
344
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


345
346
347
348
349
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
350
351
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
352
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
353
354
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
355

356

357
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
358
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
359

360
361
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
362

Michael Hanselmann's avatar
Michael Hanselmann committed
363
  """
Iustin Pop's avatar
Iustin Pop committed
364
  if qa_config.TestEnabled("instance-export"):
365
366
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

367
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
368
369
370
371
372
373
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
374
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
375
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
376
        try:
377
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
378
                  expnode, name)
379
380
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
381
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
382
        finally:
383
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
384
    finally:
385
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
386

387
  if qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"]):
388
389
    newinst = qa_config.AcquireInstance()
    try:
390
      tnode = qa_config.AcquireNode(exclude=inodes)
391
392
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
393
                inodes, tnode)
394
      finally:
395
        tnode.Release()
396
    finally:
397
      newinst.Release()
398

Michael Hanselmann's avatar
Michael Hanselmann committed
399

400
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
401
  """Test the ganeti-watcher script.
402

Michael Hanselmann's avatar
Michael Hanselmann committed
403
  """
404
  RunTest(qa_daemon.TestPauseWatcher)
405

Iustin Pop's avatar
Iustin Pop committed
406
  RunTestIf("instance-automatic-restart",
407
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
408
  RunTestIf("instance-consecutive-failures",
409
            qa_daemon.TestInstanceConsecutiveFailures, instance)
410

411
412
  RunTest(qa_daemon.TestResumeWatcher)

413

414
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
415
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
416

Michael Hanselmann's avatar
Michael Hanselmann committed
417
  """
Iustin Pop's avatar
Iustin Pop committed
418
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
419
  RunTestIf(["instance-failover", qa_rapi.Enabled],
420
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
421

Iustin Pop's avatar
Iustin Pop committed
422
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
423
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
424
            qa_rapi.TestRapiInstanceMigrate, instance)
425

Iustin Pop's avatar
Iustin Pop committed
426
  if qa_config.TestEnabled("instance-replace-disks"):
427
428
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
429
    try:
430
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
431
      RunTest(qa_instance.TestReplaceDisks,
432
              instance, inodes, othernodes)
433
    finally:
434
435
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
436

437
438
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
439
440
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
441
    except qa_error.OutOfNodesError:
442
443
444
445
446
447
448
449
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
450
451
    try:
      RunTest(qa_instance.TestRecreateDisks,
452
              instance, inodes, othernodes)
453
    finally:
454
      qa_config.ReleaseManyNodes(acquirednodes)
455

456
457
458
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
459
460


461
462
463
464
465
466
467
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
468
    old_es = qa_cluster.TestSetExclStorCluster(False)
469
    qa_node.TestExclStorSingleNode(node)
470
471

    qa_cluster.TestSetExclStorCluster(True)
472
473
    qa_cluster.TestExclStorSharedPv(node)

474
475
476
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
477
478

      # Create and allocate instances
479
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
480
481
482
483
484
485
486
487
488
489
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
490
          instance2.Release()
491
      finally:
492
        instance1.Release()
493

494
495
496
497
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
498
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
499
500
501
502
503
504
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
505
          instance.Release()
506
      finally:
507
        snode.Release()
508
509
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
510
    node.Release()
511
512


513
514
515
516
517
518
519
def RunInstanceTests():
  """Create and exercise instances."""
  instance_tests = [
    ("instance-add-plain-disk", constants.DT_PLAIN,
     qa_instance.TestInstanceAddWithPlainDisk, 1),
    ("instance-add-drbd-disk", constants.DT_DRBD8,
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
520
521
    ("instance-add-diskless", constants.DT_DISKLESS,
     qa_instance.TestInstanceAddDiskless, 1),
522
523
524
525
526
527
528
529
  ]

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
530
531
532
533
534
535
536
537
        try:
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
538
                      inodes[0].primary, inodes[1].primary)
539
540
541
542
543
544
545
546
547
548
549
550
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
          RunCommonInstanceTests(instance)
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
551
          instance.Release()
552
553
554
555
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
556
557


558
559
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
560
561

  """
562
563
564
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
565
  RunEnvTests()
566
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
567
568

  # Load RAPI certificate
569
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
570

Michael Hanselmann's avatar
Michael Hanselmann committed
571
572
  RunClusterTests()
  RunOsTests()
573

Iustin Pop's avatar
Iustin Pop committed
574
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
575

576
  RunCommonNodeTests()
577
  RunGroupListTests()
578
  RunGroupRwTests()
Helga Velroyen's avatar
Helga Velroyen committed
579
  RunNetworkTests()
580

581
582
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
583
584
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
585
586
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
587
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
588
  finally:
589
    pnode.Release()
590

591
592
593
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
594
595
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
596
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
597

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
598
599
600
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

601
      if qa_config.TestEnabled("instance-add-plain-disk"):
602
603
604
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
605
606
607
608
609
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
              RunCommonInstanceTests(rapi_instance)
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
610
            rapi_instance.Release()
611
          del rapi_instance
612

613
  finally:
614
    pnode.Release()
615

616
617
618
619
620
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
621
  ]
622
623
624
625
626
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
627

628
629
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
630
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
631
      for shutdown in [False, True]:
632
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
633
        try:
634
635
636
637
638
639
640
641
          expnode = qa_config.AcquireNode(exclude=pnode)
          try:
            if shutdown:
              # Stop instance before exporting and removing it
              RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
            RunTest(qa_instance.TestBackupList, expnode)
          finally:
642
            expnode.Release()
643
        finally:
644
          instance.Release()
645
646
        del expnode
        del instance
647
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
648

649
  finally:
650
    pnode.Release()
651

652
653
  RunExclusiveStorageTests()

654
  # Test removing instance with offline drbd secondary
655
656
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
                            "instance-add-drbd-disk"]):
657
658
659
660
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
661
      try:
662
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
663
664
665
666
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
667
      finally:
668
        pnode.Release()
669
    finally:
670
      snode.Release()
671
    qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
672

Iustin Pop's avatar
Iustin Pop committed
673
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
674

Iustin Pop's avatar
Iustin Pop committed
675
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
676

677

678
@UsesRapiClient
679
680
681
682
683
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
684
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
685
686
                    action="store_true",
                    help="Really execute the tests")
687
  (opts, args) = parser.parse_args()
688
689
690
691
692
693

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

694
  if not opts.yes_do_it:
695
696
697
698
699
700
701
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

702
  primary = qa_config.GetMasterNode().primary
703
704
705
706
707
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
708
709
710
711
712
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
713
if __name__ == "__main__":
714
  main()