ganeti-qa.py 25.8 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

Iustin Pop's avatar
Iustin Pop committed
29
import sys
30
31
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
32

33
34
35
36
import qa_cluster
import qa_config
import qa_daemon
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
Helga Velroyen's avatar
Helga Velroyen committed
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
44
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import qa_tags
46
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
47

48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

Iustin Pop's avatar
Iustin Pop committed
55

Iustin Pop's avatar
Iustin Pop committed
56
def _FormatHeader(line, end=72):
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
  """Fill a line up to the end column.

  """
  line = "---- " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
61
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
62
63
64
65
  line = line.rstrip()
  return line


Iustin Pop's avatar
Iustin Pop committed
66
67
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
70
71
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
72
  else:
Iustin Pop's avatar
Iustin Pop committed
73
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
74

Iustin Pop's avatar
Iustin Pop committed
75
76
  return desc.rstrip(".")

77

78
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
79
80
81
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
82

Iustin Pop's avatar
Iustin Pop committed
83
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
84

Iustin Pop's avatar
Iustin Pop committed
85
86
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
87
88
89
90
  print
  print _FormatHeader("%s start %s" % (tstart, desc))

  try:
91
    retval = fn(*args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
92
93
94
95
96
    return retval
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
Iustin Pop's avatar
Iustin Pop committed
97
98


99
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
104
105
106
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
108
109
110
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
111
    # TODO: Formatting test names when non-string names are involved
Iustin Pop's avatar
Iustin Pop committed
112
113
114
115
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames))


Michael Hanselmann's avatar
Michael Hanselmann committed
116
117
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
118
119

  """
Iustin Pop's avatar
Iustin Pop committed
120
121
122
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
123

124

125
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
126
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
127

128
129
130
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
131
  """
Iustin Pop's avatar
Iustin Pop committed
132
133
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
134
135
136
137
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
138
139
140
141

  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
142
  RunTestIf("job-list", qa_job.TestJobList)
143

Iustin Pop's avatar
Iustin Pop committed
144
145
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
146
147
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
148

Iustin Pop's avatar
Iustin Pop committed
149
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
150

151
152
153
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

154
155
  RunTestIf("node-list", qa_node.TestNodeList)

156
157
158
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
159
  RunTestIf("job-list", qa_job.TestJobListFields)
160
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
161

Iustin Pop's avatar
Iustin Pop committed
162
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
163
164
165
166


def RunClusterTests():
  """Runs tests related to gnt-cluster.
167

Michael Hanselmann's avatar
Michael Hanselmann committed
168
  """
Iustin Pop's avatar
Iustin Pop committed
169
  for test, fn in [
170
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
171
172
173
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
174
    # TODO: add more cluster modify tests
175
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
176
177
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
Iustin Pop's avatar
Iustin Pop committed
178
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
179
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
180
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
Iustin Pop's avatar
Iustin Pop committed
181
182
183
184
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
185
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
186
187
    (["cluster-copyfile", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterCopyfile),
Iustin Pop's avatar
Iustin Pop committed
188
189
190
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
191
192
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
193
194
    (["cluster-oob", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterOob),
195
196
197
    (qa_rapi.Enabled, qa_rapi.TestVersion),
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
198
199
    ]:
    RunTestIf(test, fn)
200

201

202
203
204
205
206
207
208
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
209
210
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
211

Michael Hanselmann's avatar
Michael Hanselmann committed
212
  """
213
214
  os_enabled = ["os", qa_config.NoVirtualCluster]

215
  if qa_config.TestEnabled(qa_rapi.Enabled):
216
217
218
219
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
220
221
222
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
223
    ]:
224
    RunTestIf(os_enabled, fn)
225
226

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
227
228
229
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
230
    ]:
231
    RunTestIf(os_enabled, fn, rapi_getos)
232
233

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
234
235
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
236
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
237
    ]:
238
    RunTestIf(os_enabled, fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
239
240
241
242
243
244


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
245
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
246
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
247
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
248
249
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
250
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
251

252
  # Test shutdown/start via RAPI
253
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
254
            qa_rapi.TestRapiInstanceShutdown, instance)
255
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
256
257
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
258
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
259

Iustin Pop's avatar
Iustin Pop committed
260
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
261

Iustin Pop's avatar
Iustin Pop committed
262
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
263
  RunTestIf(["instance-modify", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
264
            qa_rapi.TestRapiInstanceModify, instance)
265

Iustin Pop's avatar
Iustin Pop committed
266
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
267
  RunTestIf(["instance-console", qa_rapi.Enabled],
268
            qa_rapi.TestRapiInstanceConsole, instance)
269

270
271
272
273
274
275
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
276
277
278
279
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
280
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
281
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
282
            qa_rapi.TestRapiInstanceReinstall, instance)
283

Iustin Pop's avatar
Iustin Pop committed
284
  if qa_config.TestEnabled("instance-rename"):
285
286
    tgt_instance = qa_config.AcquireInstance()
    try:
287
288
      rename_source = instance.name
      rename_target = tgt_instance.name
289
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
290
      RunTest(qa_instance.TestInstanceRenameAndBack,
291
              rename_source, rename_source)
292
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
293
294
295
296
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
297
                rename_source, rename_target)
298
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
299
300
                  rename_source, rename_target)
    finally:
301
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
302

Iustin Pop's avatar
Iustin Pop committed
303
304
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
305
306
307
308
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
309

Iustin Pop's avatar
Iustin Pop committed
310
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
311

Michael Hanselmann's avatar
Michael Hanselmann committed
312
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
313

314
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
315

316
317
318
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

319
320
321
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

322
323
324
325
326

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
327
328
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
329
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
330

331

332
333
334
335
def RunGroupListTests():
  """Run tests for listing node groups.

  """
336
337
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
338
339


Helga Velroyen's avatar
Helga Velroyen committed
340
341
342
343
344
345
346
347
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


348
349
350
351
352
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
353
354
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
355
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
356
357
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
358

359

360
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
361
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
362

363
364
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
365

Michael Hanselmann's avatar
Michael Hanselmann committed
366
  """
367
368
369
370
371
  # FIXME: export explicitly bails out on file based storage. other non-lvm
  # based storage types are untested, though. Also note that import could still
  # work, but is deeply embedded into the "export" case.
  if (qa_config.TestEnabled("instance-export") and
      instance.disk_template != constants.DT_FILE):
372
373
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

374
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
375
376
377
378
379
380
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
381
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
382
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
383
        try:
384
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
385
                  expnode, name)
386
387
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
388
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
389
        finally:
390
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
391
    finally:
392
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
393

394
395
396
397
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
  # See Issue 414.
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
      and instance.disk_template != constants.DT_FILE):
398
399
    newinst = qa_config.AcquireInstance()
    try:
400
      tnode = qa_config.AcquireNode(exclude=inodes)
401
402
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
403
                inodes, tnode)
404
      finally:
405
        tnode.Release()
406
    finally:
407
      newinst.Release()
408

Michael Hanselmann's avatar
Michael Hanselmann committed
409

410
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
411
  """Test the ganeti-watcher script.
412

Michael Hanselmann's avatar
Michael Hanselmann committed
413
  """
414
  RunTest(qa_daemon.TestPauseWatcher)
415

Iustin Pop's avatar
Iustin Pop committed
416
  RunTestIf("instance-automatic-restart",
417
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
418
  RunTestIf("instance-consecutive-failures",
419
            qa_daemon.TestInstanceConsecutiveFailures, instance)
420

421
422
  RunTest(qa_daemon.TestResumeWatcher)

423

424
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
425
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
426

Michael Hanselmann's avatar
Michael Hanselmann committed
427
  """
Iustin Pop's avatar
Iustin Pop committed
428
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
429
  RunTestIf(["instance-failover", qa_rapi.Enabled],
430
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
431

Iustin Pop's avatar
Iustin Pop committed
432
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
433
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
434
            qa_rapi.TestRapiInstanceMigrate, instance)
435

Iustin Pop's avatar
Iustin Pop committed
436
  if qa_config.TestEnabled("instance-replace-disks"):
437
438
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
439
    try:
440
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
441
      RunTest(qa_instance.TestReplaceDisks,
442
              instance, inodes, othernodes)
443
    finally:
444
445
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
446

447
448
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
449
450
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
451
    except qa_error.OutOfNodesError:
452
453
454
455
456
457
458
459
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
460
461
    try:
      RunTest(qa_instance.TestRecreateDisks,
462
              instance, inodes, othernodes)
463
    finally:
464
      qa_config.ReleaseManyNodes(acquirednodes)
465

466
467
468
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
469
470


471
472
473
474
475
476
477
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
478
    old_es = qa_cluster.TestSetExclStorCluster(False)
479
    qa_node.TestExclStorSingleNode(node)
480
481

    qa_cluster.TestSetExclStorCluster(True)
482
483
    qa_cluster.TestExclStorSharedPv(node)

484
485
486
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
487
488

      # Create and allocate instances
489
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
490
491
492
493
494
495
496
497
498
499
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
500
          instance2.Release()
501
      finally:
502
        instance1.Release()
503

504
505
506
507
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
508
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
509
510
511
512
513
514
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
515
          instance.Release()
516
      finally:
517
        snode.Release()
518
519
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
520
    node.Release()
521
522


523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
def _BuildSpecDict(par, mn, st, mx):
  return {par: {"min": mn, "std": st, "max": mx}}


def TestIPolicyPlainInstance():
  """Test instance policy interaction with instances"""
  params = ["mem-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
    print "Template %s not supported" % constants.DT_PLAIN
    return

  # This test assumes that the group policy is empty
  (_, old_specs) = qa_cluster.TestClusterSetISpecs({})
  node = qa_config.AcquireNode()
  try:
538
539
    # Log of policy changes, list of tuples: (change, policy_violated)
    history = []
540
541
542
543
544
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
    try:
      policyerror = [constants.CV_EINSTANCEPOLICY]
      for par in params:
        qa_cluster.AssertClusterVerify()
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
545
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
546
547
        # Some specs must be multiple of 4
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
548
        history.append((new_spec, True))
549
550
551
552
553
554
555
556
557
        qa_cluster.TestClusterSetISpecs(new_spec)
        qa_cluster.AssertClusterVerify(warnings=policyerror)
        if iminval > 0:
          # Some specs must be multiple of 4
          if iminval >= 4:
            upper = iminval - 4
          else:
            upper = iminval - 1
          new_spec = _BuildSpecDict(par, 0, upper, upper)
558
          history.append((new_spec, True))
559
560
561
          qa_cluster.TestClusterSetISpecs(new_spec)
          qa_cluster.AssertClusterVerify(warnings=policyerror)
        qa_cluster.TestClusterSetISpecs(old_specs)
562
        history.append((old_specs, False))
563
564
      qa_instance.TestInstanceRemove(instance)
    finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
565
      instance.Release()
566
567
568
569
570
571
572
573

    # Now we replay the same policy changes, and we expect that the instance
    # cannot be created for the cases where we had a policy violation above
    for (change, failed) in history:
      qa_cluster.TestClusterSetISpecs(change)
      if failed:
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
      # Instance creation with no policy violation has been tested already
574
  finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
575
    node.Release()
576
577


578
579
def RunInstanceTests():
  """Create and exercise instances."""
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
  instance_tests = []
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
  # FIXME: Refactor this to make the code more elegant wrt to disk templates.
  if constants.DT_PLAIN in enabled_disk_templates:
    instance_tests.append(("instance-add-plain-disk", constants.DT_PLAIN,
                           qa_instance.TestInstanceAddWithPlainDisk, 1))
  if constants.DT_DRBD8 in enabled_disk_templates:
    instance_tests.append(("instance-add-drbd-disk", constants.DT_DRBD8,
                           qa_instance.TestInstanceAddWithDrbdDisk, 2))
  if constants.DT_DISKLESS in enabled_disk_templates:
    instance_tests.append(("instance-add-diskless", constants.DT_DISKLESS,
                           qa_instance.TestInstanceAddDiskless, 1))
  if constants.DT_FILE in enabled_disk_templates:
    instance_tests.append(("instance-add-file", constants.DT_FILE,
                           qa_instance.TestInstanceAddFile, 1))
595
596
597
598
599
600
601

  for (test_name, templ, create_fun, num_nodes) in instance_tests:
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
602
603
604
605
606
607
608
609
        try:
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
610
                      inodes[0].primary, inodes[1].primary)
611
612
613
614
615
616
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
          RunCommonInstanceTests(instance)
617
618
619
620
621
          if qa_config.TestEnabled("instance-modify-primary"):
            othernode = qa_config.AcquireNode()
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
                    instance, inodes[0], othernode)
            othernode.Release()
622
623
624
625
626
627
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
628
          instance.Release()
629
630
631
632
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
633

634
635
636
637
  RunTestIf(
    "instance-add-restricted-by-disktemplates",
    qa_instance.TestInstanceCreationRestrictedByDiskTemplates)

Michael Hanselmann's avatar
Michael Hanselmann committed
638

639
640
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
641
642

  """
643
644
645
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
646
  RunEnvTests()
647
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
648
649

  # Load RAPI certificate
650
  qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
651

Michael Hanselmann's avatar
Michael Hanselmann committed
652
653
  RunClusterTests()
  RunOsTests()
654

Iustin Pop's avatar
Iustin Pop committed
655
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
656

657
  RunCommonNodeTests()
658
  RunGroupListTests()
659
  RunGroupRwTests()
Helga Velroyen's avatar
Helga Velroyen committed
660
  RunNetworkTests()
661

662
663
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
664
665
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
666
667
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
668
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
669
  finally:
670
    pnode.Release()
671

672
673
674
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
675
676
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
677
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
678

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
679
680
681
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

682
      if qa_config.TestEnabled("instance-add-plain-disk"):
683
684
685
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
686
687
688
689
690
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
              RunCommonInstanceTests(rapi_instance)
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
691
            rapi_instance.Release()
692
          del rapi_instance
693

694
  finally:
695
    pnode.Release()
696

697
698
699
700
701
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
    ("exclusive-storage-instance-tests",
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
702
  ]
703
704
705
706
707
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
      RunInstanceTests()
      restore_conf_f(oldconf)
708

709
710
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
711
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
712
      for shutdown in [False, True]:
713
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
714
        try:
715
716
717
718
719
720
721
722
          expnode = qa_config.AcquireNode(exclude=pnode)
          try:
            if shutdown:
              # Stop instance before exporting and removing it
              RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
            RunTest(qa_instance.TestBackupList, expnode)
          finally:
723
            expnode.Release()
724
        finally:
725
          instance.Release()
726
727
        del expnode
        del instance
728
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
729

730
  finally:
731
    pnode.Release()
732

733
  RunExclusiveStorageTests()
734
735
  RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
            TestIPolicyPlainInstance)
736

737
  # Test removing instance with offline drbd secondary
738
739
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
                            "instance-add-drbd-disk"]):
740
741
742
743
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
744
      try:
745
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
746
747
748
749
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
750
      finally:
751
        pnode.Release()
752
    finally:
753
      snode.Release()
754
    qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
755

Iustin Pop's avatar
Iustin Pop committed
756
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
757

Iustin Pop's avatar
Iustin Pop committed
758
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
759

760

761
@UsesRapiClient
762
763
764
765
766
def main():
  """Main program.

  """
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Iustin Pop's avatar
Iustin Pop committed
767
  parser.add_option("--yes-do-it", dest="yes_do_it",
Iustin Pop's avatar
Iustin Pop committed
768
769
                    action="store_true",
                    help="Really execute the tests")
770
  (opts, args) = parser.parse_args()
771
772
773
774
775
776

  if len(args) == 1:
    (config_file, ) = args
  else:
    parser.error("Wrong number of arguments.")

777
  if not opts.yes_do_it:
778
779
780
781
782
783
784
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  qa_config.Load(config_file)

785
  primary = qa_config.GetMasterNode().primary
786
787
788
789
790
  qa_utils.StartMultiplexer(primary)
  print ("SSH command for primary node: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
  print ("SSH command for other nodes: %s" %
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
791
792
793
794
795
  try:
    RunQa()
  finally:
    qa_utils.CloseMultiplexers()

Iustin Pop's avatar
Iustin Pop committed
796
if __name__ == "__main__":
797
  main()