ganeti-qa.py 35.1 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

29
import copy
30
31
import datetime
import optparse
32
import sys
Iustin Pop's avatar
Iustin Pop committed
33

34
import colors
35
36
37
38
import qa_cluster
import qa_config
import qa_daemon
import qa_env
39
import qa_error
40
import qa_group
41
import qa_instance
42
import qa_iptables
43
import qa_monitoring
Helga Velroyen's avatar
Helga Velroyen committed
44
import qa_network
45
import qa_node
46
import qa_os
47
import qa_performance
48
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
49
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
50
import qa_tags
51
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
52

53
from ganeti import utils
54
from ganeti import rapi # pylint: disable=W0611
55
from ganeti import constants
56
from ganeti import netutils
57

58
import ganeti.rapi.client # pylint: disable=W0611
59
from ganeti.rapi.client import UsesRapiClient
60

Iustin Pop's avatar
Iustin Pop committed
61

62
def _FormatHeader(line, end=72, mark="-", color=None):
Iustin Pop's avatar
Iustin Pop committed
63
64
65
  """Fill a line up to the end column.

  """
66
  line = (mark * 4) + " " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
67
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
68
  line = line.rstrip()
Petr Pudlak's avatar
Petr Pudlak committed
69
  line = colors.colorize(line, color=color)
Iustin Pop's avatar
Iustin Pop committed
70
71
72
  return line


Iustin Pop's avatar
Iustin Pop committed
73
74
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
75
76

  """
77
78
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
79
80
81
    desc = desc.rstrip(".")
    if fn.__name__:
      desc = "[" + fn.__name__ + "] " + desc
Iustin Pop's avatar
Iustin Pop committed
82
  else:
Iustin Pop's avatar
Iustin Pop committed
83
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
84

85
  return desc
Iustin Pop's avatar
Iustin Pop committed
86

87

88
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
89
90
91
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
92

Iustin Pop's avatar
Iustin Pop committed
93
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
94

Iustin Pop's avatar
Iustin Pop committed
95
96
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
97
  print
98
99
  print _FormatHeader("%s start %s" % (tstart, desc),
                      color=colors.YELLOW, mark="<")
Iustin Pop's avatar
Iustin Pop committed
100
101

  try:
102
    retval = fn(*args, **kwargs)
103
    print _FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)
Iustin Pop's avatar
Iustin Pop committed
104
    return retval
105
106
107
  except Exception, e:
    print _FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)
    raise
Iustin Pop's avatar
Iustin Pop committed
108
109
110
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
111
112
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
                        color=colors.MAGENTA, mark=">")
Iustin Pop's avatar
Iustin Pop committed
113
114


115
116
117
118
119
120
121
122
123
124
125
126
127
def ReportTestSkip(desc, testnames):
  """Reports that tests have been skipped.

  @type desc: string
  @param desc: string
  @type testnames: string or list of string
  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  tstart = datetime.datetime.now()
  # TODO: Formatting test names when non-string names are involved
  print _FormatHeader("%s skipping %s, test(s) %s disabled" %
128
129
                      (tstart, desc, testnames),
                      color=colors.BLUE, mark="*")
130
131


132
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
133
134
135
136
137
138
139
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
140
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
141
142
  else:
    desc = _DescriptionOf(fn)
143
    ReportTestSkip(desc, testnames)
Iustin Pop's avatar
Iustin Pop committed
144
145


146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
def RunTestBlock(fn, *args, **kwargs):
  """Runs a block of tests after printing a header.

  """
  tstart = datetime.datetime.now()

  desc = _DescriptionOf(fn)

  print
  print _FormatHeader("BLOCK %s start %s" % (tstart, desc),
                      color=[colors.YELLOW, colors.BOLD], mark="v")

  try:
    return fn(*args, **kwargs)
  except Exception, e:
    print _FormatHeader("BLOCK FAILED %s: %s" % (desc, e),
                        color=[colors.RED, colors.BOLD])
    raise
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("BLOCK %s time=%s %s" % (tstop, tdelta, desc),
                        color=[colors.MAGENTA, colors.BOLD], mark="^")


Michael Hanselmann's avatar
Michael Hanselmann committed
171
172
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
173
174

  """
Iustin Pop's avatar
Iustin Pop committed
175
176
177
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
178

179

180
def SetupCluster(rapi_user):
Michael Hanselmann's avatar
Michael Hanselmann committed
181
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
182

183
  @param rapi_user: Login user for RAPI
184
  @return: Login secret for RAPI
185

Michael Hanselmann's avatar
Michael Hanselmann committed
186
  """
187
  rapi_secret = utils.GenerateSecret()
Iustin Pop's avatar
Iustin Pop committed
188
189
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
190
191
192
193
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
194
195
    if qa_rapi.Enabled():
      # To support RAPI on an existing cluster we have to find out the secret
196
      rapi_secret = qa_rapi.LookupRapiSecret(rapi_user)
197

198
199
  qa_group.ConfigureGroups()

200
201
202
  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
203
  RunTestIf("job-list", qa_job.TestJobList)
204

Iustin Pop's avatar
Iustin Pop committed
205
206
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
207
208
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
209

Iustin Pop's avatar
Iustin Pop committed
210
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
211
  RunTestIf("test-jobqueue", qa_job.TestJobCancellation)
212

213
214
215
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

216
217
  RunTestIf("node-list", qa_node.TestNodeList)

218
219
220
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
221
  RunTestIf("job-list", qa_job.TestJobListFields)
222
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
223

Iustin Pop's avatar
Iustin Pop committed
224
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
225

226
227
  return rapi_secret

Michael Hanselmann's avatar
Michael Hanselmann committed
228
229
230

def RunClusterTests():
  """Runs tests related to gnt-cluster.
231

Michael Hanselmann's avatar
Michael Hanselmann committed
232
  """
Iustin Pop's avatar
Iustin Pop committed
233
  for test, fn in [
234
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
235
236
237
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
238
    # TODO: add more cluster modify tests
239
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
240
241
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
Iustin Pop's avatar
Iustin Pop committed
242
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
243
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
244
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
245
246
    ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
    ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
Iustin Pop's avatar
Iustin Pop committed
247
248
249
250
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
251
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
252
253
    (["cluster-copyfile", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterCopyfile),
Iustin Pop's avatar
Iustin Pop committed
254
255
256
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
257
258
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
259
260
    (["cluster-oob", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterOob),
261
262
263
    (qa_rapi.Enabled, qa_rapi.TestVersion),
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
264
265
    ]:
    RunTestIf(test, fn)
266

267

268
269
270
271
272
273
274
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
275
276
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
277

Michael Hanselmann's avatar
Michael Hanselmann committed
278
  """
279
280
  os_enabled = ["os", qa_config.NoVirtualCluster]

281
  if qa_config.TestEnabled(qa_rapi.Enabled):
282
283
284
285
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
286
287
288
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
289
    ]:
290
    RunTestIf(os_enabled, fn)
291
292

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
293
294
295
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
296
    ]:
297
    RunTestIf(os_enabled, fn, rapi_getos)
298
299

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
300
301
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
302
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
303
    ]:
304
    RunTestIf(os_enabled, fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
305
306


307
def RunCommonInstanceTests(instance, inst_nodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
308
309
310
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
311
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
312
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
313
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
314
315
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
316
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
317

318
  # Test shutdown/start via RAPI
319
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
320
            qa_rapi.TestRapiInstanceShutdown, instance)
321
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
322
323
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
324
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
325

Iustin Pop's avatar
Iustin Pop committed
326
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
327

Iustin Pop's avatar
Iustin Pop committed
328
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
329
  RunTestIf(["instance-modify", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
330
            qa_rapi.TestRapiInstanceModify, instance)
331

Iustin Pop's avatar
Iustin Pop committed
332
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
333
  RunTestIf(["instance-console", qa_rapi.Enabled],
334
            qa_rapi.TestRapiInstanceConsole, instance)
335

336
337
  RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
            instance)
338
339
340
341
342
343
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
344
345
346
347
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
348
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
349
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
350
            qa_rapi.TestRapiInstanceReinstall, instance)
351

Iustin Pop's avatar
Iustin Pop committed
352
  if qa_config.TestEnabled("instance-rename"):
353
354
    tgt_instance = qa_config.AcquireInstance()
    try:
355
356
      rename_source = instance.name
      rename_target = tgt_instance.name
357
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
358
      RunTest(qa_instance.TestInstanceRenameAndBack,
359
              rename_source, rename_source)
360
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
361
362
363
364
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
365
                rename_source, rename_target)
366
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
367
368
                  rename_source, rename_target)
    finally:
369
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
370

Iustin Pop's avatar
Iustin Pop committed
371
372
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
373
374
375
376
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
377

Iustin Pop's avatar
Iustin Pop committed
378
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
379

380
381
382
  if instance.disk_template == constants.DT_DRBD8:
    RunTestIf("cluster-verify",
              qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
Michael Hanselmann's avatar
Michael Hanselmann committed
383
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
384

385
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
386

387
388
389
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

390
391
392
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

393
394
395
396
397

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
398
399
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
400
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
401

402

403
404
405
406
def RunGroupListTests():
  """Run tests for listing node groups.

  """
407
408
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
409
410


Helga Velroyen's avatar
Helga Velroyen committed
411
412
413
414
415
416
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)
Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
417
  RunTestIf(["network", "tags"], qa_network.TestNetworkTags)
Helga Velroyen's avatar
Helga Velroyen committed
418
419


420
421
422
423
424
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
425
426
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
427
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
428
429
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
430

431

432
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
433
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
434

435
436
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
437

Michael Hanselmann's avatar
Michael Hanselmann committed
438
  """
439
440
441
442
  # FIXME: export explicitly bails out on file based storage. other non-lvm
  # based storage types are untested, though. Also note that import could still
  # work, but is deeply embedded into the "export" case.
  if (qa_config.TestEnabled("instance-export") and
443
      instance.disk_template not in constants.DTS_FILEBASED):
444
445
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

446
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
447
448
449
450
451
452
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
453
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
454
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
455
        try:
456
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
457
                  expnode, name)
458
459
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
460
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
461
        finally:
462
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
463
    finally:
464
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
465

466
467
468
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
  # See Issue 414.
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
469
      and (instance.disk_template not in constants.DTS_FILEBASED)):
470
471
    newinst = qa_config.AcquireInstance()
    try:
472
      tnode = qa_config.AcquireNode(exclude=inodes)
473
474
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
475
                inodes, tnode)
476
      finally:
477
        tnode.Release()
478
    finally:
479
      newinst.Release()
480

Michael Hanselmann's avatar
Michael Hanselmann committed
481

482
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
483
  """Test the ganeti-watcher script.
484

Michael Hanselmann's avatar
Michael Hanselmann committed
485
  """
486
  RunTest(qa_daemon.TestPauseWatcher)
487

Iustin Pop's avatar
Iustin Pop committed
488
  RunTestIf("instance-automatic-restart",
489
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
490
  RunTestIf("instance-consecutive-failures",
491
            qa_daemon.TestInstanceConsecutiveFailures, instance)
492

493
494
  RunTest(qa_daemon.TestResumeWatcher)

495

496
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
497
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
498

Michael Hanselmann's avatar
Michael Hanselmann committed
499
  """
Iustin Pop's avatar
Iustin Pop committed
500
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
501
  RunTestIf(["instance-failover", qa_rapi.Enabled],
502
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
503

Iustin Pop's avatar
Iustin Pop committed
504
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
505
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
506
            qa_rapi.TestRapiInstanceMigrate, instance)
507

Iustin Pop's avatar
Iustin Pop committed
508
  if qa_config.TestEnabled("instance-replace-disks"):
509
510
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
511
    try:
512
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
513
      RunTest(qa_instance.TestReplaceDisks,
514
              instance, inodes, othernodes)
515
    finally:
516
517
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
518

519
520
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
521
522
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
523
    except qa_error.OutOfNodesError:
524
525
526
527
528
529
530
531
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
532
533
    try:
      RunTest(qa_instance.TestRecreateDisks,
534
              instance, inodes, othernodes)
535
    finally:
536
      qa_config.ReleaseManyNodes(acquirednodes)
537

538
539
540
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Thomas Thrainer's avatar
Thomas Thrainer committed
541
    RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
542
543


544
545
546
547
548
549
550
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
551
    old_es = qa_cluster.TestSetExclStorCluster(False)
552
    qa_node.TestExclStorSingleNode(node)
553
554

    qa_cluster.TestSetExclStorCluster(True)
555
556
    qa_cluster.TestExclStorSharedPv(node)

557
558
559
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
560
561

      # Create and allocate instances
562
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
563
564
565
566
567
568
569
570
571
572
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
573
          instance2.Release()
574
      finally:
575
        instance1.Release()
576

577
578
579
580
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
581
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
582
583
584
585
586
587
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
588
          instance.Release()
589
      finally:
590
        snode.Release()
591
592
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
593
    node.Release()
594
595


596
597
598
599
600
601
602
603
604
def RunCustomSshPortTests():
  """Test accessing nodes with custom SSH ports.

  This requires removing nodes, adding them to a new group, and then undoing
  the change.
  """
  if not qa_config.TestEnabled("group-custom-ssh-port"):
    return

605
  std_port = netutils.GetDaemonPort(constants.SSH)
606
607
608
  port = 211
  master = qa_config.GetMasterNode()
  with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
609
610
611
612
613
614
615
616
617
    # Checks if the node(s) could be contacted through IPv6.
    # If yes, better skip the whole test.

    for node in nodes:
      if qa_utils.UsesIPv6Connection(node.primary, std_port):
        print ("Node %s is likely to be reached using IPv6,"
               "skipping the test" % (node.primary, ))
        return

618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
    for node in nodes:
      qa_node.NodeRemove(node)
    with qa_iptables.RulesContext(nodes) as r:
      with qa_group.NewGroupCtx() as group:
        qa_group.ModifyGroupSshPort(r, group, nodes, port)

        for node in nodes:
          qa_node.NodeAdd(node, group=group)

        # Make sure that the cluster doesn't have any pre-existing problem
        qa_cluster.AssertClusterVerify()

        # Create and allocate instances
        instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
        try:
          instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
          try:
            # cluster-verify checks that disks are allocated correctly
            qa_cluster.AssertClusterVerify()

            # Remove instances
            qa_instance.TestInstanceRemove(instance2)
            qa_instance.TestInstanceRemove(instance1)
          finally:
            instance2.Release()
        finally:
          instance1.Release()

        for node in nodes:
          qa_node.NodeRemove(node)

    for node in nodes:
      qa_node.NodeAdd(node)

    qa_cluster.AssertClusterVerify()


655
def _BuildSpecDict(par, mn, st, mx):
656
  return {
657
658
659
660
661
    constants.ISPECS_MINMAX: [{
      constants.ISPECS_MIN: {par: mn},
      constants.ISPECS_MAX: {par: mx},
      }],
    constants.ISPECS_STD: {par: st},
662
    }
663
664


665
666
667
668
669
670
671
672
673
674
675
676
677
def _BuildDoubleSpecDict(index, par, mn, st, mx):
  new_spec = {
    constants.ISPECS_MINMAX: [{}, {}],
    }
  if st is not None:
    new_spec[constants.ISPECS_STD] = {par: st}
  new_spec[constants.ISPECS_MINMAX][index] = {
    constants.ISPECS_MIN: {par: mn},
    constants.ISPECS_MAX: {par: mx},
    }
  return new_spec


678
679
def TestIPolicyPlainInstance():
  """Test instance policy interaction with instances"""
680
  params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
681
682
683
684
685
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
    print "Template %s not supported" % constants.DT_PLAIN
    return

  # This test assumes that the group policy is empty
686
  (_, old_specs) = qa_cluster.TestClusterSetISpecs()
687
688
  # We also assume to have only one min/max bound
  assert len(old_specs[constants.ISPECS_MINMAX]) == 1
689
690
  node = qa_config.AcquireNode()
  try:
691
692
    # Log of policy changes, list of tuples:
    # (full_change, incremental_change, policy_violated)
693
    history = []
694
695
696
697
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
    try:
      policyerror = [constants.CV_EINSTANCEPOLICY]
      for par in params:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
698
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
699
700
        # Some specs must be multiple of 4
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
701
        history.append((None, new_spec, True))
702
703
704
705
706
707
708
        if iminval > 0:
          # Some specs must be multiple of 4
          if iminval >= 4:
            upper = iminval - 4
          else:
            upper = iminval - 1
          new_spec = _BuildSpecDict(par, 0, upper, upper)
709
710
          history.append((None, new_spec, True))
        history.append((old_specs, None, False))
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750

      # Test with two instance specs
      double_specs = copy.deepcopy(old_specs)
      double_specs[constants.ISPECS_MINMAX] = \
          double_specs[constants.ISPECS_MINMAX] * 2
      (par1, par2) = params[0:2]
      (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
      (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
      old_minmax = old_specs[constants.ISPECS_MINMAX][0]
      history.extend([
        (double_specs, None, False),
        # The first min/max limit is being violated
        (None,
         _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
                              imaxval1 + 4),
         False),
        # Both min/max limits are being violated
        (None,
         _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
         True),
        # The second min/max limit is being violated
        (None,
         _BuildDoubleSpecDict(0, par1,
                              old_minmax[constants.ISPECS_MIN][par1],
                              old_specs[constants.ISPECS_STD][par1],
                              old_minmax[constants.ISPECS_MAX][par1]),
         False),
        (old_specs, None, False),
        ])

      # Apply the changes, and check policy violations after each change
      qa_cluster.AssertClusterVerify()
      for (new_specs, diff_specs, failed) in history:
        qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                        diff_specs=diff_specs)
        if failed:
          qa_cluster.AssertClusterVerify(warnings=policyerror)
        else:
          qa_cluster.AssertClusterVerify()

751
752
      qa_instance.TestInstanceRemove(instance)
    finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
753
      instance.Release()
754
755
756

    # Now we replay the same policy changes, and we expect that the instance
    # cannot be created for the cases where we had a policy violation above
757
758
759
    for (new_specs, diff_specs, failed) in history:
      qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                      diff_specs=diff_specs)
760
761
762
      if failed:
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
      # Instance creation with no policy violation has been tested already
763
  finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
764
    node.Release()
765
766


767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
def IsExclusiveStorageInstanceTestEnabled():
  test_name = "exclusive-storage-instance-tests"
  if qa_config.TestEnabled(test_name):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    vgscmd = utils.ShellQuoteArgs([
      "vgs", "--noheadings", "-o", "pv_count", vgname,
      ])
    nodes = qa_config.GetConfig()["nodes"]
    for node in nodes:
      try:
        pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
      except Exception, e:
        msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
               (node.primary, test_name, e))
        raise qa_error.Error(msg)
      if pvnum < 2:
        raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
                             (node.primary, pvnum, test_name))
    res = True
  else:
    res = False
  return res


791
792
def RunInstanceTests():
  """Create and exercise instances."""
793
794
795

  for (test_name, templ, create_fun, num_nodes) in \
      qa_instance.available_instance_tests:
796
797
798
799
800
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
801
        try:
802
803
          RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
                    instance, qa_config.GetMasterNode())
804
805
806
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
807
808
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node,
                      templ == constants.DT_DRBD8)
809
810
811
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
812
                      inodes[0].primary, inodes[1].primary)
813
814
815
816
817
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
818
819
          RunTestIf("instance-modify-disks",
                    qa_instance.TestInstanceModifyDisks, instance)
820
          RunCommonInstanceTests(instance, inodes)
821
822
823
824
825
          if qa_config.TestEnabled("instance-modify-primary"):
            othernode = qa_config.AcquireNode()
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
                    instance, inodes[0], othernode)
            othernode.Release()
826
827
828
829
830
831
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
832
          instance.Release()
833
834
835
836
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
837
838
839
840
841
842
    else:
      test_desc = "Creating instances of template %s" % templ
      if not qa_config.TestEnabled(test_name):
        ReportTestSkip(test_desc, test_name)
      else:
        ReportTestSkip(test_desc, "disk template %s" % templ)
Michael Hanselmann's avatar
Michael Hanselmann committed
843
844


845
846
847
848
849
def RunMonitoringTests():
  if qa_config.TestEnabled("mon-collector"):
    RunTest(qa_monitoring.TestInstStatusCollector)


850
851
def RunPerformanceTests():
  if qa_config.TestEnabled("jobqueue-performance"):
852
    RunTest(qa_performance.TestParallelMaxInstanceCreationPerformance)
853
    RunTest(qa_performance.TestParallelNodeCountInstanceCreationPerformance)
854

855
856
857
    instances = qa_performance.CreateAllInstances()

    RunTest(qa_performance.TestParallelModify, instances)
858
    RunTest(qa_performance.TestParallelInstanceOSOperations, instances)
859
    RunTest(qa_performance.TestParallelInstanceQueries, instances)
860
861
862

    qa_performance.RemoveAllInstances(instances)

863
864
    RunTest(qa_performance.TestJobQueueSubmissionPerformance)

865
  if qa_config.TestEnabled("parallel-performance"):
866
867
868
869
    if qa_config.IsTemplateSupported(constants.DT_DRBD8):
      RunTest(qa_performance.TestParallelDRBDInstanceCreationPerformance)
    if qa_config.IsTemplateSupported(constants.DT_PLAIN):
      RunTest(qa_performance.TestParallelPlainInstanceCreationPerformance)
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886

    if qa_config.IsTemplateSupported(constants.DT_DRBD8):
      inodes = qa_config.AcquireManyNodes(2)
      try:
        instance = qa_instance.TestInstanceAddWithDrbdDisk(inodes)
        try:
          RunTest(qa_performance.TestParallelInstanceFailover, instance)
          RunTest(qa_performance.TestParallelInstanceMigration, instance)
          RunTest(qa_performance.TestParallelInstanceReplaceDisks, instance)
          RunTest(qa_performance.TestParallelInstanceReboot, instance)
          RunTest(qa_performance.TestParallelInstanceReinstall, instance)
          RunTest(qa_performance.TestParallelInstanceRename, instance)
        finally:
          qa_instance.TestInstanceRemove(instance)
          instance.Release()
      finally:
        qa_config.ReleaseManyNodes(inodes)
887

888

889
890
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
891
892

  """
893
894
  rapi_user = "ganeti-qa"

895
  RunTestBlock(RunEnvTests)
896
  rapi_secret = SetupCluster(rapi_user)
Michael Hanselmann's avatar
Michael Hanselmann committed
897

898
899
900
  if qa_rapi.Enabled():
    # Load RAPI certificate
    qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
901

902
903
  RunTestBlock(RunClusterTests)
  RunTestBlock(RunOsTests)
904

Iustin Pop's avatar
Iustin Pop committed
905
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
906

907
908
909
910
  RunTestBlock(RunCommonNodeTests)
  RunTestBlock(RunGroupListTests)
  RunTestBlock(RunGroupRwTests)
  RunTestBlock(RunNetworkTests)
911

912
913
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
914
915
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
916
917
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
918
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
919
  finally:
920
    pnode.Release()
921

922
923
924
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
925
926
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
927
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
928

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
929
930
931
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

932
933
      if (qa_config.TestEnabled("instance-add-plain-disk")
          and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
934
        # Normal instance allocation via RAPI
935
936
937
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
938
939
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
940
              RunCommonInstanceTests(rapi_instance, [pnode])
941
942
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
943
            rapi_instance.Release()
944
          del rapi_instance
945

946
947
948
949
950
951
952
953
954
955
        # Multi-instance allocation
        rapi_instance_one, rapi_instance_two = \
          RunTest(qa_rapi.TestRapiInstanceMultiAlloc, pnode)

        try:
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_one, True)
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_two, True)
        finally:
          rapi_instance_one.Release()
          rapi_instance_two.Release()
956
  finally:
957
    pnode.Release()
958

959
960
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
961
    (IsExclusiveStorageInstanceTestEnabled,
962
963
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
964
  ]
965
966
967
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
968
      RunTestBlock(RunInstanceTests)
969
      restore_conf_f(oldconf)
970

971
972
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
973
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
974
      for shutdown in [False, True]:
975
        instance = RunTest