ganeti-qa.py 32.7 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#!/usr/bin/python -u
Iustin Pop's avatar
Iustin Pop committed
2
3
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
# due to invalid name

29
import copy
30
31
import datetime
import optparse
32
import sys
Iustin Pop's avatar
Iustin Pop committed
33

34
import colors
35
36
37
38
import qa_cluster
import qa_config
import qa_daemon
import qa_env
39
import qa_error
40
import qa_group
41
import qa_instance
42
import qa_iptables
43
import qa_monitoring
Helga Velroyen's avatar
Helga Velroyen committed
44
import qa_network
45
import qa_node
46
import qa_os
47
import qa_job
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
48
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
49
import qa_tags
50
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
51

52
from ganeti import utils
53
from ganeti import rapi # pylint: disable=W0611
54
from ganeti import constants
55
from ganeti import pathutils
56

57
from ganeti.http.auth import ParsePasswordFile
58
import ganeti.rapi.client # pylint: disable=W0611
59
from ganeti.rapi.client import UsesRapiClient
60

Iustin Pop's avatar
Iustin Pop committed
61

62
def _FormatHeader(line, end=72, mark="-", color=None):
Iustin Pop's avatar
Iustin Pop committed
63
64
65
  """Fill a line up to the end column.

  """
66
  line = (mark * 4) + " " + line + " "
Andrea Spadaccini's avatar
Andrea Spadaccini committed
67
  line += "-" * (end - len(line))
Iustin Pop's avatar
Iustin Pop committed
68
  line = line.rstrip()
Petr Pudlak's avatar
Petr Pudlak committed
69
  line = colors.colorize(line, color=color)
Iustin Pop's avatar
Iustin Pop committed
70
71
72
  return line


Iustin Pop's avatar
Iustin Pop committed
73
74
def _DescriptionOf(fn):
  """Computes the description of an item.
Iustin Pop's avatar
Iustin Pop committed
75
76

  """
77
78
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
79
80
81
    desc = desc.rstrip(".")
    if fn.__name__:
      desc = "[" + fn.__name__ + "] " + desc
Iustin Pop's avatar
Iustin Pop committed
82
  else:
Iustin Pop's avatar
Iustin Pop committed
83
    desc = "%r" % fn
Iustin Pop's avatar
Iustin Pop committed
84

85
  return desc
Iustin Pop's avatar
Iustin Pop committed
86

87

88
def RunTest(fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
89
90
91
  """Runs a test after printing a header.

  """
Iustin Pop's avatar
Iustin Pop committed
92

Iustin Pop's avatar
Iustin Pop committed
93
  tstart = datetime.datetime.now()
Iustin Pop's avatar
Iustin Pop committed
94

Iustin Pop's avatar
Iustin Pop committed
95
96
  desc = _DescriptionOf(fn)

Iustin Pop's avatar
Iustin Pop committed
97
  print
98
99
  print _FormatHeader("%s start %s" % (tstart, desc),
                      color=colors.YELLOW, mark="<")
Iustin Pop's avatar
Iustin Pop committed
100
101

  try:
102
    retval = fn(*args, **kwargs)
103
    print _FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)
Iustin Pop's avatar
Iustin Pop committed
104
    return retval
105
106
107
  except Exception, e:
    print _FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)
    raise
Iustin Pop's avatar
Iustin Pop committed
108
109
110
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
111
112
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
                        color=colors.MAGENTA, mark=">")
Iustin Pop's avatar
Iustin Pop committed
113
114


115
def RunTestIf(testnames, fn, *args, **kwargs):
Iustin Pop's avatar
Iustin Pop committed
116
117
118
119
120
121
122
  """Runs a test conditionally.

  @param testnames: either a single test name in the configuration
      file, or a list of testnames (which will be AND-ed together)

  """
  if qa_config.TestEnabled(testnames):
123
    RunTest(fn, *args, **kwargs)
Iustin Pop's avatar
Iustin Pop committed
124
125
126
  else:
    tstart = datetime.datetime.now()
    desc = _DescriptionOf(fn)
127
    # TODO: Formatting test names when non-string names are involved
Iustin Pop's avatar
Iustin Pop committed
128
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
129
130
                        (tstart, desc, testnames),
                        color=colors.BLUE, mark="*")
Iustin Pop's avatar
Iustin Pop committed
131
132


133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
def RunTestBlock(fn, *args, **kwargs):
  """Runs a block of tests after printing a header.

  """
  tstart = datetime.datetime.now()

  desc = _DescriptionOf(fn)

  print
  print _FormatHeader("BLOCK %s start %s" % (tstart, desc),
                      color=[colors.YELLOW, colors.BOLD], mark="v")

  try:
    return fn(*args, **kwargs)
  except Exception, e:
    print _FormatHeader("BLOCK FAILED %s: %s" % (desc, e),
                        color=[colors.RED, colors.BOLD])
    raise
  finally:
    tstop = datetime.datetime.now()
    tdelta = tstop - tstart
    print _FormatHeader("BLOCK %s time=%s %s" % (tstop, tdelta, desc),
                        color=[colors.MAGENTA, colors.BOLD], mark="^")


Michael Hanselmann's avatar
Michael Hanselmann committed
158
159
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
160
161

  """
Iustin Pop's avatar
Iustin Pop committed
162
163
164
  RunTestIf("env", qa_env.TestSshConnection)
  RunTestIf("env", qa_env.TestIcmpPing)
  RunTestIf("env", qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
165

166

167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
def _LookupRapiSecret(rapi_user):
  """Find the RAPI secret for the given user.

  @param rapi_user: Login user
  @return: Login secret for the user

  """
  CTEXT = "{CLEARTEXT}"
  master = qa_config.GetMasterNode()
  cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)]
  file_content = qa_utils.GetCommandOutput(master.primary,
                                           utils.ShellQuoteArgs(cmd))
  users = ParsePasswordFile(file_content)
  entry = users.get(rapi_user)
  if not entry:
    raise qa_error.Error("User %s not found in RAPI users file" % rapi_user)
  secret = entry.password
  if secret.upper().startswith(CTEXT):
    secret = secret[len(CTEXT):]
  elif secret.startswith("{"):
    raise qa_error.Error("Unsupported password schema for RAPI user %s:"
                         " not a clear text password" % rapi_user)
  return secret


def SetupCluster(rapi_user):
Michael Hanselmann's avatar
Michael Hanselmann committed
193
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
194

195
  @param rapi_user: Login user for RAPI
196
  @return: Login secret for RAPI
197

Michael Hanselmann's avatar
Michael Hanselmann committed
198
  """
199
  rapi_secret = utils.GenerateSecret()
Iustin Pop's avatar
Iustin Pop committed
200
201
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
            rapi_user, rapi_secret)
202
203
204
205
  if not qa_config.TestEnabled("create-cluster"):
    # If the cluster is already in place, we assume that exclusive-storage is
    # already set according to the configuration
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
206
207
208
    if qa_rapi.Enabled():
      # To support RAPI on an existing cluster we have to find out the secret
      rapi_secret = _LookupRapiSecret(rapi_user)
209

210
211
  qa_group.ConfigureGroups()

212
213
214
  # Test on empty cluster
  RunTestIf("node-list", qa_node.TestNodeList)
  RunTestIf("instance-list", qa_instance.TestInstanceList)
215
  RunTestIf("job-list", qa_job.TestJobList)
216

Iustin Pop's avatar
Iustin Pop committed
217
218
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
  if not qa_config.TestEnabled("create-cluster"):
219
220
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
221

Iustin Pop's avatar
Iustin Pop committed
222
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
223

224
225
226
  # enable the watcher (unconditionally)
  RunTest(qa_daemon.TestResumeWatcher)

227
228
  RunTestIf("node-list", qa_node.TestNodeList)

229
230
231
  # Test listing fields
  RunTestIf("node-list", qa_node.TestNodeListFields)
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
232
  RunTestIf("job-list", qa_job.TestJobListFields)
233
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
234

Iustin Pop's avatar
Iustin Pop committed
235
  RunTestIf("node-info", qa_node.TestNodeInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
236

237
238
  return rapi_secret

Michael Hanselmann's avatar
Michael Hanselmann committed
239
240
241

def RunClusterTests():
  """Runs tests related to gnt-cluster.
242

Michael Hanselmann's avatar
Michael Hanselmann committed
243
  """
Iustin Pop's avatar
Iustin Pop committed
244
  for test, fn in [
245
    ("create-cluster", qa_cluster.TestClusterInitDisk),
Iustin Pop's avatar
Iustin Pop committed
246
247
248
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
    ("cluster-verify", qa_cluster.TestClusterVerify),
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
249
    # TODO: add more cluster modify tests
250
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
251
252
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
Iustin Pop's avatar
Iustin Pop committed
253
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
254
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
255
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
256
257
    ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
    ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
Iustin Pop's avatar
Iustin Pop committed
258
259
260
261
    ("cluster-rename", qa_cluster.TestClusterRename),
    ("cluster-info", qa_cluster.TestClusterVersion),
    ("cluster-info", qa_cluster.TestClusterInfo),
    ("cluster-info", qa_cluster.TestClusterGetmaster),
Iustin Pop's avatar
Iustin Pop committed
262
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
263
264
    (["cluster-copyfile", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterCopyfile),
Iustin Pop's avatar
Iustin Pop committed
265
266
267
    ("cluster-command", qa_cluster.TestClusterCommand),
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
268
269
    ("cluster-master-failover",
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
270
271
    (["cluster-oob", qa_config.NoVirtualCluster],
     qa_cluster.TestClusterOob),
272
273
274
    (qa_rapi.Enabled, qa_rapi.TestVersion),
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
Iustin Pop's avatar
Iustin Pop committed
275
276
    ]:
    RunTestIf(test, fn)
277

278

279
280
281
282
283
284
285
def RunRepairDiskSizes():
  """Run the repair disk-sizes test.

  """
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)


Michael Hanselmann's avatar
Michael Hanselmann committed
286
287
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
288

Michael Hanselmann's avatar
Michael Hanselmann committed
289
  """
290
291
  os_enabled = ["os", qa_config.NoVirtualCluster]

292
  if qa_config.TestEnabled(qa_rapi.Enabled):
293
294
295
296
    rapi_getos = qa_rapi.GetOperatingSystems
  else:
    rapi_getos = None

Iustin Pop's avatar
Iustin Pop committed
297
298
299
  for fn in [
    qa_os.TestOsList,
    qa_os.TestOsDiagnose,
300
    ]:
301
    RunTestIf(os_enabled, fn)
302
303

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
304
305
306
    qa_os.TestOsValid,
    qa_os.TestOsInvalid,
    qa_os.TestOsPartiallyValid,
307
    ]:
308
    RunTestIf(os_enabled, fn, rapi_getos)
309
310

  for fn in [
Iustin Pop's avatar
Iustin Pop committed
311
312
    qa_os.TestOsModifyValid,
    qa_os.TestOsModifyInvalid,
313
    qa_os.TestOsStatesNonExisting,
Iustin Pop's avatar
Iustin Pop committed
314
    ]:
315
    RunTestIf(os_enabled, fn)
Michael Hanselmann's avatar
Michael Hanselmann committed
316
317


318
def RunCommonInstanceTests(instance, inst_nodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
319
320
321
  """Runs a few tests that are common to all disk types.

  """
Iustin Pop's avatar
Iustin Pop committed
322
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
323
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
324
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
325
326
  RunTestIf(["instance-shutdown", "instance-modify"],
            qa_instance.TestInstanceStoppedModify, instance)
Iustin Pop's avatar
Iustin Pop committed
327
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
328

329
  # Test shutdown/start via RAPI
330
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
331
            qa_rapi.TestRapiInstanceShutdown, instance)
332
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
333
334
            qa_rapi.TestRapiInstanceStartup, instance)

Iustin Pop's avatar
Iustin Pop committed
335
  RunTestIf("instance-list", qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
336

Iustin Pop's avatar
Iustin Pop committed
337
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
338

Iustin Pop's avatar
Iustin Pop committed
339
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
340
  RunTestIf(["instance-modify", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
341
            qa_rapi.TestRapiInstanceModify, instance)
342

Iustin Pop's avatar
Iustin Pop committed
343
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
344
  RunTestIf(["instance-console", qa_rapi.Enabled],
345
            qa_rapi.TestRapiInstanceConsole, instance)
346

347
348
  RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
            instance)
349
350
351
352
353
354
  DOWN_TESTS = qa_config.Either([
    "instance-reinstall",
    "instance-rename",
    "instance-grow-disk",
    ])

Iustin Pop's avatar
Iustin Pop committed
355
356
357
358
  # shutdown instance for any 'down' tests
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)

  # now run the 'down' state tests
Iustin Pop's avatar
Iustin Pop committed
359
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
360
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
361
            qa_rapi.TestRapiInstanceReinstall, instance)
362

Iustin Pop's avatar
Iustin Pop committed
363
  if qa_config.TestEnabled("instance-rename"):
364
365
    tgt_instance = qa_config.AcquireInstance()
    try:
366
367
      rename_source = instance.name
      rename_target = tgt_instance.name
368
      # perform instance rename to the same name
Iustin Pop's avatar
Iustin Pop committed
369
      RunTest(qa_instance.TestInstanceRenameAndBack,
370
              rename_source, rename_source)
371
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
372
373
374
375
                rename_source, rename_source)
      if rename_target is not None:
        # perform instance rename to a different name, if we have one configured
        RunTest(qa_instance.TestInstanceRenameAndBack,
376
                rename_source, rename_target)
377
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
378
379
                  rename_source, rename_target)
    finally:
380
      tgt_instance.Release()
Iustin Pop's avatar
Iustin Pop committed
381

Iustin Pop's avatar
Iustin Pop committed
382
383
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)

Iustin Pop's avatar
Iustin Pop committed
384
385
386
387
  # and now start the instance again
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)

  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
388

Iustin Pop's avatar
Iustin Pop committed
389
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
390

391
392
393
  if instance.disk_template == constants.DT_DRBD8:
    RunTestIf("cluster-verify",
              qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
Michael Hanselmann's avatar
Michael Hanselmann committed
394
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
Michael Hanselmann's avatar
Michael Hanselmann committed
395

396
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
397

398
399
400
  # Lists instances, too
  RunTestIf("node-list", qa_node.TestNodeList)

401
402
403
  # Some jobs have been run, let's test listing them
  RunTestIf("job-list", qa_job.TestJobList)

404
405
406
407
408

def RunCommonNodeTests():
  """Run a few common node tests.

  """
Iustin Pop's avatar
Iustin Pop committed
409
410
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
  RunTestIf("node-storage", qa_node.TestNodeStorage)
411
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
412

413

414
415
416
417
def RunGroupListTests():
  """Run tests for listing node groups.

  """
418
419
  RunTestIf("group-list", qa_group.TestGroupList)
  RunTestIf("group-list", qa_group.TestGroupListFields)
420
421


Helga Velroyen's avatar
Helga Velroyen committed
422
423
424
425
426
427
428
429
def RunNetworkTests():
  """Run tests for network management.

  """
  RunTestIf("network", qa_network.TestNetworkAddRemove)
  RunTestIf("network", qa_network.TestNetworkConnect)


430
431
432
433
434
def RunGroupRwTests():
  """Run tests for adding/removing/renaming groups.

  """
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
435
436
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
  RunTestIf("group-rwops", qa_group.TestGroupModify)
437
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
438
439
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
            qa_group.GetDefaultGroup())
440

441

442
def RunExportImportTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
443
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
444

445
446
  @type inodes: list of nodes
  @param inodes: current nodes of the instance
447

Michael Hanselmann's avatar
Michael Hanselmann committed
448
  """
449
450
451
452
  # FIXME: export explicitly bails out on file based storage. other non-lvm
  # based storage types are untested, though. Also note that import could still
  # work, but is deeply embedded into the "export" case.
  if (qa_config.TestEnabled("instance-export") and
453
      instance.disk_template not in constants.DTS_FILEBASED):
454
455
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)

456
    pnode = inodes[0]
Michael Hanselmann's avatar
Michael Hanselmann committed
457
458
459
460
461
462
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

Iustin Pop's avatar
Iustin Pop committed
463
      if qa_config.TestEnabled("instance-import"):
Michael Hanselmann's avatar
Michael Hanselmann committed
464
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
465
        try:
466
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
Michael Hanselmann's avatar
Michael Hanselmann committed
467
                  expnode, name)
468
469
          # Check if starting the instance works
          RunTest(qa_instance.TestInstanceStartup, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
470
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
471
        finally:
472
          newinst.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
473
    finally:
474
      expnode.Release()
Michael Hanselmann's avatar
Michael Hanselmann committed
475

476
477
478
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
  # See Issue 414.
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
479
      and (instance.disk_template not in constants.DTS_FILEBASED)):
480
481
    newinst = qa_config.AcquireInstance()
    try:
482
      tnode = qa_config.AcquireNode(exclude=inodes)
483
484
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
485
                inodes, tnode)
486
      finally:
487
        tnode.Release()
488
    finally:
489
      newinst.Release()
490

Michael Hanselmann's avatar
Michael Hanselmann committed
491

492
def RunDaemonTests(instance):
Michael Hanselmann's avatar
Michael Hanselmann committed
493
  """Test the ganeti-watcher script.
494

Michael Hanselmann's avatar
Michael Hanselmann committed
495
  """
496
  RunTest(qa_daemon.TestPauseWatcher)
497

Iustin Pop's avatar
Iustin Pop committed
498
  RunTestIf("instance-automatic-restart",
499
            qa_daemon.TestInstanceAutomaticRestart, instance)
Iustin Pop's avatar
Iustin Pop committed
500
  RunTestIf("instance-consecutive-failures",
501
            qa_daemon.TestInstanceConsecutiveFailures, instance)
502

503
504
  RunTest(qa_daemon.TestResumeWatcher)

505

506
def RunHardwareFailureTests(instance, inodes):
Michael Hanselmann's avatar
Michael Hanselmann committed
507
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
508

Michael Hanselmann's avatar
Michael Hanselmann committed
509
  """
Iustin Pop's avatar
Iustin Pop committed
510
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
511
  RunTestIf(["instance-failover", qa_rapi.Enabled],
512
            qa_rapi.TestRapiInstanceFailover, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
513

Iustin Pop's avatar
Iustin Pop committed
514
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
515
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
Iustin Pop's avatar
Iustin Pop committed
516
            qa_rapi.TestRapiInstanceMigrate, instance)
517

Iustin Pop's avatar
Iustin Pop committed
518
  if qa_config.TestEnabled("instance-replace-disks"):
519
520
    # We just need alternative secondary nodes, hence "- 1"
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
521
    try:
522
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
523
      RunTest(qa_instance.TestReplaceDisks,
524
              instance, inodes, othernodes)
525
    finally:
526
527
      qa_config.ReleaseManyNodes(othernodes)
    del othernodes
528

529
530
  if qa_config.TestEnabled("instance-recreate-disks"):
    try:
531
532
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
      othernodes = acquirednodes
533
    except qa_error.OutOfNodesError:
534
535
536
537
538
539
540
541
      if len(inodes) > 1:
        # If the cluster is not big enough, let's reuse some of the nodes, but
        # with different roles. In this way, we can test a DRBD instance even on
        # a 3-node cluster.
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
        othernodes = acquirednodes + inodes[:-1]
      else:
        raise
542
543
    try:
      RunTest(qa_instance.TestRecreateDisks,
544
              instance, inodes, othernodes)
545
    finally:
546
      qa_config.ReleaseManyNodes(acquirednodes)
547

548
549
550
  if len(inodes) >= 2:
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
Thomas Thrainer's avatar
Thomas Thrainer committed
551
    RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
Michael Hanselmann's avatar
Michael Hanselmann committed
552
553


554
555
556
557
558
559
560
def RunExclusiveStorageTests():
  """Test exclusive storage."""
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
    return

  node = qa_config.AcquireNode()
  try:
561
    old_es = qa_cluster.TestSetExclStorCluster(False)
562
    qa_node.TestExclStorSingleNode(node)
563
564

    qa_cluster.TestSetExclStorCluster(True)
565
566
    qa_cluster.TestExclStorSharedPv(node)

567
568
569
    if qa_config.TestEnabled("instance-add-plain-disk"):
      # Make sure that the cluster doesn't have any pre-existing problem
      qa_cluster.AssertClusterVerify()
570
571

      # Create and allocate instances
572
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
573
574
575
576
577
578
579
580
581
582
      try:
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          # cluster-verify checks that disks are allocated correctly
          qa_cluster.AssertClusterVerify()

          # Remove instances
          qa_instance.TestInstanceRemove(instance2)
          qa_instance.TestInstanceRemove(instance1)
        finally:
583
          instance2.Release()
584
      finally:
585
        instance1.Release()
586

587
588
589
590
    if qa_config.TestEnabled("instance-add-drbd-disk"):
      snode = qa_config.AcquireNode()
      try:
        qa_cluster.TestSetExclStorCluster(False)
591
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
592
593
594
595
596
597
        try:
          qa_cluster.TestSetExclStorCluster(True)
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
          qa_instance.TestInstanceRemove(instance)
        finally:
598
          instance.Release()
599
      finally:
600
        snode.Release()
601
602
    qa_cluster.TestSetExclStorCluster(old_es)
  finally:
603
    node.Release()
604
605


606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
def RunCustomSshPortTests():
  """Test accessing nodes with custom SSH ports.

  This requires removing nodes, adding them to a new group, and then undoing
  the change.
  """
  if not qa_config.TestEnabled("group-custom-ssh-port"):
    return

  port = 211
  master = qa_config.GetMasterNode()
  with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
    for node in nodes:
      qa_node.NodeRemove(node)
    with qa_iptables.RulesContext(nodes) as r:
      with qa_group.NewGroupCtx() as group:
        qa_group.ModifyGroupSshPort(r, group, nodes, port)

        for node in nodes:
          qa_node.NodeAdd(node, group=group)

        # Make sure that the cluster doesn't have any pre-existing problem
        qa_cluster.AssertClusterVerify()

        # Create and allocate instances
        instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
        try:
          instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
          try:
            # cluster-verify checks that disks are allocated correctly
            qa_cluster.AssertClusterVerify()

            # Remove instances
            qa_instance.TestInstanceRemove(instance2)
            qa_instance.TestInstanceRemove(instance1)
          finally:
            instance2.Release()
        finally:
          instance1.Release()

        for node in nodes:
          qa_node.NodeRemove(node)

    for node in nodes:
      qa_node.NodeAdd(node)

    qa_cluster.AssertClusterVerify()


655
def _BuildSpecDict(par, mn, st, mx):
656
  return {
657
658
659
660
661
    constants.ISPECS_MINMAX: [{
      constants.ISPECS_MIN: {par: mn},
      constants.ISPECS_MAX: {par: mx},
      }],
    constants.ISPECS_STD: {par: st},
662
    }
663
664


665
666
667
668
669
670
671
672
673
674
675
676
677
def _BuildDoubleSpecDict(index, par, mn, st, mx):
  new_spec = {
    constants.ISPECS_MINMAX: [{}, {}],
    }
  if st is not None:
    new_spec[constants.ISPECS_STD] = {par: st}
  new_spec[constants.ISPECS_MINMAX][index] = {
    constants.ISPECS_MIN: {par: mn},
    constants.ISPECS_MAX: {par: mx},
    }
  return new_spec


678
679
def TestIPolicyPlainInstance():
  """Test instance policy interaction with instances"""
680
  params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
681
682
683
684
685
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
    print "Template %s not supported" % constants.DT_PLAIN
    return

  # This test assumes that the group policy is empty
686
  (_, old_specs) = qa_cluster.TestClusterSetISpecs()
687
688
  # We also assume to have only one min/max bound
  assert len(old_specs[constants.ISPECS_MINMAX]) == 1
689
690
  node = qa_config.AcquireNode()
  try:
691
692
    # Log of policy changes, list of tuples:
    # (full_change, incremental_change, policy_violated)
693
    history = []
694
695
696
697
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
    try:
      policyerror = [constants.CV_EINSTANCEPOLICY]
      for par in params:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
698
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
699
700
        # Some specs must be multiple of 4
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
701
        history.append((None, new_spec, True))
702
703
704
705
706
707
708
        if iminval > 0:
          # Some specs must be multiple of 4
          if iminval >= 4:
            upper = iminval - 4
          else:
            upper = iminval - 1
          new_spec = _BuildSpecDict(par, 0, upper, upper)
709
710
          history.append((None, new_spec, True))
        history.append((old_specs, None, False))
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750

      # Test with two instance specs
      double_specs = copy.deepcopy(old_specs)
      double_specs[constants.ISPECS_MINMAX] = \
          double_specs[constants.ISPECS_MINMAX] * 2
      (par1, par2) = params[0:2]
      (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
      (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
      old_minmax = old_specs[constants.ISPECS_MINMAX][0]
      history.extend([
        (double_specs, None, False),
        # The first min/max limit is being violated
        (None,
         _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
                              imaxval1 + 4),
         False),
        # Both min/max limits are being violated
        (None,
         _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
         True),
        # The second min/max limit is being violated
        (None,
         _BuildDoubleSpecDict(0, par1,
                              old_minmax[constants.ISPECS_MIN][par1],
                              old_specs[constants.ISPECS_STD][par1],
                              old_minmax[constants.ISPECS_MAX][par1]),
         False),
        (old_specs, None, False),
        ])

      # Apply the changes, and check policy violations after each change
      qa_cluster.AssertClusterVerify()
      for (new_specs, diff_specs, failed) in history:
        qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                        diff_specs=diff_specs)
        if failed:
          qa_cluster.AssertClusterVerify(warnings=policyerror)
        else:
          qa_cluster.AssertClusterVerify()

751
752
      qa_instance.TestInstanceRemove(instance)
    finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
753
      instance.Release()
754
755
756

    # Now we replay the same policy changes, and we expect that the instance
    # cannot be created for the cases where we had a policy violation above
757
758
759
    for (new_specs, diff_specs, failed) in history:
      qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                      diff_specs=diff_specs)
760
761
762
      if failed:
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
      # Instance creation with no policy violation has been tested already
763
  finally:
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
764
    node.Release()
765
766


767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
def IsExclusiveStorageInstanceTestEnabled():
  test_name = "exclusive-storage-instance-tests"
  if qa_config.TestEnabled(test_name):
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
    vgscmd = utils.ShellQuoteArgs([
      "vgs", "--noheadings", "-o", "pv_count", vgname,
      ])
    nodes = qa_config.GetConfig()["nodes"]
    for node in nodes:
      try:
        pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
      except Exception, e:
        msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
               (node.primary, test_name, e))
        raise qa_error.Error(msg)
      if pvnum < 2:
        raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
                             (node.primary, pvnum, test_name))
    res = True
  else:
    res = False
  return res


791
792
def RunInstanceTests():
  """Create and exercise instances."""
793
794
795

  for (test_name, templ, create_fun, num_nodes) in \
      qa_instance.available_instance_tests:
796
797
798
799
800
    if (qa_config.TestEnabled(test_name) and
        qa_config.IsTemplateSupported(templ)):
      inodes = qa_config.AcquireManyNodes(num_nodes)
      try:
        instance = RunTest(create_fun, inodes)
801
        try:
802
803
          RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
                    instance, qa_config.GetMasterNode())
804
805
806
807
808
809
810
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
          RunDaemonTests(instance)
          for node in inodes:
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
          if len(inodes) > 1:
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                      constants.INITIAL_NODE_GROUP_NAME,
811
                      inodes[0].primary, inodes[1].primary)
812
813
814
815
816
          if qa_config.TestEnabled("instance-convert-disk"):
            RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
                    instance, inodes)
            RunTest(qa_instance.TestInstanceStartup, instance)
817
818
          RunTestIf("instance-modify-disks",
                    qa_instance.TestInstanceModifyDisks, instance)
819
          RunCommonInstanceTests(instance, inodes)
820
821
822
823
824
          if qa_config.TestEnabled("instance-modify-primary"):
            othernode = qa_config.AcquireNode()
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
                    instance, inodes[0], othernode)
            othernode.Release()
825
826
827
828
829
830
          RunGroupListTests()
          RunExportImportTests(instance, inodes)
          RunHardwareFailureTests(instance, inodes)
          RunRepairDiskSizes()
          RunTest(qa_instance.TestInstanceRemove, instance)
        finally:
831
          instance.Release()
832
833
834
835
        del instance
      finally:
        qa_config.ReleaseManyNodes(inodes)
      qa_cluster.AssertClusterVerify()
Michael Hanselmann's avatar
Michael Hanselmann committed
836
837


838
839
840
841
842
def RunMonitoringTests():
  if qa_config.TestEnabled("mon-collector"):
    RunTest(qa_monitoring.TestInstStatusCollector)


843
844
def RunQa():
  """Main QA body.
Michael Hanselmann's avatar
Michael Hanselmann committed
845
846

  """
847
848
  rapi_user = "ganeti-qa"

849
  RunTestBlock(RunEnvTests)
850
  rapi_secret = SetupCluster(rapi_user)
Michael Hanselmann's avatar
Michael Hanselmann committed
851

852
853
854
  if qa_rapi.Enabled():
    # Load RAPI certificate
    qa_rapi.Setup(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
855

856
857
  RunTestBlock(RunClusterTests)
  RunTestBlock(RunOsTests)
858

Iustin Pop's avatar
Iustin Pop committed
859
  RunTestIf("tags", qa_tags.TestClusterTags)
Michael Hanselmann's avatar
Michael Hanselmann committed
860

861
862
863
864
  RunTestBlock(RunCommonNodeTests)
  RunTestBlock(RunGroupListTests)
  RunTestBlock(RunGroupRwTests)
  RunTestBlock(RunNetworkTests)
865

866
867
  # The master shouldn't be readded or put offline; "delay" needs a non-master
  # node to test
868
869
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
  try:
Iustin Pop's avatar
Iustin Pop committed
870
871
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
872
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
873
  finally:
874
    pnode.Release()
875

876
877
878
  # Make sure the cluster is clean before running instance tests
  qa_cluster.AssertClusterVerify()

Michael Hanselmann's avatar
Michael Hanselmann committed
879
880
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
881
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
882

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
883
884
885
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

886
887
      if (qa_config.TestEnabled("instance-add-plain-disk")
          and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
888
889
890
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
891
892
          try:
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
893
              RunCommonInstanceTests(rapi_instance, [pnode])
894
895
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          finally:
896
            rapi_instance.Release()
897
          del rapi_instance
898

899
  finally:
900
    pnode.Release()
901

902
903
  config_list = [
    ("default-instance-tests", lambda: None, lambda _: None),
904
    (IsExclusiveStorageInstanceTestEnabled,
905
906
     lambda: qa_cluster.TestSetExclStorCluster(True),
     qa_cluster.TestSetExclStorCluster),
907
  ]
908
909
910
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
    if qa_config.TestEnabled(conf_name):
      oldconf = setup_conf_f()
911
      RunTestBlock(RunInstanceTests)
912
      restore_conf_f(oldconf)
913

914
915
  pnode = qa_config.AcquireNode()
  try:
Iustin Pop's avatar
Iustin Pop committed
916
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
917
      for shutdown in [False, True]:
918
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
919
        try:
920
921
922
923
924
925
926
927
          expnode = qa_config.AcquireNode(exclude=pnode)
          try:
            if shutdown:
              # Stop instance before exporting and removing it
              RunTest(qa_instance.TestInstanceShutdown, instance)
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
            RunTest(qa_instance.TestBackupList, expnode)
          finally:
928
            expnode.Release()
929
        finally:
930
          instance.Release()
931
932
        del expnode
        del instance
933
      qa_cluster.AssertClusterVerify()
Iustin Pop's avatar
Iustin Pop committed
934

935
  finally:
936
    pnode.Release()
937

938
939
  RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade)

940
  RunTestBlock(RunExclusiveStorageTests)
941
942
  RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
            TestIPolicyPlainInstance)
943

944
  RunTestBlock(RunCustomSshPortTests)
945

946
947
948
949
  RunTestIf(
    "instance-add-restricted-by-disktemplates",
    qa_instance.TestInstanceCreationRestrictedByDiskTemplates)

950
  # Test removing instance with offline drbd secondary
951
952
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
                            "instance-add-drbd-disk"]):
953
954
955
956
    # Make sure the master is not put offline
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      pnode = qa_config.AcquireNode(exclude=snode)
957
      try:
958
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
959
960
961
962
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                set_offline, set_online)
963
      finally:
964
        pnode.Release()
965
    finally:
966
      snode.Release()
967
    qa_cluster.AssertClusterVerify()