ganeti-qa.py 11 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
2
3
#!/usr/bin/python
#

4
# Copyright (C) 2007 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24

"""
Iustin Pop's avatar
Iustin Pop committed
25
26

import sys
27
28
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
29

30
31
32
33
34
35
import qa_cluster
import qa_config
import qa_daemon
import qa_env
import qa_instance
import qa_node
36
import qa_os
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
37
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
38
import qa_tags
39
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
40

41
from ganeti import utils
42
43
44
from ganeti import rapi

import ganeti.rapi.client
45

Iustin Pop's avatar
Iustin Pop committed
46

47
def RunTest(fn, *args):
Iustin Pop's avatar
Iustin Pop committed
48
49
50
  """Runs a test after printing a header.

  """
51
52
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
53
  else:
54
    desc = '%r' % fn
Iustin Pop's avatar
Iustin Pop committed
55

56
  now = str(datetime.datetime.now())
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
61
62

  print
  print '---', now, ('-' * (55 - len(now)))
  print desc
  print '-' * 60

63
  return fn(*args)
Iustin Pop's avatar
Iustin Pop committed
64
65


Michael Hanselmann's avatar
Michael Hanselmann committed
66
67
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
Michael Hanselmann's avatar
Michael Hanselmann committed
70
71
  if not qa_config.TestEnabled('env'):
    return
Iustin Pop's avatar
Iustin Pop committed
72

Michael Hanselmann's avatar
Michael Hanselmann committed
73
74
75
  RunTest(qa_env.TestSshConnection)
  RunTest(qa_env.TestIcmpPing)
  RunTest(qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
76

77

78
def SetupCluster(rapi_user, rapi_secret):
Michael Hanselmann's avatar
Michael Hanselmann committed
79
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
80

81
82
83
  @param rapi_user: Login user for RAPI
  @param rapi_secret: Login secret for RAPI

Michael Hanselmann's avatar
Michael Hanselmann committed
84
  """
85
  if qa_config.TestEnabled('create-cluster'):
86
    RunTest(qa_cluster.TestClusterInit, rapi_user, rapi_secret)
87
    RunTest(qa_node.TestNodeAddAll)
88
89
90
  else:
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
Michael Hanselmann's avatar
Michael Hanselmann committed
91
92
93
94
95
96
  if qa_config.TestEnabled('node-info'):
    RunTest(qa_node.TestNodeInfo)


def RunClusterTests():
  """Runs tests related to gnt-cluster.
97

Michael Hanselmann's avatar
Michael Hanselmann committed
98
  """
99
100
101
  if qa_config.TestEnabled("cluster-renew-crypto"):
    RunTest(qa_cluster.TestClusterRenewCrypto)

102
103
  if qa_config.TestEnabled('cluster-verify'):
    RunTest(qa_cluster.TestClusterVerify)
104

105
106
107
  if qa_config.TestEnabled('cluster-rename'):
    RunTest(qa_cluster.TestClusterRename)

108
  if qa_config.TestEnabled('cluster-info'):
109
    RunTest(qa_cluster.TestClusterVersion)
110
    RunTest(qa_cluster.TestClusterInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
111
112
    RunTest(qa_cluster.TestClusterGetmaster)

113
114
  if qa_config.TestEnabled('cluster-copyfile'):
    RunTest(qa_cluster.TestClusterCopyfile)
Iustin Pop's avatar
Iustin Pop committed
115

116
117
118
  if qa_config.TestEnabled('cluster-command'):
    RunTest(qa_cluster.TestClusterCommand)

119
120
  if qa_config.TestEnabled('cluster-burnin'):
    RunTest(qa_cluster.TestClusterBurnin)
Iustin Pop's avatar
Iustin Pop committed
121

122
123
  if qa_config.TestEnabled('cluster-master-failover'):
    RunTest(qa_cluster.TestClusterMasterFailover)
Iustin Pop's avatar
Iustin Pop committed
124

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
125
126
127
  if qa_rapi.Enabled():
    RunTest(qa_rapi.TestVersion)
    RunTest(qa_rapi.TestEmptyCluster)
128

129

Michael Hanselmann's avatar
Michael Hanselmann committed
130
131
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
132

Michael Hanselmann's avatar
Michael Hanselmann committed
133
134
135
136
137
138
139
140
141
  """
  if not qa_config.TestEnabled('os'):
    return

  RunTest(qa_os.TestOsList)
  RunTest(qa_os.TestOsDiagnose)
  RunTest(qa_os.TestOsValid)
  RunTest(qa_os.TestOsInvalid)
  RunTest(qa_os.TestOsPartiallyValid)
142
143
  RunTest(qa_os.TestOsModifyValid)
  RunTest(qa_os.TestOsModifyInvalid)
Michael Hanselmann's avatar
Michael Hanselmann committed
144
145
146
147
148
149
150
151
152


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
  if qa_config.TestEnabled('instance-shutdown'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
153

Michael Hanselmann's avatar
Michael Hanselmann committed
154
155
  if qa_config.TestEnabled('instance-list'):
    RunTest(qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
156

Michael Hanselmann's avatar
Michael Hanselmann committed
157
158
  if qa_config.TestEnabled('instance-info'):
    RunTest(qa_instance.TestInstanceInfo, instance)
159

160
161
162
  if qa_config.TestEnabled('instance-modify'):
    RunTest(qa_instance.TestInstanceModify, instance)

163
164
165
  if qa_config.TestEnabled('instance-console'):
    RunTest(qa_instance.TestInstanceConsole, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
166
167
168
169
  if qa_config.TestEnabled('instance-reinstall'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceReinstall, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
170

171
172
173
  if qa_config.TestEnabled('instance-reboot'):
    RunTest(qa_instance.TestInstanceReboot, instance)

174
175
176
177
178
  if qa_config.TestEnabled('instance-rename'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceRename, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
179
180
181
  if qa_config.TestEnabled('tags'):
    RunTest(qa_tags.TestInstanceTags, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
182
183
  if qa_config.TestEnabled('node-volumes'):
    RunTest(qa_node.TestNodeVolumes)
Michael Hanselmann's avatar
Michael Hanselmann committed
184

185
186
187
  if qa_config.TestEnabled("node-storage"):
    RunTest(qa_node.TestNodeStorage)

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
188
189
  if qa_rapi.Enabled():
    RunTest(qa_rapi.TestInstance, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
190

191

Michael Hanselmann's avatar
Michael Hanselmann committed
192
193
def RunExportImportTests(instance, pnode):
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
194

Michael Hanselmann's avatar
Michael Hanselmann committed
195
196
197
198
199
200
201
202
203
204
  """
  if qa_config.TestEnabled('instance-export'):
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

      if qa_config.TestEnabled('instance-import'):
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
205
        try:
Michael Hanselmann's avatar
Michael Hanselmann committed
206
207
208
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
                  expnode, name)
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
209
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
210
211
212
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
213

214
215
216
217
218
219
220
221
222
223
224
225
226
  if (qa_rapi.Enabled() and
      qa_config.TestEnabled("inter-cluster-instance-move")):
    newinst = qa_config.AcquireInstance()
    try:
      pnode2 = qa_config.AcquireNode(exclude=pnode)
      try:
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
                pnode2, pnode)
      finally:
        qa_config.ReleaseNode(pnode2)
    finally:
      qa_config.ReleaseInstance(newinst)

Michael Hanselmann's avatar
Michael Hanselmann committed
227

Michael Hanselmann's avatar
Michael Hanselmann committed
228
229
def RunDaemonTests(instance, pnode):
  """Test the ganeti-watcher script.
230

Michael Hanselmann's avatar
Michael Hanselmann committed
231
232
233
234
235
  """
  automatic_restart = \
    qa_config.TestEnabled('instance-automatic-restart')
  consecutive_failures = \
    qa_config.TestEnabled('instance-consecutive-failures')
Iustin Pop's avatar
Iustin Pop committed
236

Michael Hanselmann's avatar
Michael Hanselmann committed
237
238
  if automatic_restart or consecutive_failures:
    qa_daemon.PrintCronWarning()
Michael Hanselmann's avatar
Michael Hanselmann committed
239

Michael Hanselmann's avatar
Michael Hanselmann committed
240
241
    if automatic_restart:
      RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
242

Michael Hanselmann's avatar
Michael Hanselmann committed
243
    if consecutive_failures:
244
      RunTest(qa_daemon.TestInstanceConsecutiveFailures, pnode, instance)
245

246

Michael Hanselmann's avatar
Michael Hanselmann committed
247
248
def RunHardwareFailureTests(instance, pnode, snode):
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
249

Michael Hanselmann's avatar
Michael Hanselmann committed
250
251
252
253
  """
  if qa_config.TestEnabled('instance-failover'):
    RunTest(qa_instance.TestInstanceFailover, instance)

254
  if qa_config.TestEnabled('instance-replace-disks'):
255
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
256
257
258
259
260
261
    try:
      RunTest(qa_instance.TestReplaceDisks,
              instance, pnode, snode, othernode)
    finally:
      qa_config.ReleaseNode(othernode)

Michael Hanselmann's avatar
Michael Hanselmann committed
262
263
264
265
266
267
268
269
270
271
272
273
274
  if qa_config.TestEnabled('node-evacuate'):
    RunTest(qa_node.TestNodeEvacuate, pnode, snode)

  if qa_config.TestEnabled('node-failover'):
    RunTest(qa_node.TestNodeFailover, pnode, snode)

  if qa_config.TestEnabled('instance-disk-failure'):
    RunTest(qa_instance.TestInstanceMasterDiskFailure,
            instance, pnode, snode)
    RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
            instance, pnode, snode)


275
@rapi.client.UsesRapiClient
Michael Hanselmann's avatar
Michael Hanselmann committed
276
277
278
279
def main():
  """Main program.

  """
280
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Michael Hanselmann's avatar
Michael Hanselmann committed
281
282
283
284
  parser.add_option('--yes-do-it', dest='yes_do_it',
      action="store_true",
      help="Really execute the tests")
  (qa_config.options, args) = parser.parse_args()
Michael Hanselmann's avatar
Michael Hanselmann committed
285

286
287
  if len(args) == 1:
    (config_file, ) = args
Michael Hanselmann's avatar
Michael Hanselmann committed
288
  else:
289
    parser.error("Wrong number of arguments.")
Iustin Pop's avatar
Iustin Pop committed
290

Michael Hanselmann's avatar
Michael Hanselmann committed
291
292
293
294
295
  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)
296

Michael Hanselmann's avatar
Michael Hanselmann committed
297
  qa_config.Load(config_file)
Iustin Pop's avatar
Iustin Pop committed
298

299
300
301
  rapi_user = "ganeti-qa"
  rapi_secret = utils.GenerateSecret()

Michael Hanselmann's avatar
Michael Hanselmann committed
302
  RunEnvTests()
303
  SetupCluster(rapi_user, rapi_secret)
Michael Hanselmann's avatar
Michael Hanselmann committed
304
305
306
307

  # Load RAPI certificate
  qa_rapi.Setup(rapi_user, rapi_secret)

Michael Hanselmann's avatar
Michael Hanselmann committed
308
309
  RunClusterTests()
  RunOsTests()
310

Michael Hanselmann's avatar
Michael Hanselmann committed
311
312
313
  if qa_config.TestEnabled('tags'):
    RunTest(qa_tags.TestClusterTags)

314
315
316
317
318
319
320
321
  if qa_config.TestEnabled('node-readd'):
    master = qa_config.GetMasterNode()
    pnode = qa_config.AcquireNode(exclude=master)
    try:
      RunTest(qa_node.TestNodeReadd, pnode)
    finally:
      qa_config.ReleaseNode(pnode)

Michael Hanselmann's avatar
Michael Hanselmann committed
322
323
  pnode = qa_config.AcquireNode()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
324
325
326
    if qa_config.TestEnabled('tags'):
      RunTest(qa_tags.TestNodeTags, pnode)

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
327
328
329
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

330
      if qa_config.TestEnabled("instance-add-plain-disk"):
331
332
333
334
335
336
        for use_client in [True, False]:
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                  use_client)
          RunCommonInstanceTests(rapi_instance)
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
          del rapi_instance
337

Michael Hanselmann's avatar
Michael Hanselmann committed
338
339
340
341
342
343
344
    if qa_config.TestEnabled('instance-add-plain-disk'):
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
      RunCommonInstanceTests(instance)
      RunExportImportTests(instance, pnode)
      RunDaemonTests(instance, pnode)
      RunTest(qa_instance.TestInstanceRemove, instance)
      del instance
345

346
347
348
349
350
351
352
353
354
355
356
    multinode_tests = [
      ('instance-add-drbd-disk',
       qa_instance.TestInstanceAddWithDrbdDisk),
    ]

    for name, func in multinode_tests:
      if qa_config.TestEnabled(name):
        snode = qa_config.AcquireNode(exclude=pnode)
        try:
          instance = RunTest(func, pnode, snode)
          RunCommonInstanceTests(instance)
357
          if qa_config.TestEnabled('instance-convert-disk'):
358
            RunTest(qa_instance.TestInstanceShutdown, instance)
359
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
360
            RunTest(qa_instance.TestInstanceStartup, instance)
361
362
363
364
365
366
          RunExportImportTests(instance, pnode)
          RunHardwareFailureTests(instance, pnode, snode)
          RunTest(qa_instance.TestInstanceRemove, instance)
          del instance
        finally:
          qa_config.ReleaseNode(snode)
Iustin Pop's avatar
Iustin Pop committed
367

368
369
370
371
372
373
374
375
376
377
378
379
    if (qa_config.TestEnabled('instance-add-plain-disk') and
        qa_config.TestEnabled("instance-export")):
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
      expnode = qa_config.AcquireNode(exclude=pnode)
      try:
        RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
        RunTest(qa_instance.TestBackupList, expnode)
      finally:
        qa_config.ReleaseNode(expnode)
      del expnode
      del instance

Iustin Pop's avatar
Iustin Pop committed
380
  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
381
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
382

383
384
  if qa_config.TestEnabled('create-cluster'):
    RunTest(qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
385

386
387
  if qa_config.TestEnabled('cluster-destroy'):
    RunTest(qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
388

389
390
391

if __name__ == '__main__':
  main()