ganeti-qa.py 8.96 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
2
3
#!/usr/bin/python
#

4
# Copyright (C) 2007 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23
24
25
26
27
28

You can create the required known_hosts file using ssh-keyscan. It's mandatory
to use the full name of a node (FQDN). For security reasons, verify the keys
before using them.
Example: ssh-keyscan -t rsa node{1,2,3,4}.example.com > known_hosts
"""
Iustin Pop's avatar
Iustin Pop committed
29
30

import sys
31
32
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
33

34
35
36
37
38
39
import qa_cluster
import qa_config
import qa_daemon
import qa_env
import qa_instance
import qa_node
40
import qa_os
41
import qa_other
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
42
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
43
import qa_tags
44
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
45
46


47
def RunTest(fn, *args):
Iustin Pop's avatar
Iustin Pop committed
48
49
50
  """Runs a test after printing a header.

  """
51
52
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
53
  else:
54
    desc = '%r' % fn
Iustin Pop's avatar
Iustin Pop committed
55

56
  now = str(datetime.datetime.now())
Iustin Pop's avatar
Iustin Pop committed
57
58
59
60
61
62

  print
  print '---', now, ('-' * (55 - len(now)))
  print desc
  print '-' * 60

63
  return fn(*args)
Iustin Pop's avatar
Iustin Pop committed
64
65


Michael Hanselmann's avatar
Michael Hanselmann committed
66
67
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
68
69

  """
Michael Hanselmann's avatar
Michael Hanselmann committed
70
71
  if not qa_config.TestEnabled('env'):
    return
Iustin Pop's avatar
Iustin Pop committed
72

Michael Hanselmann's avatar
Michael Hanselmann committed
73
74
75
  RunTest(qa_env.TestSshConnection)
  RunTest(qa_env.TestIcmpPing)
  RunTest(qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
76

77

Michael Hanselmann's avatar
Michael Hanselmann committed
78
79
def SetupCluster():
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
80

Michael Hanselmann's avatar
Michael Hanselmann committed
81
  """
82
83
  RunTest(qa_cluster.TestClusterInit)
  RunTest(qa_node.TestNodeAddAll)
Michael Hanselmann's avatar
Michael Hanselmann committed
84
85
86
87
88
89
  if qa_config.TestEnabled('node-info'):
    RunTest(qa_node.TestNodeInfo)


def RunClusterTests():
  """Runs tests related to gnt-cluster.
90

Michael Hanselmann's avatar
Michael Hanselmann committed
91
  """
92
93
  if qa_config.TestEnabled('cluster-verify'):
    RunTest(qa_cluster.TestClusterVerify)
94

95
96
97
  if qa_config.TestEnabled('cluster-rename'):
    RunTest(qa_cluster.TestClusterRename)

98
  if qa_config.TestEnabled('cluster-info'):
99
    RunTest(qa_cluster.TestClusterVersion)
100
    RunTest(qa_cluster.TestClusterInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
101
102
    RunTest(qa_cluster.TestClusterGetmaster)

103
104
  if qa_config.TestEnabled('cluster-copyfile'):
    RunTest(qa_cluster.TestClusterCopyfile)
Iustin Pop's avatar
Iustin Pop committed
105

106
107
108
  if qa_config.TestEnabled('cluster-command'):
    RunTest(qa_cluster.TestClusterCommand)

109
110
  if qa_config.TestEnabled('cluster-burnin'):
    RunTest(qa_cluster.TestClusterBurnin)
Iustin Pop's avatar
Iustin Pop committed
111

112
113
  if qa_config.TestEnabled('cluster-master-failover'):
    RunTest(qa_cluster.TestClusterMasterFailover)
Iustin Pop's avatar
Iustin Pop committed
114

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
115
116
117
  if qa_rapi.Enabled():
    RunTest(qa_rapi.TestVersion)
    RunTest(qa_rapi.TestEmptyCluster)
118

Michael Hanselmann's avatar
Michael Hanselmann committed
119
120
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
121

Michael Hanselmann's avatar
Michael Hanselmann committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
  """
  if not qa_config.TestEnabled('os'):
    return

  RunTest(qa_os.TestOsList)
  RunTest(qa_os.TestOsDiagnose)
  RunTest(qa_os.TestOsValid)
  RunTest(qa_os.TestOsInvalid)
  RunTest(qa_os.TestOsPartiallyValid)


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
  if qa_config.TestEnabled('instance-shutdown'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
140

Michael Hanselmann's avatar
Michael Hanselmann committed
141
142
  if qa_config.TestEnabled('instance-list'):
    RunTest(qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
143

Michael Hanselmann's avatar
Michael Hanselmann committed
144
145
  if qa_config.TestEnabled('instance-info'):
    RunTest(qa_instance.TestInstanceInfo, instance)
146

147
148
149
  if qa_config.TestEnabled('instance-modify'):
    RunTest(qa_instance.TestInstanceModify, instance)

150
151
152
  if qa_config.TestEnabled('instance-console'):
    RunTest(qa_instance.TestInstanceConsole, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
153
154
155
156
  if qa_config.TestEnabled('instance-reinstall'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceReinstall, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
157

158
159
160
  if qa_config.TestEnabled('instance-reboot'):
    RunTest(qa_instance.TestInstanceReboot, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
161
162
163
  if qa_config.TestEnabled('tags'):
    RunTest(qa_tags.TestInstanceTags, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
164
165
  if qa_config.TestEnabled('node-volumes'):
    RunTest(qa_node.TestNodeVolumes)
Michael Hanselmann's avatar
Michael Hanselmann committed
166

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
167
168
  if qa_rapi.Enabled():
    RunTest(qa_rapi.TestInstance, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
169

Michael Hanselmann's avatar
Michael Hanselmann committed
170
171
def RunExportImportTests(instance, pnode):
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
172

Michael Hanselmann's avatar
Michael Hanselmann committed
173
174
175
176
177
178
179
180
181
182
  """
  if qa_config.TestEnabled('instance-export'):
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

      if qa_config.TestEnabled('instance-import'):
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
183
        try:
Michael Hanselmann's avatar
Michael Hanselmann committed
184
185
186
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
                  expnode, name)
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
187
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
188
189
190
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
191

Michael Hanselmann's avatar
Michael Hanselmann committed
192

Michael Hanselmann's avatar
Michael Hanselmann committed
193
194
def RunDaemonTests(instance, pnode):
  """Test the ganeti-watcher script.
195

Michael Hanselmann's avatar
Michael Hanselmann committed
196
197
198
199
200
  """
  automatic_restart = \
    qa_config.TestEnabled('instance-automatic-restart')
  consecutive_failures = \
    qa_config.TestEnabled('instance-consecutive-failures')
Iustin Pop's avatar
Iustin Pop committed
201

Michael Hanselmann's avatar
Michael Hanselmann committed
202
203
  if automatic_restart or consecutive_failures:
    qa_daemon.PrintCronWarning()
Michael Hanselmann's avatar
Michael Hanselmann committed
204

Michael Hanselmann's avatar
Michael Hanselmann committed
205
206
    if automatic_restart:
      RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
207

Michael Hanselmann's avatar
Michael Hanselmann committed
208
    if consecutive_failures:
209
      RunTest(qa_daemon.TestInstanceConsecutiveFailures, pnode, instance)
210

211

Michael Hanselmann's avatar
Michael Hanselmann committed
212
213
def RunHardwareFailureTests(instance, pnode, snode):
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
214

Michael Hanselmann's avatar
Michael Hanselmann committed
215
216
217
218
  """
  if qa_config.TestEnabled('instance-failover'):
    RunTest(qa_instance.TestInstanceFailover, instance)

219
  if qa_config.TestEnabled('instance-replace-disks'):
220
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
221
222
223
224
225
226
    try:
      RunTest(qa_instance.TestReplaceDisks,
              instance, pnode, snode, othernode)
    finally:
      qa_config.ReleaseNode(othernode)

Michael Hanselmann's avatar
Michael Hanselmann committed
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
  if qa_config.TestEnabled('node-evacuate'):
    RunTest(qa_node.TestNodeEvacuate, pnode, snode)

  if qa_config.TestEnabled('node-failover'):
    RunTest(qa_node.TestNodeFailover, pnode, snode)

  if qa_config.TestEnabled('instance-disk-failure'):
    RunTest(qa_instance.TestInstanceMasterDiskFailure,
            instance, pnode, snode)
    RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
            instance, pnode, snode)


def main():
  """Main program.

  """
244
245
  parser = optparse.OptionParser(usage="%prog [options] <config-file>"
                                       " <known-hosts-file>")
Michael Hanselmann's avatar
Michael Hanselmann committed
246
247
248
249
250
251
252
  parser.add_option('--dry-run', dest='dry_run',
      action="store_true",
      help="Show what would be done")
  parser.add_option('--yes-do-it', dest='yes_do_it',
      action="store_true",
      help="Really execute the tests")
  (qa_config.options, args) = parser.parse_args()
Michael Hanselmann's avatar
Michael Hanselmann committed
253

Michael Hanselmann's avatar
Michael Hanselmann committed
254
255
256
257
  if len(args) == 2:
    (config_file, known_hosts_file) = args
  else:
    parser.error("Not enough arguments.")
Iustin Pop's avatar
Iustin Pop committed
258

Michael Hanselmann's avatar
Michael Hanselmann committed
259
260
261
262
263
  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)
264

Michael Hanselmann's avatar
Michael Hanselmann committed
265
  qa_config.Load(config_file)
Iustin Pop's avatar
Iustin Pop committed
266

Michael Hanselmann's avatar
Michael Hanselmann committed
267
  RunTest(qa_other.UploadKnownHostsFile, known_hosts_file)
268

Michael Hanselmann's avatar
Michael Hanselmann committed
269
270
271
272
  RunEnvTests()
  SetupCluster()
  RunClusterTests()
  RunOsTests()
273

Michael Hanselmann's avatar
Michael Hanselmann committed
274
275
276
  if qa_config.TestEnabled('tags'):
    RunTest(qa_tags.TestClusterTags)

277
278
279
280
281
282
283
284
  if qa_config.TestEnabled('node-readd'):
    master = qa_config.GetMasterNode()
    pnode = qa_config.AcquireNode(exclude=master)
    try:
      RunTest(qa_node.TestNodeReadd, pnode)
    finally:
      qa_config.ReleaseNode(pnode)

Michael Hanselmann's avatar
Michael Hanselmann committed
285
286
  pnode = qa_config.AcquireNode()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
287
288
289
    if qa_config.TestEnabled('tags'):
      RunTest(qa_tags.TestNodeTags, pnode)

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
290
291
292
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

Michael Hanselmann's avatar
Michael Hanselmann committed
293
294
295
296
297
298
299
    if qa_config.TestEnabled('instance-add-plain-disk'):
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
      RunCommonInstanceTests(instance)
      RunExportImportTests(instance, pnode)
      RunDaemonTests(instance, pnode)
      RunTest(qa_instance.TestInstanceRemove, instance)
      del instance
300

301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
    multinode_tests = [
      ('instance-add-drbd-disk',
       qa_instance.TestInstanceAddWithDrbdDisk),
    ]

    for name, func in multinode_tests:
      if qa_config.TestEnabled(name):
        snode = qa_config.AcquireNode(exclude=pnode)
        try:
          instance = RunTest(func, pnode, snode)
          RunCommonInstanceTests(instance)
          RunExportImportTests(instance, pnode)
          RunHardwareFailureTests(instance, pnode, snode)
          RunTest(qa_instance.TestInstanceRemove, instance)
          del instance
        finally:
          qa_config.ReleaseNode(snode)
Iustin Pop's avatar
Iustin Pop committed
318
319

  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
320
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
321

322
  RunTest(qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
323

324
325
  if qa_config.TestEnabled('cluster-destroy'):
    RunTest(qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
326

327
328
329

if __name__ == '__main__':
  main()