ganeti-qa.py 9.23 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1 2 3
#!/usr/bin/python
#

4
# Copyright (C) 2007 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
"""Script for doing QA on Ganeti.
23 24

"""
Iustin Pop's avatar
Iustin Pop committed
25 26

import sys
27 28
import datetime
import optparse
Iustin Pop's avatar
Iustin Pop committed
29

30 31 32 33 34 35
import qa_cluster
import qa_config
import qa_daemon
import qa_env
import qa_instance
import qa_node
36
import qa_os
Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
37
import qa_rapi
Michael Hanselmann's avatar
Michael Hanselmann committed
38
import qa_tags
39
import qa_utils
Iustin Pop's avatar
Iustin Pop committed
40 41


42
def RunTest(fn, *args):
Iustin Pop's avatar
Iustin Pop committed
43 44 45
  """Runs a test after printing a header.

  """
46 47
  if fn.__doc__:
    desc = fn.__doc__.splitlines()[0].strip()
Iustin Pop's avatar
Iustin Pop committed
48
  else:
49
    desc = '%r' % fn
Iustin Pop's avatar
Iustin Pop committed
50

51
  now = str(datetime.datetime.now())
Iustin Pop's avatar
Iustin Pop committed
52 53 54 55 56 57

  print
  print '---', now, ('-' * (55 - len(now)))
  print desc
  print '-' * 60

58
  return fn(*args)
Iustin Pop's avatar
Iustin Pop committed
59 60


Michael Hanselmann's avatar
Michael Hanselmann committed
61 62
def RunEnvTests():
  """Run several environment tests.
Iustin Pop's avatar
Iustin Pop committed
63 64

  """
Michael Hanselmann's avatar
Michael Hanselmann committed
65 66
  if not qa_config.TestEnabled('env'):
    return
Iustin Pop's avatar
Iustin Pop committed
67

Michael Hanselmann's avatar
Michael Hanselmann committed
68 69 70
  RunTest(qa_env.TestSshConnection)
  RunTest(qa_env.TestIcmpPing)
  RunTest(qa_env.TestGanetiCommands)
Iustin Pop's avatar
Iustin Pop committed
71

72

Michael Hanselmann's avatar
Michael Hanselmann committed
73 74
def SetupCluster():
  """Initializes the cluster.
Iustin Pop's avatar
Iustin Pop committed
75

Michael Hanselmann's avatar
Michael Hanselmann committed
76
  """
77 78 79
  if qa_config.TestEnabled('create-cluster'):
    RunTest(qa_cluster.TestClusterInit)
    RunTest(qa_node.TestNodeAddAll)
80 81 82
  else:
    # consider the nodes are already there
    qa_node.MarkNodeAddedAll()
Michael Hanselmann's avatar
Michael Hanselmann committed
83 84 85 86 87 88
  if qa_config.TestEnabled('node-info'):
    RunTest(qa_node.TestNodeInfo)


def RunClusterTests():
  """Runs tests related to gnt-cluster.
89

Michael Hanselmann's avatar
Michael Hanselmann committed
90
  """
91 92 93
  if qa_config.TestEnabled("cluster-renew-crypto"):
    RunTest(qa_cluster.TestClusterRenewCrypto)

94 95
  if qa_config.TestEnabled('cluster-verify'):
    RunTest(qa_cluster.TestClusterVerify)
96

97 98 99
  if qa_config.TestEnabled('cluster-rename'):
    RunTest(qa_cluster.TestClusterRename)

100
  if qa_config.TestEnabled('cluster-info'):
101
    RunTest(qa_cluster.TestClusterVersion)
102
    RunTest(qa_cluster.TestClusterInfo)
Michael Hanselmann's avatar
Michael Hanselmann committed
103 104
    RunTest(qa_cluster.TestClusterGetmaster)

105 106
  if qa_config.TestEnabled('cluster-copyfile'):
    RunTest(qa_cluster.TestClusterCopyfile)
Iustin Pop's avatar
Iustin Pop committed
107

108 109 110
  if qa_config.TestEnabled('cluster-command'):
    RunTest(qa_cluster.TestClusterCommand)

111 112
  if qa_config.TestEnabled('cluster-burnin'):
    RunTest(qa_cluster.TestClusterBurnin)
Iustin Pop's avatar
Iustin Pop committed
113

114 115
  if qa_config.TestEnabled('cluster-master-failover'):
    RunTest(qa_cluster.TestClusterMasterFailover)
Iustin Pop's avatar
Iustin Pop committed
116

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
117 118 119
  if qa_rapi.Enabled():
    RunTest(qa_rapi.TestVersion)
    RunTest(qa_rapi.TestEmptyCluster)
120

121

Michael Hanselmann's avatar
Michael Hanselmann committed
122 123
def RunOsTests():
  """Runs all tests related to gnt-os.
Michael Hanselmann's avatar
Michael Hanselmann committed
124

Michael Hanselmann's avatar
Michael Hanselmann committed
125 126 127 128 129 130 131 132 133
  """
  if not qa_config.TestEnabled('os'):
    return

  RunTest(qa_os.TestOsList)
  RunTest(qa_os.TestOsDiagnose)
  RunTest(qa_os.TestOsValid)
  RunTest(qa_os.TestOsInvalid)
  RunTest(qa_os.TestOsPartiallyValid)
134 135
  RunTest(qa_os.TestOsModifyValid)
  RunTest(qa_os.TestOsModifyInvalid)
Michael Hanselmann's avatar
Michael Hanselmann committed
136 137 138 139 140 141 142 143 144


def RunCommonInstanceTests(instance):
  """Runs a few tests that are common to all disk types.

  """
  if qa_config.TestEnabled('instance-shutdown'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
145

Michael Hanselmann's avatar
Michael Hanselmann committed
146 147
  if qa_config.TestEnabled('instance-list'):
    RunTest(qa_instance.TestInstanceList)
Michael Hanselmann's avatar
Michael Hanselmann committed
148

Michael Hanselmann's avatar
Michael Hanselmann committed
149 150
  if qa_config.TestEnabled('instance-info'):
    RunTest(qa_instance.TestInstanceInfo, instance)
151

152 153 154
  if qa_config.TestEnabled('instance-modify'):
    RunTest(qa_instance.TestInstanceModify, instance)

155 156 157
  if qa_config.TestEnabled('instance-console'):
    RunTest(qa_instance.TestInstanceConsole, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
158 159 160 161
  if qa_config.TestEnabled('instance-reinstall'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceReinstall, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)
Iustin Pop's avatar
Iustin Pop committed
162

163 164 165
  if qa_config.TestEnabled('instance-reboot'):
    RunTest(qa_instance.TestInstanceReboot, instance)

166 167 168 169 170
  if qa_config.TestEnabled('instance-rename'):
    RunTest(qa_instance.TestInstanceShutdown, instance)
    RunTest(qa_instance.TestInstanceRename, instance)
    RunTest(qa_instance.TestInstanceStartup, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
171 172 173
  if qa_config.TestEnabled('tags'):
    RunTest(qa_tags.TestInstanceTags, instance)

Michael Hanselmann's avatar
Michael Hanselmann committed
174 175
  if qa_config.TestEnabled('node-volumes'):
    RunTest(qa_node.TestNodeVolumes)
Michael Hanselmann's avatar
Michael Hanselmann committed
176

177 178 179
  if qa_config.TestEnabled("node-storage"):
    RunTest(qa_node.TestNodeStorage)

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
180 181
  if qa_rapi.Enabled():
    RunTest(qa_rapi.TestInstance, instance)
Michael Hanselmann's avatar
Michael Hanselmann committed
182

183

Michael Hanselmann's avatar
Michael Hanselmann committed
184 185
def RunExportImportTests(instance, pnode):
  """Tries to export and import the instance.
Iustin Pop's avatar
Iustin Pop committed
186

Michael Hanselmann's avatar
Michael Hanselmann committed
187 188 189 190 191 192 193 194 195 196
  """
  if qa_config.TestEnabled('instance-export'):
    expnode = qa_config.AcquireNode(exclude=pnode)
    try:
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)

      RunTest(qa_instance.TestBackupList, expnode)

      if qa_config.TestEnabled('instance-import'):
        newinst = qa_config.AcquireInstance()
Michael Hanselmann's avatar
Michael Hanselmann committed
197
        try:
Michael Hanselmann's avatar
Michael Hanselmann committed
198 199 200
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
                  expnode, name)
          RunTest(qa_instance.TestInstanceRemove, newinst)
Michael Hanselmann's avatar
Michael Hanselmann committed
201
        finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
202 203 204
          qa_config.ReleaseInstance(newinst)
    finally:
      qa_config.ReleaseNode(expnode)
Michael Hanselmann's avatar
Michael Hanselmann committed
205

Michael Hanselmann's avatar
Michael Hanselmann committed
206

Michael Hanselmann's avatar
Michael Hanselmann committed
207 208
def RunDaemonTests(instance, pnode):
  """Test the ganeti-watcher script.
209

Michael Hanselmann's avatar
Michael Hanselmann committed
210 211 212 213 214
  """
  automatic_restart = \
    qa_config.TestEnabled('instance-automatic-restart')
  consecutive_failures = \
    qa_config.TestEnabled('instance-consecutive-failures')
Iustin Pop's avatar
Iustin Pop committed
215

Michael Hanselmann's avatar
Michael Hanselmann committed
216 217
  if automatic_restart or consecutive_failures:
    qa_daemon.PrintCronWarning()
Michael Hanselmann's avatar
Michael Hanselmann committed
218

Michael Hanselmann's avatar
Michael Hanselmann committed
219 220
    if automatic_restart:
      RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
221

Michael Hanselmann's avatar
Michael Hanselmann committed
222
    if consecutive_failures:
223
      RunTest(qa_daemon.TestInstanceConsecutiveFailures, pnode, instance)
224

225

Michael Hanselmann's avatar
Michael Hanselmann committed
226 227
def RunHardwareFailureTests(instance, pnode, snode):
  """Test cluster internal hardware failure recovery.
Iustin Pop's avatar
Iustin Pop committed
228

Michael Hanselmann's avatar
Michael Hanselmann committed
229 230 231 232
  """
  if qa_config.TestEnabled('instance-failover'):
    RunTest(qa_instance.TestInstanceFailover, instance)

233
  if qa_config.TestEnabled('instance-replace-disks'):
234
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
235 236 237 238 239 240
    try:
      RunTest(qa_instance.TestReplaceDisks,
              instance, pnode, snode, othernode)
    finally:
      qa_config.ReleaseNode(othernode)

Michael Hanselmann's avatar
Michael Hanselmann committed
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
  if qa_config.TestEnabled('node-evacuate'):
    RunTest(qa_node.TestNodeEvacuate, pnode, snode)

  if qa_config.TestEnabled('node-failover'):
    RunTest(qa_node.TestNodeFailover, pnode, snode)

  if qa_config.TestEnabled('instance-disk-failure'):
    RunTest(qa_instance.TestInstanceMasterDiskFailure,
            instance, pnode, snode)
    RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
            instance, pnode, snode)


def main():
  """Main program.

  """
258
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
Michael Hanselmann's avatar
Michael Hanselmann committed
259 260 261 262
  parser.add_option('--yes-do-it', dest='yes_do_it',
      action="store_true",
      help="Really execute the tests")
  (qa_config.options, args) = parser.parse_args()
Michael Hanselmann's avatar
Michael Hanselmann committed
263

264 265
  if len(args) == 1:
    (config_file, ) = args
Michael Hanselmann's avatar
Michael Hanselmann committed
266
  else:
267
    parser.error("Wrong number of arguments.")
Iustin Pop's avatar
Iustin Pop committed
268

Michael Hanselmann's avatar
Michael Hanselmann committed
269 270 271 272 273
  if not qa_config.options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)
274

Michael Hanselmann's avatar
Michael Hanselmann committed
275
  qa_config.Load(config_file)
Iustin Pop's avatar
Iustin Pop committed
276

Michael Hanselmann's avatar
Michael Hanselmann committed
277 278 279 280
  RunEnvTests()
  SetupCluster()
  RunClusterTests()
  RunOsTests()
281

Michael Hanselmann's avatar
Michael Hanselmann committed
282 283 284
  if qa_config.TestEnabled('tags'):
    RunTest(qa_tags.TestClusterTags)

285 286 287 288 289 290 291 292
  if qa_config.TestEnabled('node-readd'):
    master = qa_config.GetMasterNode()
    pnode = qa_config.AcquireNode(exclude=master)
    try:
      RunTest(qa_node.TestNodeReadd, pnode)
    finally:
      qa_config.ReleaseNode(pnode)

Michael Hanselmann's avatar
Michael Hanselmann committed
293 294
  pnode = qa_config.AcquireNode()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
295 296 297
    if qa_config.TestEnabled('tags'):
      RunTest(qa_tags.TestNodeTags, pnode)

Oleksiy Mishchenko's avatar
Oleksiy Mishchenko committed
298 299 300
    if qa_rapi.Enabled():
      RunTest(qa_rapi.TestNode, pnode)

Michael Hanselmann's avatar
Michael Hanselmann committed
301 302 303 304 305 306 307
    if qa_config.TestEnabled('instance-add-plain-disk'):
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
      RunCommonInstanceTests(instance)
      RunExportImportTests(instance, pnode)
      RunDaemonTests(instance, pnode)
      RunTest(qa_instance.TestInstanceRemove, instance)
      del instance
308

309 310 311 312 313 314 315 316 317 318 319
    multinode_tests = [
      ('instance-add-drbd-disk',
       qa_instance.TestInstanceAddWithDrbdDisk),
    ]

    for name, func in multinode_tests:
      if qa_config.TestEnabled(name):
        snode = qa_config.AcquireNode(exclude=pnode)
        try:
          instance = RunTest(func, pnode, snode)
          RunCommonInstanceTests(instance)
320 321
          if qa_config.TestEnabled('instance-convert-disk'):
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
322 323 324 325 326 327
          RunExportImportTests(instance, pnode)
          RunHardwareFailureTests(instance, pnode, snode)
          RunTest(qa_instance.TestInstanceRemove, instance)
          del instance
        finally:
          qa_config.ReleaseNode(snode)
Iustin Pop's avatar
Iustin Pop committed
328 329

  finally:
Michael Hanselmann's avatar
Michael Hanselmann committed
330
    qa_config.ReleaseNode(pnode)
Iustin Pop's avatar
Iustin Pop committed
331

332 333
  if qa_config.TestEnabled('create-cluster'):
    RunTest(qa_node.TestNodeRemoveAll)
Iustin Pop's avatar
Iustin Pop committed
334

335 336
  if qa_config.TestEnabled('cluster-destroy'):
    RunTest(qa_cluster.TestClusterDestroy)
Iustin Pop's avatar
Iustin Pop committed
337

338 339 340

if __name__ == '__main__':
  main()