ganeti-qa.py 20.2 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#!/usr/bin/python
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
23
24
25
26
27
28
"""Script for doing Q&A on Ganeti

You can create the required known_hosts file using ssh-keyscan. It's mandatory
to use the full name of a node (FQDN). For security reasons, verify the keys
before using them.
Example: ssh-keyscan -t rsa node{1,2,3,4}.example.com > known_hosts
"""
Iustin Pop's avatar
Iustin Pop committed
29
30
31
32
33
34

import os
import re
import sys
import yaml
import time
35
import tempfile
Iustin Pop's avatar
Iustin Pop committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
from datetime import datetime
from optparse import OptionParser

# I want more flexibility for testing over SSH, therefore I'm not using
# Ganeti's ssh module.
import subprocess

from ganeti import utils
from ganeti import constants

# {{{ Global variables
cfg = None
options = None
# }}}

# {{{ Errors
class Error(Exception):
  """An error occurred during Q&A testing.

  """
  pass


class OutOfNodesError(Error):
  """Out of nodes.

  """
  pass


class OutOfInstancesError(Error):
  """Out of instances.

  """
  pass
# }}}

# {{{ Utilities
def TestEnabled(test):
  """Returns True if the given test is enabled."""
  return cfg.get('tests', {}).get(test, False)


def RunTest(callable, *args):
  """Runs a test after printing a header.

  """
  if callable.__doc__:
    desc = callable.__doc__.splitlines()[0].strip()
  else:
    desc = '%r' % callable

  now = str(datetime.now())

  print
  print '---', now, ('-' * (55 - len(now)))
  print desc
  print '-' * 60

  return callable(*args)


def AssertEqual(first, second, msg=None):
  """Raises an error when values aren't equal.

  """
  if not first == second:
103
    raise Error(msg or '%r == %r' % (first, second))
Iustin Pop's avatar
Iustin Pop committed
104
105
106
107
108
109
110
111
112
113
114
115
116


def GetSSHCommand(node, cmd, strict=True):
  """Builds SSH command to be executed.

  """
  args = [ 'ssh', '-oEscapeChar=none', '-oBatchMode=yes', '-l', 'root' ]

  if strict:
    tmp = 'yes'
  else:
    tmp = 'no'
  args.append('-oStrictHostKeyChecking=%s' % tmp)
117
118
  args.append('-oClearAllForwardings=yes')
  args.append('-oForwardAgent=yes')
Iustin Pop's avatar
Iustin Pop committed
119
120
121
122
123
124
125
126
127
  args.append(node)

  if options.dry_run:
    prefix = 'exit 0; '
  else:
    prefix = ''

  args.append(prefix + cmd)

128
  print 'SSH:', utils.ShellQuoteArgs(args)
Iustin Pop's avatar
Iustin Pop committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

  return args


def StartSSH(node, cmd, strict=True):
  """Starts SSH.

  """
  args = GetSSHCommand(node, cmd, strict=strict)
  return subprocess.Popen(args, shell=False)


def UploadFile(node, file):
  """Uploads a file to a node and returns the filename.

144
145
  Caller needs to remove the returned file on the node when it's not needed
  anymore.
Iustin Pop's avatar
Iustin Pop committed
146
  """
147
148
  # Make sure nobody else has access to it while preserving local permissions
  mode = os.stat(file).st_mode & 0700
Iustin Pop's avatar
Iustin Pop committed
149

150
  cmd = ('tmp=$(tempfile --mode %o --prefix gnt) && '
Iustin Pop's avatar
Iustin Pop committed
151
152
153
154
155
156
157
158
159
160
         '[[ -f "${tmp}" ]] && '
         'cat > "${tmp}" && '
         'echo "${tmp}"') % mode

  f = open(file, 'r')
  try:
    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
                         stdout=subprocess.PIPE)
    AssertEqual(p.wait(), 0)

161
162
    # Return temporary filename
    return p.stdout.read().strip()
Iustin Pop's avatar
Iustin Pop committed
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
  finally:
    f.close()
# }}}

# {{{ Config helpers
def GetMasterNode():
  return cfg['nodes'][0]


def AcquireInstance():
  """Returns an instance which isn't in use.

  """
  # Filter out unwanted instances
  tmp_flt = lambda inst: not inst.get('_used', False)
  instances = filter(tmp_flt, cfg['instances'])
  del tmp_flt

  if len(instances) == 0:
182
    raise OutOfInstancesError("No instances left")
Iustin Pop's avatar
Iustin Pop committed
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210

  inst = instances[0]
  inst['_used'] = True
  return inst


def ReleaseInstance(inst):
  inst['_used'] = False


def AcquireNode(exclude=None):
  """Returns the least used node.

  """
  master = GetMasterNode()

  # Filter out unwanted nodes
  # TODO: Maybe combine filters
  if exclude is None:
    nodes = cfg['nodes'][:]
  else:
    nodes = filter(lambda node: node != exclude, cfg['nodes'])

  tmp_flt = lambda node: node.get('_added', False) or node == master
  nodes = filter(tmp_flt, nodes)
  del tmp_flt

  if len(nodes) == 0:
211
    raise OutOfNodesError("No nodes left")
Iustin Pop's avatar
Iustin Pop committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

  # Get node with least number of uses
  def compare(a, b):
    result = cmp(a.get('_count', 0), b.get('_count', 0))
    if result == 0:
      result = cmp(a['primary'], b['primary'])
    return result

  nodes.sort(cmp=compare)

  node = nodes[0]
  node['_count'] = node.get('_count', 0) + 1
  return node


def ReleaseNode(node):
  node['_count'] = node.get('_count', 0) - 1
# }}}

# {{{ Environment tests
def TestConfig():
  """Test configuration for sanity.

  """
  if len(cfg['nodes']) < 1:
237
    raise Error("Need at least one node")
Iustin Pop's avatar
Iustin Pop committed
238
  if len(cfg['instances']) < 1:
239
    raise Error("Need at least one instance")
Iustin Pop's avatar
Iustin Pop committed
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
  # TODO: Add more checks


def TestSshConnection():
  """Test SSH connection.

  """
  for node in cfg['nodes']:
    AssertEqual(StartSSH(node['primary'], 'exit').wait(), 0)


def TestGanetiCommands():
  """Test availibility of Ganeti commands.

  """
  cmds = ( ['gnt-cluster', '--version'],
           ['gnt-os', '--version'],
           ['gnt-node', '--version'],
           ['gnt-instance', '--version'],
           ['gnt-backup', '--version'],
           ['ganeti-noded', '--version'],
           ['ganeti-watcher', '--version'] )

263
  cmd = ' && '.join([utils.ShellQuoteArgs(i) for i in cmds])
Iustin Pop's avatar
Iustin Pop committed
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280

  for node in cfg['nodes']:
    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)


def TestIcmpPing():
  """ICMP ping each node.

  """
  for node in cfg['nodes']:
    check = []
    for i in cfg['nodes']:
      check.append(i['primary'])
      if i.has_key('secondary'):
        check.append(i['secondary'])

    ping = lambda ip: utils.ShellQuoteArgs(['ping', '-w', '3', '-c', '1', ip])
281
    cmd = ' && '.join([ping(i) for i in check])
Iustin Pop's avatar
Iustin Pop committed
282
283
284
285
286
287
288
289
290
291
292
293

    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
# }}}

# {{{ Cluster tests
def TestClusterInit():
  """gnt-cluster init"""
  master = GetMasterNode()

  cmd = ['gnt-cluster', 'init']
  if master.get('secondary', None):
    cmd.append('--secondary-ip=%s' % master['secondary'])
294
295
  if cfg.get('bridge', None):
    cmd.append('--bridge=%s' % cfg['bridge'])
296
    cmd.append('--master-netdev=%s' % cfg['bridge'])
Iustin Pop's avatar
Iustin Pop committed
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
  cmd.append(cfg['name'])

  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestClusterVerify():
  """gnt-cluster verify"""
  cmd = ['gnt-cluster', 'verify']
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestClusterInfo():
  """gnt-cluster info"""
  cmd = ['gnt-cluster', 'info']
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestClusterBurnin():
  """Burnin"""
  master = GetMasterNode()

  # Get as many instances as we need
  instances = []
  try:
    for _ in xrange(0, cfg.get('options', {}).get('burnin-instances', 1)):
      instances.append(AcquireInstance())
  except OutOfInstancesError:
    print "Not enough instances, continuing anyway."

  if len(instances) < 1:
330
    raise Error("Burnin needs at least one instance")
Iustin Pop's avatar
Iustin Pop committed
331
332
333
334
335

  # Run burnin
  try:
    script = UploadFile(master['primary'], '../tools/burnin')
    try:
336
337
338
339
      cmd = [script,
             '--os=%s' % cfg['os'],
             '--os-size=%s' % cfg['os-size'],
             '--swap-size=%s' % cfg['swap-size']]
340
      cmd += [inst['name'] for inst in instances]
Iustin Pop's avatar
Iustin Pop committed
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
      AssertEqual(StartSSH(master['primary'],
                           utils.ShellQuoteArgs(cmd)).wait(), 0)
    finally:
      cmd = ['rm', '-f', script]
      AssertEqual(StartSSH(master['primary'],
                           utils.ShellQuoteArgs(cmd)).wait(), 0)
  finally:
    for inst in instances:
      ReleaseInstance(inst)


def TestClusterMasterFailover():
  """gnt-cluster masterfailover"""
  master = GetMasterNode()

  failovermaster = AcquireNode(exclude=master)
  try:
    cmd = ['gnt-cluster', 'masterfailover']
    AssertEqual(StartSSH(failovermaster['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)

    cmd = ['gnt-cluster', 'masterfailover']
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
  finally:
    ReleaseNode(failovermaster)


369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
def TestClusterCopyfile():
  """gnt-cluster copyfile"""
  master = GetMasterNode()

  # Create temporary file
  f = tempfile.NamedTemporaryFile()
  f.write("I'm a testfile.\n")
  f.flush()
  f.seek(0)

  # Upload file to master node
  testname = UploadFile(master['primary'], f.name)
  try:
    # Copy file to all nodes
    cmd = ['gnt-cluster', 'copyfile', testname]
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
  finally:
    # Remove file from all nodes
    for node in cfg['nodes']:
      cmd = ['rm', '-f', testname]
      AssertEqual(StartSSH(node['primary'],
                           utils.ShellQuoteArgs(cmd)).wait(), 0)


Iustin Pop's avatar
Iustin Pop committed
394
395
396
397
398
399
400
401
402
403
def TestClusterDestroy():
  """gnt-cluster destroy"""
  cmd = ['gnt-cluster', 'destroy', '--yes-do-it']
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
# }}}

# {{{ Node tests
def _NodeAdd(node):
  if node.get('_added', False):
404
    raise Error("Node %s already in cluster" % node['primary'])
Iustin Pop's avatar
Iustin Pop committed
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436

  cmd = ['gnt-node', 'add']
  if node.get('secondary', None):
    cmd.append('--secondary-ip=%s' % node['secondary'])
  cmd.append(node['primary'])
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  node['_added'] = True


def TestNodeAddAll():
  """Adding all nodes to cluster."""
  master = GetMasterNode()
  for node in cfg['nodes']:
    if node != master:
      _NodeAdd(node)


def _NodeRemove(node):
  cmd = ['gnt-node', 'remove', node['primary']]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
  node['_added'] = False


def TestNodeRemoveAll():
  """Removing all nodes from cluster."""
  master = GetMasterNode()
  for node in cfg['nodes']:
    if node != master:
      _NodeRemove(node)
437
438
439
440
441
442
443


def TestNodeInfo():
  """gnt-node info"""
  cmd = ['gnt-node', 'info']
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
444
445
446
447
448
449
450


def TestNodeVolumes():
  """gnt-node volumes"""
  cmd = ['gnt-node', 'volumes']
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
Iustin Pop's avatar
Iustin Pop committed
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
# }}}

# {{{ Instance tests
def _DiskTest(node, instance, args):
  cmd = ['gnt-instance', 'add',
         '--os-type=%s' % cfg['os'],
         '--os-size=%s' % cfg['os-size'],
         '--swap-size=%s' % cfg['swap-size'],
         '--memory=%s' % cfg['mem'],
         '--node=%s' % node['primary']]
  if args:
    cmd += args
  cmd.append(instance['name'])

  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
  return instance


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
  return _DiskTest(node, AcquireInstance(), ['--disk-template=plain'])


def TestInstanceAddWithLocalMirrorDisk(node):
  """gnt-instance add -t local_raid1"""
  return _DiskTest(node, AcquireInstance(), ['--disk-template=local_raid1'])


def TestInstanceAddWithRemoteRaidDisk(node, node2):
  """gnt-instance add -t remote_raid1"""
  return _DiskTest(node, AcquireInstance(),
                   ['--disk-template=remote_raid1',
                    '--secondary-node=%s' % node2['primary']])


def TestInstanceRemove(instance):
  """gnt-instance remove"""
  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
  cmd = ['gnt-instance', 'startup', instance['name']]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
  cmd = ['gnt-instance', 'shutdown', instance['name']]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestInstanceFailover(instance):
  """gnt-instance failover"""
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
515
516
517
518
519
520
521


def TestInstanceInfo(instance):
  """gnt-instance info"""
  cmd = ['gnt-instance', 'info', instance['name']]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
Iustin Pop's avatar
Iustin Pop committed
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
# }}}

# {{{ Daemon tests
def _ResolveInstanceName(instance):
  """Gets the full Xen name of an instance.

  """
  master = GetMasterNode()

  info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
  sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])

  cmd = '%s | %s' % (info_cmd, sed_cmd)
  p = subprocess.Popen(GetSSHCommand(master['primary'], cmd), shell=False,
                       stdout=subprocess.PIPE)
  AssertEqual(p.wait(), 0)

  return p.stdout.read().strip()


def _InstanceRunning(node, name):
  """Checks whether an instance is running.

  Args:
    node: Node the instance runs on
    name: Full name of Xen instance
  """
  cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
  ret = StartSSH(node['primary'], cmd).wait()
  return ret == 0


def _XmShutdownInstance(node, name):
  """Shuts down instance using "xm" and waits for completion.

  Args:
    node: Node the instance runs on
    name: Full name of Xen instance
  """
  cmd = ['xm', 'shutdown', name]
  AssertEqual(StartSSH(GetMasterNode()['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  # Wait up to a minute
  end = time.time() + 60
  while time.time() <= end:
    if not _InstanceRunning(node, name):
      break
    time.sleep(5)
  else:
572
    raise Error("xm shutdown failed")
Iustin Pop's avatar
Iustin Pop committed
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605


def _ResetWatcherDaemon(node):
  """Removes the watcher daemon's state file.

  Args:
    node: Node to be reset
  """
  cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
  AssertEqual(StartSSH(node['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestInstanceAutomaticRestart(node, instance):
  """Test automatic restart of instance by ganeti-watcher.

  Note: takes up to 6 minutes to complete.
  """
  master = GetMasterNode()
  inst_name = _ResolveInstanceName(instance)

  _ResetWatcherDaemon(node)
  _XmShutdownInstance(node, inst_name)

  # Give it a bit more than five minutes to start again
  restart_at = time.time() + 330

  # Wait until it's running again
  while time.time() <= restart_at:
    if _InstanceRunning(node, inst_name):
      break
    time.sleep(15)
  else:
606
    raise Error("Daemon didn't restart instance in time")
Iustin Pop's avatar
Iustin Pop committed
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635

  cmd = ['gnt-instance', 'info', inst_name]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestInstanceConsecutiveFailures(node, instance):
  """Test five consecutive instance failures.

  Note: takes at least 35 minutes to complete.
  """
  master = GetMasterNode()
  inst_name = _ResolveInstanceName(instance)

  _ResetWatcherDaemon(node)
  _XmShutdownInstance(node, inst_name)

  # Do shutdowns for 30 minutes
  finished_at = time.time() + (35 * 60)

  while time.time() <= finished_at:
    if _InstanceRunning(node, inst_name):
      _XmShutdownInstance(node, inst_name)
    time.sleep(30)

  # Check for some time whether the instance doesn't start again
  check_until = time.time() + 330
  while time.time() <= check_until:
    if _InstanceRunning(node, inst_name):
636
      raise Error("Instance started when it shouldn't")
Iustin Pop's avatar
Iustin Pop committed
637
638
639
640
641
642
643
    time.sleep(30)

  cmd = ['gnt-instance', 'info', inst_name]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
# }}}

644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
# {{{ Other tests
def TestUploadKnownHostsFile(localpath):
  """Uploading known_hosts file.

  """
  master = GetMasterNode()

  tmpfile = UploadFile(master['primary'], localpath)
  try:
    cmd = ['mv', tmpfile, constants.SSH_KNOWN_HOSTS_FILE]
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
  except:
    cmd = ['rm', '-f', tmpfile]
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
    raise
# }}}

Iustin Pop's avatar
Iustin Pop committed
663
664
665
# {{{ Main program
if __name__ == '__main__':
  # {{{ Option parsing
666
667
  parser = OptionParser(usage="%prog [options] <config-file> "
                              "<known-hosts-file>")
Iustin Pop's avatar
Iustin Pop committed
668
669
670
671
672
673
674
675
676
  parser.add_option('--dry-run', dest='dry_run',
      action="store_true",
      help="Show what would be done")
  parser.add_option('--yes-do-it', dest='yes_do_it',
      action="store_true",
      help="Really execute the tests")
  (options, args) = parser.parse_args()
  # }}}

677
678
  if len(args) == 2:
    (config_file, known_hosts_file) = args
Iustin Pop's avatar
Iustin Pop committed
679
  else:
680
    parser.error("Not enough arguments.")
Iustin Pop's avatar
Iustin Pop committed
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695

  if not options.yes_do_it:
    print ("Executing this script irreversibly destroys any Ganeti\n"
           "configuration on all nodes involved. If you really want\n"
           "to start testing, supply the --yes-do-it option.")
    sys.exit(1)

  f = open(config_file, 'r')
  try:
    cfg = yaml.load(f.read())
  finally:
    f.close()

  RunTest(TestConfig)

696
697
  RunTest(TestUploadKnownHostsFile, known_hosts_file)

Iustin Pop's avatar
Iustin Pop committed
698
699
700
701
702
703
704
  if TestEnabled('env'):
    RunTest(TestSshConnection)
    RunTest(TestIcmpPing)
    RunTest(TestGanetiCommands)

  RunTest(TestClusterInit)

705
706
  RunTest(TestNodeAddAll)

Iustin Pop's avatar
Iustin Pop committed
707
708
  if TestEnabled('cluster-verify'):
    RunTest(TestClusterVerify)
709
710

  if TestEnabled('cluster-info'):
Iustin Pop's avatar
Iustin Pop committed
711
712
    RunTest(TestClusterInfo)

713
714
  if TestEnabled('cluster-copyfile'):
    RunTest(TestClusterCopyfile)
Iustin Pop's avatar
Iustin Pop committed
715

716
717
718
  if TestEnabled('node-info'):
    RunTest(TestNodeInfo)

Iustin Pop's avatar
Iustin Pop committed
719
720
721
722
723
724
725
726
727
728
729
730
731
  if TestEnabled('cluster-burnin'):
    RunTest(TestClusterBurnin)

  if TestEnabled('cluster-master-failover'):
    RunTest(TestClusterMasterFailover)

  node = AcquireNode()
  try:
    if TestEnabled('instance-add-plain-disk'):
      instance = RunTest(TestInstanceAddWithPlainDisk, node)
      RunTest(TestInstanceShutdown, instance)
      RunTest(TestInstanceStartup, instance)

732
733
734
      if TestEnabled('instance-info'):
        RunTest(TestInstanceInfo, instance)

Iustin Pop's avatar
Iustin Pop committed
735
736
737
738
739
740
      if TestEnabled('instance-automatic-restart'):
        RunTest(TestInstanceAutomaticRestart, node, instance)

      if TestEnabled('instance-consecutive-failures'):
        RunTest(TestInstanceConsecutiveFailures, node, instance)

741
742
743
      if TestEnabled('node-volumes'):
        RunTest(TestNodeVolumes)

Iustin Pop's avatar
Iustin Pop committed
744
745
746
747
748
749
750
      RunTest(TestInstanceRemove, instance)
      del instance

    if TestEnabled('instance-add-local-mirror-disk'):
      instance = RunTest(TestInstanceAddWithLocalMirrorDisk, node)
      RunTest(TestInstanceShutdown, instance)
      RunTest(TestInstanceStartup, instance)
751
752
753
754

      if TestEnabled('instance-info'):
        RunTest(TestInstanceInfo, instance)

755
756
757
      if TestEnabled('node-volumes'):
        RunTest(TestNodeVolumes)

Iustin Pop's avatar
Iustin Pop committed
758
759
760
761
762
763
764
765
766
767
      RunTest(TestInstanceRemove, instance)
      del instance

    if TestEnabled('instance-add-remote-raid-disk'):
      node2 = AcquireNode(exclude=node)
      try:
        instance = RunTest(TestInstanceAddWithRemoteRaidDisk, node, node2)
        RunTest(TestInstanceShutdown, instance)
        RunTest(TestInstanceStartup, instance)

768
769
770
        if TestEnabled('instance-info'):
          RunTest(TestInstanceInfo, instance)

Iustin Pop's avatar
Iustin Pop committed
771
772
773
        if TestEnabled('instance-failover'):
          RunTest(TestInstanceFailover, instance)

774
775
776
        if TestEnabled('node-volumes'):
          RunTest(TestNodeVolumes)

Iustin Pop's avatar
Iustin Pop committed
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
        RunTest(TestInstanceRemove, instance)
        del instance
      finally:
        ReleaseNode(node2)

  finally:
    ReleaseNode(node)

  RunTest(TestNodeRemoveAll)

  if TestEnabled('cluster-destroy'):
    RunTest(TestClusterDestroy)
# }}}

# vim: foldmethod=marker :