qa_cluster.py 15.2 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2010, 2011 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Cluster related QA tests.

"""

import tempfile
27
import os.path
28

29
from ganeti import constants
René Nussbaumer's avatar
René Nussbaumer committed
30
from ganeti import compat
31
32
33
34
35
36
from ganeti import utils

import qa_config
import qa_utils
import qa_error

René Nussbaumer's avatar
René Nussbaumer committed
37
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
38
39


40
41
42
#: cluster verify command
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]

43
44
45
46
def _RemoveFileFromAllNodes(filename):
  """Removes a file from all nodes.

  """
Iustin Pop's avatar
Iustin Pop committed
47
48
  for node in qa_config.get("nodes"):
    AssertCommand(["rm", "-f", filename], node=node)
49
50
51
52
53
54
55


def _CheckFileOnAllNodes(filename, content):
  """Verifies the content of the given file on all nodes.

  """
  cmd = utils.ShellQuoteArgs(["cat", filename])
Iustin Pop's avatar
Iustin Pop committed
56
57
  for node in qa_config.get("nodes"):
    AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
58
59


60
def TestClusterInit(rapi_user, rapi_secret):
61
62
63
  """gnt-cluster init"""
  master = qa_config.GetMasterNode()

64
65
  rapi_dir = os.path.dirname(constants.RAPI_USERS_FILE)

66
  # First create the RAPI credentials
67
68
69
70
71
72
73
  fh = tempfile.NamedTemporaryFile()
  try:
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
    fh.flush()

    tmpru = qa_utils.UploadFile(master["primary"], fh.name)
    try:
74
      AssertCommand(["mkdir", "-p", rapi_dir])
Iustin Pop's avatar
Iustin Pop committed
75
      AssertCommand(["mv", tmpru, constants.RAPI_USERS_FILE])
76
    finally:
Iustin Pop's avatar
Iustin Pop committed
77
      AssertCommand(["rm", "-f", tmpru])
78
79
80
81
  finally:
    fh.close()

  # Initialize cluster
Iustin Pop's avatar
Iustin Pop committed
82
  cmd = ["gnt-cluster", "init"]
83

Manuel Franceschini's avatar
Manuel Franceschini committed
84
85
86
  cmd.append("--primary-ip-version=%d" %
             qa_config.get("primary_ip_version", 4))

Iustin Pop's avatar
Iustin Pop committed
87
88
  if master.get("secondary", None):
    cmd.append("--secondary-ip=%s" % master["secondary"])
89

Iustin Pop's avatar
Iustin Pop committed
90
  bridge = qa_config.get("bridge", None)
91
  if bridge:
Iustin Pop's avatar
Iustin Pop committed
92
93
    cmd.append("--bridge=%s" % bridge)
    cmd.append("--master-netdev=%s" % bridge)
94

Iustin Pop's avatar
Iustin Pop committed
95
  htype = qa_config.get("enabled-hypervisors", None)
96
  if htype:
Iustin Pop's avatar
Iustin Pop committed
97
    cmd.append("--enabled-hypervisors=%s" % htype)
98

Iustin Pop's avatar
Iustin Pop committed
99
  cmd.append(qa_config.get("name"))
100

Iustin Pop's avatar
Iustin Pop committed
101
  AssertCommand(cmd)
102

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
  cmd = ["gnt-cluster", "modify"]
  # hypervisor parameter modifications
  hvp = qa_config.get("hypervisor-parameters", {})
  for k, v in hvp.items():
    cmd.extend(["-H", "%s:%s" % (k, v)])
  # backend parameter modifications
  bep = qa_config.get("backend-parameters", "")
  if bep:
    cmd.extend(["-B", bep])

  if len(cmd) > 2:
    AssertCommand(cmd)

  # OS parameters
  osp = qa_config.get("os-parameters", {})
  for k, v in osp.items():
    AssertCommand(["gnt-os", "modify", "-O", v, k])

  # OS hypervisor parameters
  os_hvp = qa_config.get("os-hvp", {})
  for os_name in os_hvp:
    for hv, hvp in os_hvp[os_name].items():
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])

127

128
129
def TestClusterRename():
  """gnt-cluster rename"""
Iustin Pop's avatar
Iustin Pop committed
130
  cmd = ["gnt-cluster", "rename", "-f"]
131

Iustin Pop's avatar
Iustin Pop committed
132
133
  original_name = qa_config.get("name")
  rename_target = qa_config.get("rename", None)
134
135
136
137
  if rename_target is None:
    print qa_utils.FormatError('"rename" entry is missing')
    return

Iustin Pop's avatar
Iustin Pop committed
138
139
  for data in [
    cmd + [rename_target],
140
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
141
    cmd + [original_name],
142
    _CLUSTER_VERIFY,
Iustin Pop's avatar
Iustin Pop committed
143
144
    ]:
    AssertCommand(data)
145
146


Iustin Pop's avatar
Iustin Pop committed
147
148
def TestClusterOob():
  """out-of-band framework"""
149
150
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()

151
  AssertCommand(_CLUSTER_VERIFY)
152
153
154
155
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
                 utils.NewUUID()])

156
  AssertCommand(_CLUSTER_VERIFY, fail=True)
157

Iustin Pop's avatar
Iustin Pop committed
158
159
160
  AssertCommand(["touch", oob_path_exists])
  AssertCommand(["chmod", "0400", oob_path_exists])
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
161
162
163
164
165

  try:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program=%s" % oob_path_exists])

166
    AssertCommand(_CLUSTER_VERIFY, fail=True)
167

Iustin Pop's avatar
Iustin Pop committed
168
169
    AssertCommand(["chmod", "0500", oob_path_exists])
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
170

171
    AssertCommand(_CLUSTER_VERIFY)
172
  finally:
Iustin Pop's avatar
Iustin Pop committed
173
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
174
175
176

  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                 "oob_program="])
Iustin Pop's avatar
Iustin Pop committed
177
178


René Nussbaumer's avatar
René Nussbaumer committed
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
def TestClusterEpo():
  """gnt-cluster epo"""
  master = qa_config.GetMasterNode()

  # Assert that OOB is unavailable for all nodes
  result_output = GetCommandOutput(master["primary"],
                                   "gnt-node list --verbose --no-header -o"
                                   " powered")
  AssertEqual(compat.all(powered == "(unavail)"
                         for powered in result_output.splitlines()), True)

  # Conflicting
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
  # --all doesn't expect arguments
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)

  # Unless --all is given master is not allowed to be in the list
  AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)

  # This shouldn't fail
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])

  # All instances should have been stopped now
  result_output = GetCommandOutput(master["primary"],
                                   "gnt-instance list --no-header -o status")
  AssertEqual(compat.all(status == "ADMIN_down"
                         for status in result_output.splitlines()), True)

  # Now start everything again
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])

  # All instances should have been started now
  result_output = GetCommandOutput(master["primary"],
                                   "gnt-instance list --no-header -o status")
  AssertEqual(compat.all(status == "running"
                         for status in result_output.splitlines()), True)


Iustin Pop's avatar
Iustin Pop committed
217
218
def TestClusterVerify():
  """gnt-cluster verify"""
219
  AssertCommand(_CLUSTER_VERIFY)
220
  AssertCommand(["gnt-cluster", "verify-disks"])
221

222
223
224

def TestJobqueue():
  """gnt-debug test-jobqueue"""
Iustin Pop's avatar
Iustin Pop committed
225
  AssertCommand(["gnt-debug", "test-jobqueue"])
226
227


228
229
def TestClusterReservedLvs():
  """gnt-cluster reserved lvs"""
Iustin Pop's avatar
Iustin Pop committed
230
  for fail, cmd in [
231
    (False, _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
232
233
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
    (False, ["lvcreate", "-L1G", "-nqa-test", "xenvg"]),
234
    (True,  _CLUSTER_VERIFY),
235
236
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
             "xenvg/qa-test,.*/other-test"]),
237
    (False, _CLUSTER_VERIFY),
238
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ".*/qa-.*"]),
239
    (False, _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
240
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
241
    (True,  _CLUSTER_VERIFY),
Iustin Pop's avatar
Iustin Pop committed
242
    (False, ["lvremove", "-f", "xenvg/qa-test"]),
243
    (False, _CLUSTER_VERIFY),
244
    ]:
Iustin Pop's avatar
Iustin Pop committed
245
    AssertCommand(cmd, fail=fail)
246

247

248
249
def TestClusterModifyBe():
  """gnt-cluster modify -B"""
Iustin Pop's avatar
Iustin Pop committed
250
  for fail, cmd in [
251
    # mem
Iustin Pop's avatar
Iustin Pop committed
252
253
254
255
256
    (False, ["gnt-cluster", "modify", "-B", "memory=256"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *memory: 256$'"]),
    (True,  ["gnt-cluster", "modify", "-B", "memory=a"]),
    (False, ["gnt-cluster", "modify", "-B", "memory=128"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *memory: 128$'"]),
257
    # vcpus
Iustin Pop's avatar
Iustin Pop committed
258
259
260
261
262
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
    (True,  ["gnt-cluster", "modify", "-B", "vcpus=a"]),
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
263
    # auto_balance
Iustin Pop's avatar
Iustin Pop committed
264
265
266
267
268
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
    (True,  ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
269
    ]:
Iustin Pop's avatar
Iustin Pop committed
270
    AssertCommand(cmd, fail=fail)
271

272
273
274
275
  # redo the original-requested BE parameters, if any
  bep = qa_config.get("backend-parameters", "")
  if bep:
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
276

277
278
def TestClusterInfo():
  """gnt-cluster info"""
Iustin Pop's avatar
Iustin Pop committed
279
  AssertCommand(["gnt-cluster", "info"])
Michael Hanselmann's avatar
Michael Hanselmann committed
280
281


Iustin Pop's avatar
Iustin Pop committed
282
283
284
285
286
def TestClusterRedistConf():
  """gnt-cluster redist-conf"""
  AssertCommand(["gnt-cluster", "redist-conf"])


Michael Hanselmann's avatar
Michael Hanselmann committed
287
288
def TestClusterGetmaster():
  """gnt-cluster getmaster"""
Iustin Pop's avatar
Iustin Pop committed
289
  AssertCommand(["gnt-cluster", "getmaster"])
Michael Hanselmann's avatar
Michael Hanselmann committed
290
291
292
293


def TestClusterVersion():
  """gnt-cluster version"""
Iustin Pop's avatar
Iustin Pop committed
294
  AssertCommand(["gnt-cluster", "version"])
295
296


297
298
299
300
301
302
def TestClusterRenewCrypto():
  """gnt-cluster renew-crypto"""
  master = qa_config.GetMasterNode()

  # Conflicting options
  cmd = ["gnt-cluster", "renew-crypto", "--force",
Michael Hanselmann's avatar
Michael Hanselmann committed
303
304
305
306
307
308
         "--new-cluster-certificate", "--new-confd-hmac-key"]
  conflicting = [
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
    ]
  for i in conflicting:
Iustin Pop's avatar
Iustin Pop committed
309
    AssertCommand(cmd+i, fail=True)
310
311
312
313

  # Invalid RAPI certificate
  cmd = ["gnt-cluster", "renew-crypto", "--force",
         "--rapi-certificate=/dev/null"]
Iustin Pop's avatar
Iustin Pop committed
314
  AssertCommand(cmd, fail=True)
315

316
317
318
319
320
  rapi_cert_backup = qa_utils.BackupFile(master["primary"],
                                         constants.RAPI_CERT_FILE)
  try:
    # Custom RAPI certificate
    fh = tempfile.NamedTemporaryFile()
321

322
323
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
324

Michael Hanselmann's avatar
Michael Hanselmann committed
325
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
326

327
328
    tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
    try:
Iustin Pop's avatar
Iustin Pop committed
329
330
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--rapi-certificate=%s" % tmpcert])
331
    finally:
Iustin Pop's avatar
Iustin Pop committed
332
      AssertCommand(["rm", "-f", tmpcert])
333

Michael Hanselmann's avatar
Michael Hanselmann committed
334
335
336
337
338
339
340
341
    # Custom cluster domain secret
    cds_fh = tempfile.NamedTemporaryFile()
    cds_fh.write(utils.GenerateSecret())
    cds_fh.write("\n")
    cds_fh.flush()

    tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
    try:
Iustin Pop's avatar
Iustin Pop committed
342
343
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                     "--cluster-domain-secret=%s" % tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
344
    finally:
Iustin Pop's avatar
Iustin Pop committed
345
      AssertCommand(["rm", "-f", tmpcds])
Michael Hanselmann's avatar
Michael Hanselmann committed
346

347
    # Normal case
Iustin Pop's avatar
Iustin Pop committed
348
349
350
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--new-cluster-certificate", "--new-confd-hmac-key",
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
Michael Hanselmann's avatar
Michael Hanselmann committed
351

352
    # Restore RAPI certificate
Iustin Pop's avatar
Iustin Pop committed
353
354
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                   "--rapi-certificate=%s" % rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
355
  finally:
Iustin Pop's avatar
Iustin Pop committed
356
    AssertCommand(["rm", "-f", rapi_cert_backup])
Michael Hanselmann's avatar
Michael Hanselmann committed
357

358

359
360
361
362
def TestClusterBurnin():
  """Burnin"""
  master = qa_config.GetMasterNode()

Iustin Pop's avatar
Iustin Pop committed
363
364
365
366
367
368
  options = qa_config.get("options", {})
  disk_template = options.get("burnin-disk-template", "drbd")
  parallel = options.get("burnin-in-parallel", False)
  check_inst = options.get("burnin-check-instances", False)
  do_rename = options.get("burnin-rename", "")
  do_reboot = options.get("burnin-reboot", True)
369
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
370

371
372
373
  # Get as many instances as we need
  instances = []
  try:
374
    try:
Iustin Pop's avatar
Iustin Pop committed
375
      num = qa_config.get("options", {}).get("burnin-instances", 1)
376
      for _ in range(0, num):
377
378
379
        instances.append(qa_config.AcquireInstance())
    except qa_error.OutOfInstancesError:
      print "Not enough instances, continuing anyway."
380

381
382
    if len(instances) < 1:
      raise qa_error.Error("Burnin needs at least one instance")
383

Iustin Pop's avatar
Iustin Pop committed
384
    script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
385
    try:
386
      # Run burnin
387
      cmd = [script,
Iustin Pop's avatar
Iustin Pop committed
388
389
390
391
             "--os=%s" % qa_config.get("os"),
             "--disk-size=%s" % ",".join(qa_config.get("disk")),
             "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
             "--disk-template=%s" % disk_template]
392
      if parallel:
Iustin Pop's avatar
Iustin Pop committed
393
394
        cmd.append("--parallel")
        cmd.append("--early-release")
395
      if check_inst:
Iustin Pop's avatar
Iustin Pop committed
396
        cmd.append("--http-check")
Iustin Pop's avatar
Iustin Pop committed
397
      if do_rename:
Iustin Pop's avatar
Iustin Pop committed
398
        cmd.append("--rename=%s" % do_rename)
399
      if not do_reboot:
Iustin Pop's avatar
Iustin Pop committed
400
        cmd.append("--no-reboot")
401
      else:
Iustin Pop's avatar
Iustin Pop committed
402
403
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
      cmd += [inst["name"] for inst in instances]
Iustin Pop's avatar
Iustin Pop committed
404
      AssertCommand(cmd)
405
    finally:
Iustin Pop's avatar
Iustin Pop committed
406
407
      AssertCommand(["rm", "-f", script])

408
409
410
411
412
413
  finally:
    for inst in instances:
      qa_config.ReleaseInstance(inst)


def TestClusterMasterFailover():
414
  """gnt-cluster master-failover"""
415
416
417
  master = qa_config.GetMasterNode()
  failovermaster = qa_config.AcquireNode(exclude=master)

Iustin Pop's avatar
Iustin Pop committed
418
419
420
  cmd = ["gnt-cluster", "master-failover"]
  try:
    AssertCommand(cmd, node=failovermaster)
421
    # Back to original master node
Iustin Pop's avatar
Iustin Pop committed
422
    AssertCommand(cmd, node=master)
423
424
425
426
  finally:
    qa_config.ReleaseNode(failovermaster)


427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
def TestClusterMasterFailoverWithDrainedQueue():
  """gnt-cluster master-failover with drained queue"""
  drain_check = ["test", "-f", constants.JOB_QUEUE_DRAIN_FILE]

  master = qa_config.GetMasterNode()
  failovermaster = qa_config.AcquireNode(exclude=master)

  # Ensure queue is not drained
  for node in [master, failovermaster]:
    AssertCommand(drain_check, node=node, fail=True)

  # Drain queue on failover master
  AssertCommand(["touch", constants.JOB_QUEUE_DRAIN_FILE], node=failovermaster)

  cmd = ["gnt-cluster", "master-failover"]
  try:
    AssertCommand(drain_check, node=failovermaster)
    AssertCommand(cmd, node=failovermaster)
    AssertCommand(drain_check, fail=True)
    AssertCommand(drain_check, node=failovermaster, fail=True)

    # Back to original master node
    AssertCommand(cmd, node=master)
  finally:
    qa_config.ReleaseNode(failovermaster)

  AssertCommand(drain_check, fail=True)
  AssertCommand(drain_check, node=failovermaster, fail=True)


457
458
459
460
def TestClusterCopyfile():
  """gnt-cluster copyfile"""
  master = qa_config.GetMasterNode()

461
  uniqueid = utils.NewUUID()
462

463
464
  # Create temporary file
  f = tempfile.NamedTemporaryFile()
465
  f.write(uniqueid)
466
467
468
469
  f.flush()
  f.seek(0)

  # Upload file to master node
Iustin Pop's avatar
Iustin Pop committed
470
  testname = qa_utils.UploadFile(master["primary"], f.name)
471
472
  try:
    # Copy file to all nodes
Iustin Pop's avatar
Iustin Pop committed
473
    AssertCommand(["gnt-cluster", "copyfile", testname])
474
    _CheckFileOnAllNodes(testname, uniqueid)
475
  finally:
476
477
478
479
480
    _RemoveFileFromAllNodes(testname)


def TestClusterCommand():
  """gnt-cluster command"""
481
482
  uniqueid = utils.NewUUID()
  rfile = "/tmp/gnt%s" % utils.NewUUID()
Iustin Pop's avatar
Iustin Pop committed
483
484
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
485
486
487
                              "%s >%s" % (rcmd, rfile)])

  try:
Iustin Pop's avatar
Iustin Pop committed
488
    AssertCommand(cmd)
489
490
491
    _CheckFileOnAllNodes(rfile, uniqueid)
  finally:
    _RemoveFileFromAllNodes(rfile)
492
493
494
495


def TestClusterDestroy():
  """gnt-cluster destroy"""
Iustin Pop's avatar
Iustin Pop committed
496
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
497
498
499
500
501


def TestClusterRepairDiskSizes():
  """gnt-cluster repair-disk-sizes"""
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])