qa_instance.py 15 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
49
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
50
  for idx, size in enumerate(qa_config.get("disk")):
51
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
53
54


55
def _DiskTest(node, disk_template):
56
57
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
58
59
60
61
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
62
           _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
63
    cmd.append(instance["name"])
64

Iustin Pop's avatar
Iustin Pop committed
65
    AssertCommand(cmd)
66
67
68

    _CheckSsconfInstanceList(instance["name"])

69
70
71
72
73
74
75
76
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
77
  return _DiskTest(node["primary"], "plain")
78
79


80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
82
83
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
84
85


86
87
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
88
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
89
90
91
92
93
94

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
95
  AssertCommand(["gnt-instance", "startup", instance["name"]])
96
97
98
99


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
100
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
101
102


103
104
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
105
  options = qa_config.get("options", {})
106
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
107
  name = instance["name"]
108
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
109
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
110

111
112
113
114
115
116
117
118
119
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
  cmd = ["gnt-instance", "list", "--no-header", "-o", "status", name]
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

120

Michael Hanselmann's avatar
Michael Hanselmann committed
121
122
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
123
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
124
125


126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


150
def TestInstanceRename(rename_source, rename_target):
151
  """gnt-instance rename"""
152
153
154
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
155
156
157
158
159
160
161
162
163
164
165
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
166
167


168
169
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
170
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
171
172
  # failover ...
  AssertCommand(cmd)
173
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
174
  AssertCommand(cmd)
175

176

177
178
179
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
180
181
  # migrate ...
  AssertCommand(cmd)
182
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
183
  AssertCommand(cmd)
184
185
186
187
188
189
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
190
191
192
193
194
195
196
197
198
199
200
201
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                 instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "modify", "-B",
                 ("%s=%s" %
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                 instance["name"]])
  AssertCommand(cmd)
202
203


204
205
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
206
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
207
208


209
210
def TestInstanceModify(instance):
  """gnt-instance modify"""
211
212
213
214
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

215
216
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
217
  #orig_bridge = qa_config.get("bridge", "xen-br0")
218
  args = [
219
220
221
222
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
223
224
225
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
226
227
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
228
229
230
231

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
232
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
233
234
235
236
237
238
239
240
241
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
242
243
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
244
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
245
246

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
247
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
248
249


250
251
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
252
253
254
255
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
256
257


Michael Hanselmann's avatar
Michael Hanselmann committed
258
259
def TestInstanceList():
  """gnt-instance list"""
260
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
261
262


263
264
265
266
267
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


268
269
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
270
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
271
272


273
274
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
275
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
276
277
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
278
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
279
    cmd = ["gnt-instance", "replace-disks"]
280
281
282
283
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
284
285
286
287
288
289
290
291
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
292

293
294
295
296
297
298
299
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

300

Michael Hanselmann's avatar
Michael Hanselmann committed
301
def TestInstanceExport(instance, node):
302
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
303
304
305
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
306
307


308
309
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
310
311
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
312
313


314
315
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
316
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
317
318


Michael Hanselmann's avatar
Michael Hanselmann committed
319
320
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
321
322
323
324
325
326
327
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--net", "0:mac=generate",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
Michael Hanselmann's avatar
Michael Hanselmann committed
328
         _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
329
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
330
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
331
332
333
334


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
335
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
336
337
338
339
340
341
342


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

343
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
344
345
346
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

347
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
348
349
350
351
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
352
353

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
354
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
355
356
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
357
    (node_name, phys, inst) = line.split("|")
358
359
360
361
362
363
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
364
        raise qa_error.Error("Unknown disk name format: %s" % phys)
365
366
367
368
369
370

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
371
372
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
373

374
375
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
376
377
378
379
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
380
    AssertCommand(" && ".join(cmds), node=node_name)
381

382
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
383
384
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
385
386
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
387
    (_, _, tmpdevpath) = line.split(":")
388
    devpath.append(tmpdevpath)
389
  print devpath
390

391
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
392
393
394
395
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
396
  drbddevs = re.findall(pattern, output, re.M)
397
  print drbddevs
398
399
400

  halted_disks = []
  try:
401
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
402
403
404
405
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
406
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
407

408
409
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
410
411
412
413
414
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
415
      AssertCommand(" && ".join(cmds), node=node)
416
417
      time.sleep(3)

418
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
419
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
420
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
421

Iustin Pop's avatar
Iustin Pop committed
422
    AssertCommand(["gnt-instance", "info", instance["name"]])
423
424

  finally:
425
    print qa_utils.FormatInfo("Activating disks again")
426
427
428
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
429
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
430

Michael Hanselmann's avatar
Michael Hanselmann committed
431
432
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
433
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
434
435
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
436
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
437

438
  # TODO
Iustin Pop's avatar
Iustin Pop committed
439
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
440

441
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
442
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
443
444

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
445
446
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
447

Iustin Pop's avatar
Iustin Pop committed
448
  AssertCommand(["gnt-cluster", "verify"])
449
450
451
452


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
453
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
454
  # due to unused args
455
456
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
457
  # The following can cause crashes, thus it's disabled until fixed
458
  #return _TestInstanceDiskFailure(instance, node, node2, True)
459
460
461
462
463


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)