qa_instance.py 14.4 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
49
  params = ["-B"]
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                 qa_config.get(constants.BE_MINMEM),
                                 constants.BE_MAXMEM,
                                 qa_config.get(constants.BE_MAXMEM)))
Iustin Pop's avatar
Iustin Pop committed
50
  for idx, size in enumerate(qa_config.get("disk")):
51
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
53
54


55
def _DiskTest(node, disk_template):
56
57
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
58
59
60
61
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
62
           _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
63
    cmd.append(instance["name"])
64

Iustin Pop's avatar
Iustin Pop committed
65
    AssertCommand(cmd)
66
67
68

    _CheckSsconfInstanceList(instance["name"])

69
70
71
72
73
74
75
76
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
77
  return _DiskTest(node["primary"], "plain")
78
79


80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
82
83
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
84
85


86
87
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
88
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
89
90
91
92
93
94

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
95
  AssertCommand(["gnt-instance", "startup", instance["name"]])
96
97
98
99


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
100
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
101
102


103
104
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
105
  options = qa_config.get("options", {})
106
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
107
  name = instance["name"]
108
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
109
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
110

111
112
113
114
115
116
117
118
119
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
  cmd = ["gnt-instance", "list", "--no-header", "-o", "status", name]
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

120

Michael Hanselmann's avatar
Michael Hanselmann committed
121
122
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
123
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
124
125


126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


150
def TestInstanceRename(rename_source, rename_target):
151
  """gnt-instance rename"""
152
153
154
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
155
156
157
158
159
160
161
162
163
164
165
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
166
167


168
169
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
170
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
171
172
  # failover ...
  AssertCommand(cmd)
173
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
174
  AssertCommand(cmd)
175

176

177
178
179
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
180
181
  # migrate ...
  AssertCommand(cmd)
182
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
183
  AssertCommand(cmd)
184
185
186
187
188
189
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
190
191


192
193
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
194
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
195
196


197
198
def TestInstanceModify(instance):
  """gnt-instance modify"""
199
200
201
202
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

203
204
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
  orig_minmem = qa_config.get(constants.BE_MINMEM)
Iustin Pop's avatar
Iustin Pop committed
205
  #orig_bridge = qa_config.get("bridge", "xen-br0")
206
  args = [
207
208
209
210
    ["-B", "%s=128" % constants.BE_MINMEM],
    ["-B", "%s=128" % constants.BE_MAXMEM],
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
                            constants.BE_MAXMEM, orig_maxmem)],
211
212
213
214
215
216
217
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
218
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
219
220
221
222
223
224
225
226
227
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
228
229
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
230
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
231
232

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
233
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
234
235


236
237
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
238
239
240
241
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
242
243


Michael Hanselmann's avatar
Michael Hanselmann committed
244
245
def TestInstanceList():
  """gnt-instance list"""
246
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
247
248


249
250
251
252
253
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


254
255
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
256
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
257
258


259
260
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
261
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
262
263
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
264
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
265
    cmd = ["gnt-instance", "replace-disks"]
266
267
268
269
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
270
271
272
273
274
275
276
277
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
278

279
280
281
282
283
284
285
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

286

Michael Hanselmann's avatar
Michael Hanselmann committed
287
def TestInstanceExport(instance, node):
288
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
289
290
291
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
292
293


294
295
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
296
297
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
298
299


300
301
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
302
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
303
304


Michael Hanselmann's avatar
Michael Hanselmann committed
305
306
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
307
308
309
310
311
312
313
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--net", "0:mac=generate",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
Michael Hanselmann's avatar
Michael Hanselmann committed
314
         _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
315
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
316
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
317
318
319
320


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
321
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
322
323
324
325
326
327
328


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

329
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
330
331
332
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

333
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
334
335
336
337
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
338
339

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
340
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
341
342
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
343
    (node_name, phys, inst) = line.split("|")
344
345
346
347
348
349
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
350
        raise qa_error.Error("Unknown disk name format: %s" % phys)
351
352
353
354
355
356

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
357
358
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
359

360
361
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
362
363
364
365
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
366
    AssertCommand(" && ".join(cmds), node=node_name)
367

368
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
369
370
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
371
372
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
373
    (_, _, tmpdevpath) = line.split(":")
374
    devpath.append(tmpdevpath)
375
  print devpath
376

377
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
378
379
380
381
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
382
  drbddevs = re.findall(pattern, output, re.M)
383
  print drbddevs
384
385
386

  halted_disks = []
  try:
387
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
388
389
390
391
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
392
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
393

394
395
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
396
397
398
399
400
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
401
      AssertCommand(" && ".join(cmds), node=node)
402
403
      time.sleep(3)

404
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
405
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
406
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
407

Iustin Pop's avatar
Iustin Pop committed
408
    AssertCommand(["gnt-instance", "info", instance["name"]])
409
410

  finally:
411
    print qa_utils.FormatInfo("Activating disks again")
412
413
414
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
415
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
416

Michael Hanselmann's avatar
Michael Hanselmann committed
417
418
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
419
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
420
421
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
422
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
423

424
  # TODO
Iustin Pop's avatar
Iustin Pop committed
425
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
426

427
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
428
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
429
430

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
431
432
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
433

Iustin Pop's avatar
Iustin Pop committed
434
  AssertCommand(["gnt-cluster", "verify"])
435
436
437
438


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
439
  # pylint: disable=W0613
Iustin Pop's avatar
Iustin Pop committed
440
  # due to unused args
441
442
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
443
  # The following can cause crashes, thus it's disabled until fixed
444
  #return _TestInstanceDiskFailure(instance, node, node2, True)
445
446
447
448
449


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)