qa_instance.py 14 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
Iustin Pop's avatar
Iustin Pop committed
45
46
  params = ["-B", "%s=%s" % (constants.BE_MEMORY, qa_config.get("mem"))]
  for idx, size in enumerate(qa_config.get("disk")):
47
48
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
49
50


51
def _DiskTest(node, disk_template):
52
53
  instance = qa_config.AcquireInstance()
  try:
Iustin Pop's avatar
Iustin Pop committed
54
55
56
57
    cmd = (["gnt-instance", "add",
            "--os-type=%s" % qa_config.get("os"),
            "--disk-template=%s" % disk_template,
            "--node=%s" % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
58
           _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
59
    cmd.append(instance["name"])
60

Iustin Pop's avatar
Iustin Pop committed
61
    AssertCommand(cmd)
62
63
64

    _CheckSsconfInstanceList(instance["name"])

65
66
67
68
69
70
71
72
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
Iustin Pop's avatar
Iustin Pop committed
73
  return _DiskTest(node["primary"], "plain")
74
75


76
77
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
Iustin Pop's avatar
Iustin Pop committed
78
79
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                   "drbd")
80
81


82
83
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
84
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
85
86
87
88
89
90

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
91
  AssertCommand(["gnt-instance", "startup", instance["name"]])
92
93
94
95


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
96
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
97
98


99
100
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
Iustin Pop's avatar
Iustin Pop committed
101
  options = qa_config.get("options", {})
102
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
103
  name = instance["name"]
104
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
105
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
106

107
108
109
110
111
112
113
114
115
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
  cmd = ["gnt-instance", "list", "--no-header", "-o", "status", name]
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

116

Michael Hanselmann's avatar
Michael Hanselmann committed
117
118
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
119
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
120
121


122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


146
def TestInstanceRename(rename_source, rename_target):
147
  """gnt-instance rename"""
148
149
150
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
151
152
153
154
155
156
157
158
159
160
161
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
162
163


164
165
def TestInstanceFailover(instance):
  """gnt-instance failover"""
Iustin Pop's avatar
Iustin Pop committed
166
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
167
168
  # failover ...
  AssertCommand(cmd)
169
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
170
  AssertCommand(cmd)
171

172

173
174
175
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
176
177
  # migrate ...
  AssertCommand(cmd)
178
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
179
  AssertCommand(cmd)
180
181
182
183
184
185
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(cmd, fail=True)
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                 instance["name"]])
  AssertCommand(["gnt-instance", "start", instance["name"]])
  AssertCommand(cmd)
186
187


188
189
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
190
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
191
192


193
194
def TestInstanceModify(instance):
  """gnt-instance modify"""
195
196
197
198
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

Iustin Pop's avatar
Iustin Pop committed
199
  orig_memory = qa_config.get("mem")
Iustin Pop's avatar
Iustin Pop committed
200
  #orig_bridge = qa_config.get("bridge", "xen-br0")
201
  args = [
202
203
204
205
206
207
208
209
210
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
211
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
212
213
214
215
216
217
218
219
220
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
221
222
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
223
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
224
225

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
226
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
227
228


229
230
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
231
232
233
234
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
235
236


Michael Hanselmann's avatar
Michael Hanselmann committed
237
238
def TestInstanceList():
  """gnt-instance list"""
239
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
240
241


242
243
244
245
246
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


247
248
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
249
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
250
251


252
253
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
Iustin Pop's avatar
Iustin Pop committed
254
255
256
  # pylint: disable-msg=W0613
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
257
  def buildcmd(args):
Iustin Pop's avatar
Iustin Pop committed
258
    cmd = ["gnt-instance", "replace-disks"]
259
260
261
262
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
263
264
265
266
267
268
269
270
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
271

272
273
274
275
276
277
278
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "stop", instance["name"]])
  AssertCommand(buildcmd(["-a"]), fail=True)
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
  AssertCommand(buildcmd(["-a"]))
  AssertCommand(["gnt-instance", "start", instance["name"]])

279

Michael Hanselmann's avatar
Michael Hanselmann committed
280
def TestInstanceExport(instance, node):
281
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
282
283
284
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
285
286


287
288
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
289
290
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
291
292


293
294
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
295
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
296
297


Michael Hanselmann's avatar
Michael Hanselmann committed
298
299
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
Iustin Pop's avatar
Iustin Pop committed
300
301
302
303
304
305
306
  cmd = (["gnt-backup", "import",
          "--disk-template=plain",
          "--no-ip-check",
          "--net", "0:mac=generate",
          "--src-node=%s" % expnode["primary"],
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
          "--node=%s" % node["primary"]] +
Michael Hanselmann's avatar
Michael Hanselmann committed
307
         _GetGenericAddParameters())
Iustin Pop's avatar
Iustin Pop committed
308
  cmd.append(newinst["name"])
Iustin Pop's avatar
Iustin Pop committed
309
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
310
311
312
313


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
314
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
315
316
317
318
319
320
321


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

322
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
323
324
325
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

326
  print qa_utils.FormatInfo("Getting physical disk names")
Iustin Pop's avatar
Iustin Pop committed
327
328
329
330
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
         "--output=node,phys,instance",
         node["primary"], node2["primary"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
331
332

  # Get physical disk names
Iustin Pop's avatar
Iustin Pop committed
333
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
334
335
  node2disk = {}
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
336
    (node_name, phys, inst) = line.split("|")
337
338
339
340
341
342
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
343
        raise qa_error.Error("Unknown disk name format: %s" % phys)
344
345
346
347
348
349

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
350
351
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
352

353
354
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
355
356
357
358
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
359
    AssertCommand(" && ".join(cmds), node=node_name)
360

361
  print qa_utils.FormatInfo("Getting device paths")
Iustin Pop's avatar
Iustin Pop committed
362
363
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
364
365
  devpath = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
366
    (_, _, tmpdevpath) = line.split(":")
367
    devpath.append(tmpdevpath)
368
  print devpath
369

370
  print qa_utils.FormatInfo("Getting drbd device paths")
Iustin Pop's avatar
Iustin Pop committed
371
372
373
374
  cmd = ["gnt-instance", "info", instance["name"]]
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
375
  drbddevs = re.findall(pattern, output, re.M)
376
  print drbddevs
377
378
379

  halted_disks = []
  try:
380
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
381
382
383
384
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
385
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
386

387
388
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
389
390
391
392
393
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
394
      AssertCommand(" && ".join(cmds), node=node)
395
396
      time.sleep(3)

397
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
398
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
399
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
400

Iustin Pop's avatar
Iustin Pop committed
401
    AssertCommand(["gnt-instance", "info", instance["name"]])
402
403

  finally:
404
    print qa_utils.FormatInfo("Activating disks again")
405
406
407
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
408
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
409

Michael Hanselmann's avatar
Michael Hanselmann committed
410
411
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
412
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
413
414
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
415
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
416

417
  # TODO
Iustin Pop's avatar
Iustin Pop committed
418
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
419

420
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
421
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
422
423

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
424
425
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
426

Iustin Pop's avatar
Iustin Pop committed
427
  AssertCommand(["gnt-cluster", "verify"])
428
429
430
431


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
Iustin Pop's avatar
Iustin Pop committed
432
433
  # pylint: disable-msg=W0613
  # due to unused args
434
435
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
436
  # The following can cause crashes, thus it's disabled until fixed
437
  #return _TestInstanceDiskFailure(instance, node, node2, True)
438
439
440
441
442


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)