qa_instance.py 12.9 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
  params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  for idx, size in enumerate(qa_config.get('disk')):
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
49
50


51
def _DiskTest(node, disk_template):
52
53
  instance = qa_config.AcquireInstance()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
54
55
    cmd = (['gnt-instance', 'add',
            '--os-type=%s' % qa_config.get('os'),
56
57
            '--disk-template=%s' % disk_template,
            '--node=%s' % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
58
           _GetGenericAddParameters())
59
60
    cmd.append(instance['name'])

Iustin Pop's avatar
Iustin Pop committed
61
    AssertCommand(cmd)
62
63
64

    _CheckSsconfInstanceList(instance["name"])

65
66
67
68
69
70
71
72
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
73
  return _DiskTest(node['primary'], 'plain')
74
75


76
77
78
79
80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
                   'drbd')


82
83
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
84
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
85
86
87
88
89
90

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
91
  AssertCommand(["gnt-instance", "startup", instance["name"]])
92
93
94
95


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
96
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
97
98


99
100
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
101
102
  options = qa_config.get('options', {})
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
103
  name = instance["name"]
104
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
105
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
106

107
108
109
110
111
112
113
114
115
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
  cmd = ["gnt-instance", "list", "--no-header", "-o", "status", name]
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

116

Michael Hanselmann's avatar
Michael Hanselmann committed
117
118
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
119
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
120
121


122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


146
def TestInstanceRename(rename_source, rename_target):
147
  """gnt-instance rename"""
148
149
150
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
151
152


153
154
155
def TestInstanceFailover(instance):
  """gnt-instance failover"""
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
Iustin Pop's avatar
Iustin Pop committed
156
157
  # failover ...
  AssertCommand(cmd)
158
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
159
  AssertCommand(cmd)
160

161

162
163
164
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
165
166
  # migrate ...
  AssertCommand(cmd)
167
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
168
  AssertCommand(cmd)
169
170


171
172
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
173
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
174
175


176
177
def TestInstanceModify(instance):
  """gnt-instance modify"""
178
179
180
181
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

182
  orig_memory = qa_config.get('mem')
Iustin Pop's avatar
Iustin Pop committed
183
  #orig_bridge = qa_config.get("bridge", "xen-br0")
184
  args = [
185
186
187
188
189
190
191
192
193
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
194
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
195
196
197
198
199
200
201
202
203
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
204
205
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
206
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
207
208

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
209
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
210
211


212
213
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
214
215
216
217
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
218
219


Michael Hanselmann's avatar
Michael Hanselmann committed
220
221
def TestInstanceList():
  """gnt-instance list"""
222
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
223
224


225
226
227
228
229
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


230
231
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
232
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
233
234


235
236
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
Iustin Pop's avatar
Iustin Pop committed
237
238
239
  # pylint: disable-msg=W0613
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
240
241
242
243
244
245
  def buildcmd(args):
    cmd = ['gnt-instance', 'replace-disks']
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
246
247
248
249
250
251
252
253
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
254
255


Michael Hanselmann's avatar
Michael Hanselmann committed
256
def TestInstanceExport(instance, node):
257
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
258
259
260
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
261
262


263
264
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
265
266
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
267
268


269
270
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
271
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
272
273


Michael Hanselmann's avatar
Michael Hanselmann committed
274
275
276
277
278
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
  cmd = (['gnt-backup', 'import',
          '--disk-template=plain',
          '--no-ip-check',
279
          '--net', '0:mac=generate',
Michael Hanselmann's avatar
Michael Hanselmann committed
280
281
282
283
284
          '--src-node=%s' % expnode['primary'],
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
          '--node=%s' % node['primary']] +
         _GetGenericAddParameters())
  cmd.append(newinst['name'])
Iustin Pop's avatar
Iustin Pop committed
285
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
286
287
288
289


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
290
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
291
292
293
294
295
296
297


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

298
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
299
300
301
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

302
  print qa_utils.FormatInfo("Getting physical disk names")
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
         '--output=node,phys,instance',
         node['primary'], node2['primary']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))

  # Get physical disk names
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
  node2disk = {}
  for line in output.splitlines():
    (node_name, phys, inst) = line.split('|')
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
319
        raise qa_error.Error("Unknown disk name format: %s" % phys)
320
321
322
323
324
325

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
326
327
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
328

329
330
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
331
332
333
334
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
335
    AssertCommand(" && ".join(cmds), node=node_name)
336

337
  print qa_utils.FormatInfo("Getting device paths")
338
339
340
341
342
343
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
  devpath = []
  for line in output.splitlines():
    (_, _, tmpdevpath) = line.split(':')
    devpath.append(tmpdevpath)
344
  print devpath
345

346
  print qa_utils.FormatInfo("Getting drbd device paths")
347
348
  cmd = ['gnt-instance', 'info', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
349
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
350
351
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
  drbddevs = re.findall(pattern, output, re.M)
352
  print drbddevs
353
354
355

  halted_disks = []
  try:
356
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
357
358
359
360
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
361
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
362

363
364
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
365
366
367
368
369
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
370
      AssertCommand(" && ".join(cmds), node=node)
371
372
      time.sleep(3)

373
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
374
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
375
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
376

Iustin Pop's avatar
Iustin Pop committed
377
    AssertCommand(["gnt-instance", "info", instance["name"]])
378
379

  finally:
380
    print qa_utils.FormatInfo("Activating disks again")
381
382
383
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
384
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
385

Michael Hanselmann's avatar
Michael Hanselmann committed
386
387
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
388
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
389
390
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
391
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
392

393
  # TODO
Iustin Pop's avatar
Iustin Pop committed
394
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
395

396
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
397
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
398
399

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
400
401
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
402

Iustin Pop's avatar
Iustin Pop committed
403
  AssertCommand(["gnt-cluster", "verify"])
404
405
406
407


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
Iustin Pop's avatar
Iustin Pop committed
408
409
  # pylint: disable-msg=W0613
  # due to unused args
410
411
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
412
  # The following can cause crashes, thus it's disabled until fixed
413
  #return _TestInstanceDiskFailure(instance, node, node2, True)
414
415
416
417
418


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)