qa_instance.py 13.4 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
  params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  for idx, size in enumerate(qa_config.get('disk')):
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
49
50


51
def _DiskTest(node, disk_template):
52
53
  instance = qa_config.AcquireInstance()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
54
55
    cmd = (['gnt-instance', 'add',
            '--os-type=%s' % qa_config.get('os'),
56
57
            '--disk-template=%s' % disk_template,
            '--node=%s' % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
58
           _GetGenericAddParameters())
59
60
    cmd.append(instance['name'])

Iustin Pop's avatar
Iustin Pop committed
61
    AssertCommand(cmd)
62
63
64

    _CheckSsconfInstanceList(instance["name"])

65
66
67
68
69
70
71
72
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
73
  return _DiskTest(node['primary'], 'plain')
74
75


76
77
78
79
80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
                   'drbd')


82
83
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
84
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
85
86
87
88
89
90

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
91
  AssertCommand(["gnt-instance", "startup", instance["name"]])
92
93
94
95


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
96
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
97
98


99
100
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
101
102
  options = qa_config.get('options', {})
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
103
  name = instance["name"]
104
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
105
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
106

107
108
109
110
111
112
113
114
115
  AssertCommand(["gnt-instance", "shutdown", name])
  AssertCommand(["gnt-instance", "reboot", name])

  master = qa_config.GetMasterNode()
  cmd = ["gnt-instance", "list", "--no-header", "-o", "status", name]
  result_output = qa_utils.GetCommandOutput(master["primary"],
                                            utils.ShellQuoteArgs(cmd))
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)

116

Michael Hanselmann's avatar
Michael Hanselmann committed
117
118
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
119
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
120
121


122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


146
def TestInstanceRename(rename_source, rename_target):
147
  """gnt-instance rename"""
148
149
150
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
151
152
153
154
155
156
157
158
159
160
161
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
  _CheckSsconfInstanceList(rename_source)
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
  try:
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
                  fail=True)
    _CheckSsconfInstanceList(rename_source)
  finally:
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
162
163


164
165
166
def TestInstanceFailover(instance):
  """gnt-instance failover"""
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
Iustin Pop's avatar
Iustin Pop committed
167
168
  # failover ...
  AssertCommand(cmd)
169
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
170
  AssertCommand(cmd)
171

172

173
174
175
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
176
177
  # migrate ...
  AssertCommand(cmd)
178
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
179
  AssertCommand(cmd)
180
181


182
183
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
184
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
185
186


187
188
def TestInstanceModify(instance):
  """gnt-instance modify"""
189
190
191
192
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

193
  orig_memory = qa_config.get('mem')
Iustin Pop's avatar
Iustin Pop committed
194
  #orig_bridge = qa_config.get("bridge", "xen-br0")
195
  args = [
196
197
198
199
200
201
202
203
204
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
205
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
206
207
208
209
210
211
212
213
214
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
215
216
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
217
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
218
219

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
220
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
221
222


223
224
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
225
226
227
228
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
229
230


Michael Hanselmann's avatar
Michael Hanselmann committed
231
232
def TestInstanceList():
  """gnt-instance list"""
233
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
234
235


236
237
238
239
240
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


241
242
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
243
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
244
245


246
247
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
Iustin Pop's avatar
Iustin Pop committed
248
249
250
  # pylint: disable-msg=W0613
  # due to unused pnode arg
  # FIXME: should be removed from the function completely
251
252
253
254
255
256
  def buildcmd(args):
    cmd = ['gnt-instance', 'replace-disks']
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
257
258
259
260
261
262
263
264
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
265
266


Michael Hanselmann's avatar
Michael Hanselmann committed
267
def TestInstanceExport(instance, node):
268
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
269
270
271
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
272
273


274
275
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
276
277
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
278
279


280
281
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
282
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
283
284


Michael Hanselmann's avatar
Michael Hanselmann committed
285
286
287
288
289
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
  cmd = (['gnt-backup', 'import',
          '--disk-template=plain',
          '--no-ip-check',
290
          '--net', '0:mac=generate',
Michael Hanselmann's avatar
Michael Hanselmann committed
291
292
293
294
295
          '--src-node=%s' % expnode['primary'],
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
          '--node=%s' % node['primary']] +
         _GetGenericAddParameters())
  cmd.append(newinst['name'])
Iustin Pop's avatar
Iustin Pop committed
296
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
297
298
299
300


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
301
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
302
303
304
305
306
307
308


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

309
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
310
311
312
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

313
  print qa_utils.FormatInfo("Getting physical disk names")
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
         '--output=node,phys,instance',
         node['primary'], node2['primary']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))

  # Get physical disk names
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
  node2disk = {}
  for line in output.splitlines():
    (node_name, phys, inst) = line.split('|')
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
330
        raise qa_error.Error("Unknown disk name format: %s" % phys)
331
332
333
334
335
336

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
337
338
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
339

340
341
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
342
343
344
345
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
346
    AssertCommand(" && ".join(cmds), node=node_name)
347

348
  print qa_utils.FormatInfo("Getting device paths")
349
350
351
352
353
354
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
  devpath = []
  for line in output.splitlines():
    (_, _, tmpdevpath) = line.split(':')
    devpath.append(tmpdevpath)
355
  print devpath
356

357
  print qa_utils.FormatInfo("Getting drbd device paths")
358
359
  cmd = ['gnt-instance', 'info', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
360
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
361
362
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
  drbddevs = re.findall(pattern, output, re.M)
363
  print drbddevs
364
365
366

  halted_disks = []
  try:
367
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
368
369
370
371
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
372
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
373

374
375
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
376
377
378
379
380
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
381
      AssertCommand(" && ".join(cmds), node=node)
382
383
      time.sleep(3)

384
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
385
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
386
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
387

Iustin Pop's avatar
Iustin Pop committed
388
    AssertCommand(["gnt-instance", "info", instance["name"]])
389
390

  finally:
391
    print qa_utils.FormatInfo("Activating disks again")
392
393
394
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
395
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
396

Michael Hanselmann's avatar
Michael Hanselmann committed
397
398
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
399
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
400
401
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
402
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
403

404
  # TODO
Iustin Pop's avatar
Iustin Pop committed
405
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
406

407
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
408
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
409
410

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
411
412
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
413

Iustin Pop's avatar
Iustin Pop committed
414
  AssertCommand(["gnt-cluster", "verify"])
415
416
417
418


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
Iustin Pop's avatar
Iustin Pop committed
419
420
  # pylint: disable-msg=W0613
  # due to unused args
421
422
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
423
  # The following can cause crashes, thus it's disabled until fixed
424
  #return _TestInstanceDiskFailure(instance, node, node2, True)
425
426
427
428
429


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)