qa_instance.py 12.1 KB
Newer Older
1
2
3
#
#

4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
32

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
33
import qa_utils
34
35
import qa_error

Iustin Pop's avatar
Iustin Pop committed
36
from qa_utils import AssertIn, AssertCommand
37

38
39
40

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
41
42


Michael Hanselmann's avatar
Michael Hanselmann committed
43
def _GetGenericAddParameters():
44
45
46
47
  params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  for idx, size in enumerate(qa_config.get('disk')):
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
48
49


50
def _DiskTest(node, disk_template):
51
52
  instance = qa_config.AcquireInstance()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
53
54
    cmd = (['gnt-instance', 'add',
            '--os-type=%s' % qa_config.get('os'),
55
56
            '--disk-template=%s' % disk_template,
            '--node=%s' % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
57
           _GetGenericAddParameters())
58
59
    cmd.append(instance['name'])

Iustin Pop's avatar
Iustin Pop committed
60
    AssertCommand(cmd)
61
62
63

    _CheckSsconfInstanceList(instance["name"])

64
65
66
67
68
69
70
71
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
72
  return _DiskTest(node['primary'], 'plain')
73
74


75
76
77
78
79
80
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
                   'drbd')


81
82
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
83
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
84
85
86
87
88
89

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
90
  AssertCommand(["gnt-instance", "startup", instance["name"]])
91
92
93
94


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
95
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
96
97


98
99
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
100
101
  options = qa_config.get('options', {})
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
102
  name = instance["name"]
103
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
104
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
105
106


Michael Hanselmann's avatar
Michael Hanselmann committed
107
108
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
109
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
110
111


112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


136
def TestInstanceRename(rename_source, rename_target):
137
  """gnt-instance rename"""
138
139
140
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
141
142


143
144
145
def TestInstanceFailover(instance):
  """gnt-instance failover"""
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
Iustin Pop's avatar
Iustin Pop committed
146
147
  # failover ...
  AssertCommand(cmd)
148
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
149
  AssertCommand(cmd)
150

151

152
153
154
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
155
156
  # migrate ...
  AssertCommand(cmd)
157
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
158
  AssertCommand(cmd)
159
160


161
162
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
163
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
164
165


166
167
def TestInstanceModify(instance):
  """gnt-instance modify"""
168
169
170
171
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

172
  orig_memory = qa_config.get('mem')
Iustin Pop's avatar
Iustin Pop committed
173
  #orig_bridge = qa_config.get("bridge", "xen-br0")
174
  args = [
175
176
177
178
179
180
181
182
183
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
184
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
185
186
187
188
189
190
191
192
193
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
194
195
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
196
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
197
198

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
199
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
200
201


202
203
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
204
205
206
207
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
208
209


Michael Hanselmann's avatar
Michael Hanselmann committed
210
211
def TestInstanceList():
  """gnt-instance list"""
Iustin Pop's avatar
Iustin Pop committed
212
  AssertCommand(["gnt-instance", "list"])
Michael Hanselmann's avatar
Michael Hanselmann committed
213
214


215
216
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
217
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
218
219


220
221
222
223
224
225
226
227
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
  def buildcmd(args):
    cmd = ['gnt-instance', 'replace-disks']
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
228
229
230
231
232
233
234
235
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
236
237


Michael Hanselmann's avatar
Michael Hanselmann committed
238
def TestInstanceExport(instance, node):
239
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
240
241
242
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
243
244


245
246
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
247
248
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
249
250


251
252
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
253
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
254
255


Michael Hanselmann's avatar
Michael Hanselmann committed
256
257
258
259
260
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
  cmd = (['gnt-backup', 'import',
          '--disk-template=plain',
          '--no-ip-check',
261
          '--net', '0:mac=generate',
Michael Hanselmann's avatar
Michael Hanselmann committed
262
263
264
265
266
          '--src-node=%s' % expnode['primary'],
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
          '--node=%s' % node['primary']] +
         _GetGenericAddParameters())
  cmd.append(newinst['name'])
Iustin Pop's avatar
Iustin Pop committed
267
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
268
269
270
271


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
272
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
273
274
275
276
277
278
279


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

280
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
281
282
283
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

284
  print qa_utils.FormatInfo("Getting physical disk names")
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
         '--output=node,phys,instance',
         node['primary'], node2['primary']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))

  # Get physical disk names
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
  node2disk = {}
  for line in output.splitlines():
    (node_name, phys, inst) = line.split('|')
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
301
        raise qa_error.Error("Unknown disk name format: %s" % phys)
302
303
304
305
306
307

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
308
309
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
310

311
312
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
313
314
315
316
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
317
    AssertCommand(" && ".join(cmds), node=node_name)
318

319
  print qa_utils.FormatInfo("Getting device paths")
320
321
322
323
324
325
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
  devpath = []
  for line in output.splitlines():
    (_, _, tmpdevpath) = line.split(':')
    devpath.append(tmpdevpath)
326
  print devpath
327

328
  print qa_utils.FormatInfo("Getting drbd device paths")
329
330
  cmd = ['gnt-instance', 'info', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
331
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
332
333
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
  drbddevs = re.findall(pattern, output, re.M)
334
  print drbddevs
335
336
337

  halted_disks = []
  try:
338
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
339
340
341
342
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
343
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
344

345
346
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
347
348
349
350
351
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
352
      AssertCommand(" && ".join(cmds), node=node)
353
354
      time.sleep(3)

355
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
356
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
357
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
358

Iustin Pop's avatar
Iustin Pop committed
359
    AssertCommand(["gnt-instance", "info", instance["name"]])
360
361

  finally:
362
    print qa_utils.FormatInfo("Activating disks again")
363
364
365
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
366
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
367

Michael Hanselmann's avatar
Michael Hanselmann committed
368
369
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
370
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
371
372
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
373
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
374

375
  # TODO
Iustin Pop's avatar
Iustin Pop committed
376
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
377

378
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
379
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
380
381

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
382
383
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
384

Iustin Pop's avatar
Iustin Pop committed
385
  AssertCommand(["gnt-cluster", "verify"])
386
387
388
389


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
390
391
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
392
  # The following can cause crashes, thus it's disabled until fixed
393
  #return _TestInstanceDiskFailure(instance, node, node2, True)
394
395
396
397
398


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)