qa_instance.py 12.3 KB
Newer Older
1
2
3
#
#

4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

Iustin Pop's avatar
Iustin Pop committed
37
from qa_utils import AssertIn, AssertCommand
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
  params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  for idx, size in enumerate(qa_config.get('disk')):
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
49
50


51
def _DiskTest(node, disk_template):
52
53
  instance = qa_config.AcquireInstance()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
54
55
    cmd = (['gnt-instance', 'add',
            '--os-type=%s' % qa_config.get('os'),
56
57
            '--disk-template=%s' % disk_template,
            '--node=%s' % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
58
           _GetGenericAddParameters())
59
60
    cmd.append(instance['name'])

Iustin Pop's avatar
Iustin Pop committed
61
    AssertCommand(cmd)
62
63
64

    _CheckSsconfInstanceList(instance["name"])

65
66
67
68
69
70
71
72
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
73
  return _DiskTest(node['primary'], 'plain')
74
75


76
77
78
79
80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
                   'drbd')


82
83
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
84
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
85
86
87
88
89
90

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
91
  AssertCommand(["gnt-instance", "startup", instance["name"]])
92
93
94
95


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
96
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
97
98


99
100
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
101
102
  options = qa_config.get('options', {})
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
103
  name = instance["name"]
104
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
105
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
106
107


Michael Hanselmann's avatar
Michael Hanselmann committed
108
109
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
110
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
111
112


113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


137
def TestInstanceRename(rename_source, rename_target):
138
  """gnt-instance rename"""
139
140
141
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
142
143


144
145
146
def TestInstanceFailover(instance):
  """gnt-instance failover"""
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
Iustin Pop's avatar
Iustin Pop committed
147
148
  # failover ...
  AssertCommand(cmd)
149
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
150
  AssertCommand(cmd)
151

152

153
154
155
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
156
157
  # migrate ...
  AssertCommand(cmd)
158
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
159
  AssertCommand(cmd)
160
161


162
163
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
164
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
165
166


167
168
def TestInstanceModify(instance):
  """gnt-instance modify"""
169
170
171
172
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

173
  orig_memory = qa_config.get('mem')
Iustin Pop's avatar
Iustin Pop committed
174
  #orig_bridge = qa_config.get("bridge", "xen-br0")
175
  args = [
176
177
178
179
180
181
182
183
184
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
185
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
186
187
188
189
190
191
192
193
194
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
195
196
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
197
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
198
199

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
200
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
201
202


203
204
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
205
206
207
208
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
209
210


Michael Hanselmann's avatar
Michael Hanselmann committed
211
212
def TestInstanceList():
  """gnt-instance list"""
213
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
214
215


216
217
218
219
220
def TestInstanceListFields():
  """gnt-instance list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())


221
222
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
223
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
224
225


226
227
228
229
230
231
232
233
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
  def buildcmd(args):
    cmd = ['gnt-instance', 'replace-disks']
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
234
235
236
237
238
239
240
241
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
242
243


Michael Hanselmann's avatar
Michael Hanselmann committed
244
def TestInstanceExport(instance, node):
245
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
246
247
248
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
249
250


251
252
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
253
254
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
255
256


257
258
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
259
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
260
261


Michael Hanselmann's avatar
Michael Hanselmann committed
262
263
264
265
266
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
  cmd = (['gnt-backup', 'import',
          '--disk-template=plain',
          '--no-ip-check',
267
          '--net', '0:mac=generate',
Michael Hanselmann's avatar
Michael Hanselmann committed
268
269
270
271
272
          '--src-node=%s' % expnode['primary'],
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
          '--node=%s' % node['primary']] +
         _GetGenericAddParameters())
  cmd.append(newinst['name'])
Iustin Pop's avatar
Iustin Pop committed
273
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
274
275
276
277


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
278
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
279
280
281
282
283
284
285


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

286
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
287
288
289
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

290
  print qa_utils.FormatInfo("Getting physical disk names")
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
         '--output=node,phys,instance',
         node['primary'], node2['primary']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))

  # Get physical disk names
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
  node2disk = {}
  for line in output.splitlines():
    (node_name, phys, inst) = line.split('|')
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
307
        raise qa_error.Error("Unknown disk name format: %s" % phys)
308
309
310
311
312
313

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
314
315
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
316

317
318
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
319
320
321
322
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
323
    AssertCommand(" && ".join(cmds), node=node_name)
324

325
  print qa_utils.FormatInfo("Getting device paths")
326
327
328
329
330
331
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
  devpath = []
  for line in output.splitlines():
    (_, _, tmpdevpath) = line.split(':')
    devpath.append(tmpdevpath)
332
  print devpath
333

334
  print qa_utils.FormatInfo("Getting drbd device paths")
335
336
  cmd = ['gnt-instance', 'info', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
337
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
338
339
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
  drbddevs = re.findall(pattern, output, re.M)
340
  print drbddevs
341
342
343

  halted_disks = []
  try:
344
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
345
346
347
348
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
349
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
350

351
352
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
353
354
355
356
357
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
358
      AssertCommand(" && ".join(cmds), node=node)
359
360
      time.sleep(3)

361
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
362
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
363
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
364

Iustin Pop's avatar
Iustin Pop committed
365
    AssertCommand(["gnt-instance", "info", instance["name"]])
366
367

  finally:
368
    print qa_utils.FormatInfo("Activating disks again")
369
370
371
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
372
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
373

Michael Hanselmann's avatar
Michael Hanselmann committed
374
375
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
376
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
377
378
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
379
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
380

381
  # TODO
Iustin Pop's avatar
Iustin Pop committed
382
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
383

384
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
385
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
386
387

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
388
389
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
390

Iustin Pop's avatar
Iustin Pop committed
391
  AssertCommand(["gnt-cluster", "verify"])
392
393
394
395


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
396
397
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
398
  # The following can cause crashes, thus it's disabled until fixed
399
  #return _TestInstanceDiskFailure(instance, node, node2, True)
400
401
402
403
404


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)