qa_instance.py 12.2 KB
Newer Older
1
2
3
#
#

4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26
27
28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31
from ganeti import query
32
33

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
34
import qa_utils
35
36
import qa_error

Iustin Pop's avatar
Iustin Pop committed
37
from qa_utils import AssertIn, AssertCommand
38

39
40
41

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
42
43


Michael Hanselmann's avatar
Michael Hanselmann committed
44
def _GetGenericAddParameters():
45
46
47
48
  params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  for idx, size in enumerate(qa_config.get('disk')):
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
49
50


51
def _DiskTest(node, disk_template):
52
53
  instance = qa_config.AcquireInstance()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
54
55
    cmd = (['gnt-instance', 'add',
            '--os-type=%s' % qa_config.get('os'),
56
57
            '--disk-template=%s' % disk_template,
            '--node=%s' % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
58
           _GetGenericAddParameters())
59
60
    cmd.append(instance['name'])

Iustin Pop's avatar
Iustin Pop committed
61
    AssertCommand(cmd)
62
63
64

    _CheckSsconfInstanceList(instance["name"])

65
66
67
68
69
70
71
72
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
73
  return _DiskTest(node['primary'], 'plain')
74
75


76
77
78
79
80
81
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
                   'drbd')


82
83
def TestInstanceRemove(instance):
  """gnt-instance remove"""
Iustin Pop's avatar
Iustin Pop committed
84
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
85
86
87
88
89
90

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
Iustin Pop's avatar
Iustin Pop committed
91
  AssertCommand(["gnt-instance", "startup", instance["name"]])
92
93
94
95


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
Iustin Pop's avatar
Iustin Pop committed
96
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
97
98


99
100
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
101
102
  options = qa_config.get('options', {})
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
Iustin Pop's avatar
Iustin Pop committed
103
  name = instance["name"]
104
  for rtype in reboot_types:
Iustin Pop's avatar
Iustin Pop committed
105
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
106
107


Michael Hanselmann's avatar
Michael Hanselmann committed
108
109
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
Iustin Pop's avatar
Iustin Pop committed
110
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
111
112


113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def _ReadSsconfInstanceList():
  """Reads ssconf_instance_list from the master node.

  """
  master = qa_config.GetMasterNode()

  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]

  return qa_utils.GetCommandOutput(master["primary"],
                                   utils.ShellQuoteArgs(cmd)).splitlines()


def _CheckSsconfInstanceList(instance):
  """Checks if a certain instance is in the ssconf instance list.

  @type instance: string
  @param instance: Instance name

  """
  AssertIn(qa_utils.ResolveInstanceName(instance),
           _ReadSsconfInstanceList())


137
def TestInstanceRename(rename_source, rename_target):
138
  """gnt-instance rename"""
139
140
141
  _CheckSsconfInstanceList(rename_source)
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
  _CheckSsconfInstanceList(rename_target)
142
143


144
145
146
def TestInstanceFailover(instance):
  """gnt-instance failover"""
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
Iustin Pop's avatar
Iustin Pop committed
147
148
  # failover ...
  AssertCommand(cmd)
149
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
150
  AssertCommand(cmd)
151

152

153
154
155
def TestInstanceMigrate(instance):
  """gnt-instance migrate"""
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
Iustin Pop's avatar
Iustin Pop committed
156
157
  # migrate ...
  AssertCommand(cmd)
158
  # ... and back
Iustin Pop's avatar
Iustin Pop committed
159
  AssertCommand(cmd)
160
161


162
163
def TestInstanceInfo(instance):
  """gnt-instance info"""
Iustin Pop's avatar
Iustin Pop committed
164
  AssertCommand(["gnt-instance", "info", instance["name"]])
Michael Hanselmann's avatar
Michael Hanselmann committed
165
166


167
168
def TestInstanceModify(instance):
  """gnt-instance modify"""
169
170
171
172
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

173
  orig_memory = qa_config.get('mem')
Iustin Pop's avatar
Iustin Pop committed
174
  #orig_bridge = qa_config.get("bridge", "xen-br0")
175
  args = [
176
177
178
179
180
181
182
183
184
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
185
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
186
187
188
189
190
191
192
193
194
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
195
196
    ]
  for alist in args:
Iustin Pop's avatar
Iustin Pop committed
197
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
198
199

  # check no-modify
Iustin Pop's avatar
Iustin Pop committed
200
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
201
202


203
204
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
Iustin Pop's avatar
Iustin Pop committed
205
206
207
208
  name = instance["name"]
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
                 "-n", snode["primary"], name])
209
210


Michael Hanselmann's avatar
Michael Hanselmann committed
211
212
def TestInstanceList():
  """gnt-instance list"""
213
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
Michael Hanselmann's avatar
Michael Hanselmann committed
214
215


216
217
def TestInstanceConsole(instance):
  """gnt-instance console"""
Iustin Pop's avatar
Iustin Pop committed
218
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
219
220


221
222
223
224
225
226
227
228
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
  def buildcmd(args):
    cmd = ['gnt-instance', 'replace-disks']
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

Iustin Pop's avatar
Iustin Pop committed
229
230
231
232
233
234
235
236
  for data in [
    ["-p"],
    ["-s"],
    ["--new-secondary=%s" % othernode["primary"]],
    # and restore
    ["--new-secondary=%s" % snode["primary"]],
    ]:
    AssertCommand(buildcmd(data))
237
238


Michael Hanselmann's avatar
Michael Hanselmann committed
239
def TestInstanceExport(instance, node):
240
  """gnt-backup export -n ..."""
Iustin Pop's avatar
Iustin Pop committed
241
242
243
  name = instance["name"]
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
  return qa_utils.ResolveInstanceName(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
244
245


246
247
def TestInstanceExportWithRemove(instance, node):
  """gnt-backup export --remove-instance"""
Iustin Pop's avatar
Iustin Pop committed
248
249
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                 "--remove-instance", instance["name"]])
250
251


252
253
def TestInstanceExportNoTarget(instance):
  """gnt-backup export (without target node, should fail)"""
Iustin Pop's avatar
Iustin Pop committed
254
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
255
256


Michael Hanselmann's avatar
Michael Hanselmann committed
257
258
259
260
261
def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
  cmd = (['gnt-backup', 'import',
          '--disk-template=plain',
          '--no-ip-check',
262
          '--net', '0:mac=generate',
Michael Hanselmann's avatar
Michael Hanselmann committed
263
264
265
266
267
          '--src-node=%s' % expnode['primary'],
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
          '--node=%s' % node['primary']] +
         _GetGenericAddParameters())
  cmd.append(newinst['name'])
Iustin Pop's avatar
Iustin Pop committed
268
  AssertCommand(cmd)
Michael Hanselmann's avatar
Michael Hanselmann committed
269
270
271
272


def TestBackupList(expnode):
  """gnt-backup list"""
Iustin Pop's avatar
Iustin Pop committed
273
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
274
275
276
277
278
279
280


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

281
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
282
283
284
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

285
  print qa_utils.FormatInfo("Getting physical disk names")
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
         '--output=node,phys,instance',
         node['primary'], node2['primary']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))

  # Get physical disk names
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
  node2disk = {}
  for line in output.splitlines():
    (node_name, phys, inst) = line.split('|')
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
Iustin Pop's avatar
Iustin Pop committed
302
        raise qa_error.Error("Unknown disk name format: %s" % phys)
303
304
305
306
307
308

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
309
310
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
311

312
313
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
314
315
316
317
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
Iustin Pop's avatar
Iustin Pop committed
318
    AssertCommand(" && ".join(cmds), node=node_name)
319

320
  print qa_utils.FormatInfo("Getting device paths")
321
322
323
324
325
326
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
  devpath = []
  for line in output.splitlines():
    (_, _, tmpdevpath) = line.split(':')
    devpath.append(tmpdevpath)
327
  print devpath
328

329
  print qa_utils.FormatInfo("Getting drbd device paths")
330
331
  cmd = ['gnt-instance', 'info', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
332
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
333
334
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
  drbddevs = re.findall(pattern, output, re.M)
335
  print drbddevs
336
337
338

  halted_disks = []
  try:
339
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
340
341
342
343
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
344
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
345

346
347
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
348
349
350
351
352
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
Iustin Pop's avatar
Iustin Pop committed
353
      AssertCommand(" && ".join(cmds), node=node)
354
355
      time.sleep(3)

356
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
357
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
358
      AssertCommand(["drbdsetup", name, "show"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
359

Iustin Pop's avatar
Iustin Pop committed
360
    AssertCommand(["gnt-instance", "info", instance["name"]])
361
362

  finally:
363
    print qa_utils.FormatInfo("Activating disks again")
364
365
366
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
Iustin Pop's avatar
Iustin Pop committed
367
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
368

Michael Hanselmann's avatar
Michael Hanselmann committed
369
370
  if onmaster:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
371
      AssertCommand(["drbdsetup", name, "detach"], node=node)
Michael Hanselmann's avatar
Michael Hanselmann committed
372
373
  else:
    for name in drbddevs:
Iustin Pop's avatar
Iustin Pop committed
374
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
Michael Hanselmann's avatar
Michael Hanselmann committed
375

376
  # TODO
Iustin Pop's avatar
Iustin Pop committed
377
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
Michael Hanselmann's avatar
Michael Hanselmann committed
378

379
  print qa_utils.FormatInfo("Making sure disks are up again")
Iustin Pop's avatar
Iustin Pop committed
380
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
381
382

  print qa_utils.FormatInfo("Restarting instance")
Iustin Pop's avatar
Iustin Pop committed
383
384
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
  AssertCommand(["gnt-instance", "startup", instance["name"]])
385

Iustin Pop's avatar
Iustin Pop committed
386
  AssertCommand(["gnt-cluster", "verify"])
387
388
389
390


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
391
392
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
393
  # The following can cause crashes, thus it's disabled until fixed
394
  #return _TestInstanceDiskFailure(instance, node, node2, True)
395
396
397
398
399


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)