qa_instance.py 14.2 KB
Newer Older
1 2 3
#
#

4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Instance related QA tests.

"""

26 27 28
import re
import time

29
from ganeti import utils
Michael Hanselmann's avatar
Michael Hanselmann committed
30
from ganeti import constants
31 32

import qa_config
Michael Hanselmann's avatar
Michael Hanselmann committed
33
import qa_utils
34 35 36
import qa_error

from qa_utils import AssertEqual, AssertNotEqual, StartSSH
37

38 39 40

def _GetDiskStatePath(disk):
  return "/sys/block/%s/device/state" % disk
41 42


Michael Hanselmann's avatar
Michael Hanselmann committed
43
def _GetGenericAddParameters():
44 45 46 47
  params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  for idx, size in enumerate(qa_config.get('disk')):
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
  return params
Michael Hanselmann's avatar
Michael Hanselmann committed
48 49


50
def _DiskTest(node, disk_template):
51 52 53 54
  master = qa_config.GetMasterNode()

  instance = qa_config.AcquireInstance()
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
55 56
    cmd = (['gnt-instance', 'add',
            '--os-type=%s' % qa_config.get('os'),
57 58
            '--disk-template=%s' % disk_template,
            '--node=%s' % node] +
Michael Hanselmann's avatar
Michael Hanselmann committed
59
           _GetGenericAddParameters())
60 61 62 63 64 65 66 67 68 69 70 71
    cmd.append(instance['name'])

    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
    return instance
  except:
    qa_config.ReleaseInstance(instance)
    raise


def TestInstanceAddWithPlainDisk(node):
  """gnt-instance add -t plain"""
72
  return _DiskTest(node['primary'], 'plain')
73 74


75 76 77 78 79 80
def TestInstanceAddWithDrbdDisk(node, node2):
  """gnt-instance add -t drbd"""
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
                   'drbd')


81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
def TestInstanceRemove(instance):
  """gnt-instance remove"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  qa_config.ReleaseInstance(instance)


def TestInstanceStartup(instance):
  """gnt-instance startup"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'startup', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


def TestInstanceShutdown(instance):
  """gnt-instance shutdown"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'shutdown', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


110 111 112 113 114 115 116 117 118 119 120
def TestInstanceReboot(instance):
  """gnt-instance reboot"""
  master = qa_config.GetMasterNode()

  for reboottype in ["soft", "hard", "full"]:
    cmd = ['gnt-instance', 'reboot', '--type=%s' % reboottype,
           instance['name']]
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)


Michael Hanselmann's avatar
Michael Hanselmann committed
121 122 123 124 125 126 127 128 129
def TestInstanceReinstall(instance):
  """gnt-instance reinstall"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'reinstall', '-f', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
def TestInstanceRename(instance):
  """gnt-instance rename"""
  master = qa_config.GetMasterNode()

  rename_source = instance['name']
  rename_target = qa_config.get('rename', None)
  if rename_target is None:
    print qa_utils.FormatError('"rename" entry is missing')
    return

  for name1, name2 in [(rename_source, rename_target),
                       (rename_target, rename_source)]:
    cmd = ['gnt-instance', 'rename', name1, name2]
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)


147 148 149 150 151 152 153 154
def TestInstanceFailover(instance):
  """gnt-instance failover"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

155 156 157 158 159
  # ... and back
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

160 161 162 163 164 165 166 167

def TestInstanceInfo(instance):
  """gnt-instance info"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'info', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
Michael Hanselmann's avatar
Michael Hanselmann committed
168 169


170 171 172 173
def TestInstanceModify(instance):
  """gnt-instance modify"""
  master = qa_config.GetMasterNode()

174 175 176 177
  # Assume /sbin/init exists on all systems
  test_kernel = "/sbin/init"
  test_initrd = test_kernel

178 179 180
  orig_memory = qa_config.get('mem')
  orig_bridge = qa_config.get('bridge', 'xen-br0')
  args = [
181 182 183 184 185 186 187 188 189
    ["-B", "%s=128" % constants.BE_MEMORY],
    ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
    ["-B", "%s=2" % constants.BE_VCPUS],
    ["-B", "%s=1" % constants.BE_VCPUS],
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],

    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
190
    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
191 192 193 194 195 196 197 198 199
    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],

    # TODO: bridge tests
    #["--bridge", "xen-br1"],
    #["--bridge", orig_bridge],

    # TODO: Do these tests only with xen-hvm
    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
200 201 202 203 204 205 206 207 208 209 210 211
    ]
  for alist in args:
    cmd = ['gnt-instance', 'modify'] + alist + [instance['name']]
    AssertEqual(StartSSH(master['primary'],
                         utils.ShellQuoteArgs(cmd)).wait(), 0)

  # check no-modify
  cmd = ['gnt-instance', 'modify', instance['name']]
  AssertNotEqual(StartSSH(master['primary'],
                          utils.ShellQuoteArgs(cmd)).wait(), 0)


212 213 214 215 216 217 218 219 220 221 222
def TestInstanceConvertDisk(instance, snode):
  """gnt-instance modify -t"""
  master = qa_config.GetMasterNode()
  cmd = ['gnt-instance', 'modify', '-t', 'plain', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
  cmd = ['gnt-instance', 'modify', '-t', 'drbd', '-n', snode, instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


Michael Hanselmann's avatar
Michael Hanselmann committed
223 224 225 226 227 228 229 230 231
def TestInstanceList():
  """gnt-instance list"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'list']
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


232 233 234 235 236 237 238 239 240
def TestInstanceConsole(instance):
  """gnt-instance console"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-instance', 'console', '--show-cmd', instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
def TestReplaceDisks(instance, pnode, snode, othernode):
  """gnt-instance replace-disks"""
  master = qa_config.GetMasterNode()

  def buildcmd(args):
    cmd = ['gnt-instance', 'replace-disks']
    cmd.extend(args)
    cmd.append(instance["name"])
    return cmd

  cmd = buildcmd(["-p"])
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  cmd = buildcmd(["-s"])
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  cmd = buildcmd(["--new-secondary=%s" % othernode["primary"]])
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  # Restore
  cmd = buildcmd(["--new-secondary=%s" % snode["primary"]])
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)


Michael Hanselmann's avatar
Michael Hanselmann committed
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
def TestInstanceExport(instance, node):
  """gnt-backup export"""
  master = qa_config.GetMasterNode()

  cmd = ['gnt-backup', 'export', '-n', node['primary'], instance['name']]
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)

  return qa_utils.ResolveInstanceName(instance)


def TestInstanceImport(node, newinst, expnode, name):
  """gnt-backup import"""
  master = qa_config.GetMasterNode()

  cmd = (['gnt-backup', 'import',
          '--disk-template=plain',
          '--no-ip-check',
          '--src-node=%s' % expnode['primary'],
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
          '--node=%s' % node['primary']] +
         _GetGenericAddParameters())
  cmd.append(newinst['name'])
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
Michael Hanselmann's avatar
Michael Hanselmann committed
294 295 296 297 298 299


def TestBackupList(expnode):
  """gnt-backup list"""
  master = qa_config.GetMasterNode()

300
  cmd = ['gnt-backup', 'list', '--node=%s' % expnode['primary']]
Michael Hanselmann's avatar
Michael Hanselmann committed
301 302
  AssertEqual(StartSSH(master['primary'],
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
303 304 305 306 307 308 309 310 311 312 313


def _TestInstanceDiskFailure(instance, node, node2, onmaster):
  """Testing disk failure."""
  master = qa_config.GetMasterNode()
  sq = utils.ShellQuoteArgs

  instance_full = qa_utils.ResolveInstanceName(instance)
  node_full = qa_utils.ResolveNodeName(node)
  node2_full = qa_utils.ResolveNodeName(node2)

314
  print qa_utils.FormatInfo("Getting physical disk names")
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
         '--output=node,phys,instance',
         node['primary'], node2['primary']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))

  # Get physical disk names
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
  node2disk = {}
  for line in output.splitlines():
    (node_name, phys, inst) = line.split('|')
    if inst == instance_full:
      if node_name not in node2disk:
        node2disk[node_name] = []

      m = re_disk.match(phys)
      if not m:
        raise qa_error.Error("Unknown disk name format: %s" % disk)

      name = m.group(1)
      if name not in node2disk[node_name]:
        node2disk[node_name].append(name)

  if [node2_full, node_full][int(onmaster)] not in node2disk:
338 339
    raise qa_error.Error("Couldn't find physical disks used on"
                         " %s node" % ["secondary", "master"][int(onmaster)])
340

341 342
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
                            " disks")
343 344 345 346 347 348
  for node_name, disks in node2disk.iteritems():
    cmds = []
    for disk in disks:
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
    AssertEqual(StartSSH(node_name, ' && '.join(cmds)).wait(), 0)

349
  print qa_utils.FormatInfo("Getting device paths")
350 351 352 353 354 355
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
  devpath = []
  for line in output.splitlines():
    (_, _, tmpdevpath) = line.split(':')
    devpath.append(tmpdevpath)
356
  print devpath
357

358
  print qa_utils.FormatInfo("Getting drbd device paths")
359 360
  cmd = ['gnt-instance', 'info', instance['name']]
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
361
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
362 363
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
  drbddevs = re.findall(pattern, output, re.M)
364
  print drbddevs
365 366 367

  halted_disks = []
  try:
368
    print qa_utils.FormatInfo("Deactivating disks")
Michael Hanselmann's avatar
Michael Hanselmann committed
369 370 371 372 373 374 375
    cmds = []
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
      halted_disks.append(name)
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
                         ' && '.join(cmds)).wait(), 0)

376 377
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
                              " to notice the problem")
378 379 380 381 382 383 384 385
    cmds = []
    for disk in devpath:
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
                      "if=%s" % disk, "of=%s" % disk]))
    for _ in (0, 1, 2):
      AssertEqual(StartSSH(node['primary'], ' && '.join(cmds)).wait(), 0)
      time.sleep(3)

386
    print qa_utils.FormatInfo("Debugging info")
Michael Hanselmann's avatar
Michael Hanselmann committed
387 388 389 390
    for name in drbddevs:
      cmd = ['drbdsetup', name, 'show']
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)

391 392 393 394
    cmd = ['gnt-instance', 'info', instance['name']]
    AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)

  finally:
395
    print qa_utils.FormatInfo("Activating disks again")
396 397 398 399 400 401
    cmds = []
    for name in halted_disks:
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
                         '; '.join(cmds)).wait(), 0)

Michael Hanselmann's avatar
Michael Hanselmann committed
402 403 404 405 406 407 408 409 410
  if onmaster:
    for name in drbddevs:
      cmd = ['drbdsetup', name, 'detach']
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
  else:
    for name in drbddevs:
      cmd = ['drbdsetup', name, 'disconnect']
      AssertEqual(StartSSH(node2['primary'], sq(cmd)).wait(), 0)

411 412 413 414
  # TODO
  #cmd = ['vgs']
  #AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
  #                     sq(cmd)).wait(), 0)
Michael Hanselmann's avatar
Michael Hanselmann committed
415

416 417 418 419 420
  print qa_utils.FormatInfo("Making sure disks are up again")
  cmd = ['gnt-instance', 'replace-disks', instance['name']]
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)

  print qa_utils.FormatInfo("Restarting instance")
421 422 423
  cmd = ['gnt-instance', 'shutdown', instance['name']]
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)

Michael Hanselmann's avatar
Michael Hanselmann committed
424
  cmd = ['gnt-instance', 'startup', instance['name']]
425 426 427 428 429 430 431 432
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)

  cmd = ['gnt-cluster', 'verify']
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)


def TestInstanceMasterDiskFailure(instance, node, node2):
  """Testing disk failure on master node."""
433 434
  print qa_utils.FormatError("Disk failure on primary node cannot be"
                             " tested due to potential crashes.")
435
  # The following can cause crashes, thus it's disabled until fixed
436
  #return _TestInstanceDiskFailure(instance, node, node2, True)
437 438 439 440 441


def TestInstanceSecondaryDiskFailure(instance, node, node2):
  """Testing disk failure on secondary node."""
  return _TestInstanceDiskFailure(instance, node, node2, False)