qa_utils.py 27.1 KB
Newer Older
1
2
3
#
#

4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
Klaus Aehlig's avatar
Klaus Aehlig committed
5
# All rights reserved.
6
#
Klaus Aehlig's avatar
Klaus Aehlig committed
7
8
9
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
10
#
Klaus Aehlig's avatar
Klaus Aehlig committed
11
12
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
13
#
Klaus Aehlig's avatar
Klaus Aehlig committed
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31
32
33
34


"""Utilities for QA tests.

"""

35
import copy
36
import datetime
37
import operator
38
import os
39
import random
40
import re
41
import socket
42
import subprocess
43
import sys
44
import tempfile
45
import yaml
46

47
48
49
50
51
try:
  import functools
except ImportError, err:
  raise ImportError("Python 2.5 or higher is required: %s" % err)

52
from ganeti import utils
53
from ganeti import compat
54
from ganeti import constants
55
from ganeti import ht
56
from ganeti import pathutils
57
from ganeti import vcluster
58

59
import colors
60
61
62
import qa_config
import qa_error

63
from qa_logging import FormatInfo
64

Michael Hanselmann's avatar
Michael Hanselmann committed
65

66
67
_MULTIPLEXERS = {}

68
69
70
#: Unique ID per QA run
_RUN_UUID = utils.NewUUID()

71
72
73
#: Path to the QA query output log file
_QA_OUTPUT = pathutils.GetLogFilename("qa-output")

74
75
76
77
78
79
80

(INST_DOWN,
 INST_UP) = range(500, 502)

(FIRST_ARG,
 RETURN_VALUE) = range(1000, 1002)

Michael Hanselmann's avatar
Michael Hanselmann committed
81

82
83
84
85
86
87
88
89
90
91
92
93
def _RaiseWithInfo(msg, error_desc):
  """Raises a QA error with the given content, and adds a message if present.

  """
  if msg:
    output = "%s: %s" % (msg, error_desc)
  else:
    output = error_desc
  raise qa_error.Error(output)


def AssertIn(item, sequence, msg=None):
94
95
96
97
  """Raises an error when item is not in sequence.

  """
  if item not in sequence:
98
    _RaiseWithInfo(msg, "%r not in %r" % (item, sequence))
99
100


101
def AssertNotIn(item, sequence, msg=None):
102
103
104
105
  """Raises an error when item is in sequence.

  """
  if item in sequence:
106
    _RaiseWithInfo(msg, "%r in %r" % (item, sequence))
107
108


109
def AssertEqual(first, second, msg=None):
110
111
112
113
  """Raises an error when values aren't equal.

  """
  if not first == second:
114
    _RaiseWithInfo(msg, "%r == %r" % (first, second))
115
116


117
def AssertMatch(string, pattern, msg=None):
118
119
120
121
  """Raises an error when string doesn't match regexp pattern.

  """
  if not re.match(pattern, string):
122
    _RaiseWithInfo(msg, "%r doesn't match /%r/" % (string, pattern))
123
124


125
def _GetName(entity, fn):
126
127
128
  """Tries to get name of an entity.

  @type entity: string or dict
129
  @param fn: Function retrieving name from entity
130
131
132
133
134

  """
  if isinstance(entity, basestring):
    result = entity
  else:
135
    result = fn(entity)
136
137
138
139
140
141
142

  if not ht.TNonEmptyString(result):
    raise Exception("Invalid name '%s'" % result)

  return result


143
144
145
146
147
148
149
150
151
152
153
154
def _AssertRetCode(rcode, fail, cmdstr, nodename):
  """Check the return value from a command and possibly raise an exception.

  """
  if fail and rcode == 0:
    raise qa_error.Error("Command '%s' on node %s was expected to fail but"
                         " didn't" % (cmdstr, nodename))
  elif not fail and rcode != 0:
    raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
                         (cmdstr, nodename, rcode))


155
def AssertCommand(cmd, fail=False, node=None, log_cmd=True, max_seconds=None):
Iustin Pop's avatar
Iustin Pop committed
156
157
158
159
  """Checks that a remote command succeeds.

  @param cmd: either a string (the command to execute) or a list (to
      be converted using L{utils.ShellQuoteArgs} into a string)
160
161
162
  @type fail: boolean or None
  @param fail: if the command is expected to fail instead of succeeding,
               or None if we don't care
Iustin Pop's avatar
Iustin Pop committed
163
164
165
  @param node: if passed, it should be the node on which the command
      should be executed, instead of the master node (can be either a
      dict or a string)
166
167
  @param log_cmd: if False, the command won't be logged (simply passed to
      StartSSH)
168
169
170
  @type max_seconds: double
  @param max_seconds: fail if the command takes more than C{max_seconds}
      seconds
171
  @return: the return code, stdout and stderr of the command
172
  @raise qa_error.Error: if the command fails when it shouldn't or vice versa
Iustin Pop's avatar
Iustin Pop committed
173
174
175
176
177

  """
  if node is None:
    node = qa_config.GetMasterNode()

178
  nodename = _GetName(node, operator.attrgetter("primary"))
Iustin Pop's avatar
Iustin Pop committed
179
180
181
182
183
184

  if isinstance(cmd, basestring):
    cmdstr = cmd
  else:
    cmdstr = utils.ShellQuoteArgs(cmd)

185
  start = datetime.datetime.now()
186
187
188
189
  popen = StartSSH(nodename, cmdstr, log_cmd=log_cmd)
  # Run the command
  stdout, stderr = popen.communicate()
  rcode = popen.returncode
190
  duration_seconds = TimedeltaToTotalSeconds(datetime.datetime.now() - start)
191
192
  if fail is not None:
    _AssertRetCode(rcode, fail, cmdstr, nodename)
Iustin Pop's avatar
Iustin Pop committed
193

194
195
196
197
198
199
  if max_seconds is not None:
    if duration_seconds > max_seconds:
      raise qa_error.Error(
        "Cmd '%s' took %f seconds, maximum of %f was exceeded" %
        (cmdstr, duration_seconds, max_seconds))

200
  return rcode, stdout, stderr
201

Iustin Pop's avatar
Iustin Pop committed
202

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True):
  """Executes a command with redirected output.

  The log will go to the qa-output log file in the ganeti log
  directory on the node where the command is executed. The fail and
  node parameters are passed unchanged to AssertCommand.

  @param cmd: the command to be executed, as a list; a string is not
      supported

  """
  if not isinstance(cmd, list):
    raise qa_error.Error("Non-list passed to AssertRedirectedCommand")
  ofile = utils.ShellQuote(_QA_OUTPUT)
  cmdstr = utils.ShellQuoteArgs(cmd)
  AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile),
                fail=False, node=node, log_cmd=False)
  return AssertCommand(cmdstr + " >> %s" % ofile,
                       fail=fail, node=node, log_cmd=log_cmd)


224
def GetSSHCommand(node, cmd, strict=True, opts=None, tty=False,
225
                  use_multiplexer=True):
226
227
  """Builds SSH command to be executed.

228
229
230
  @type node: string
  @param node: node the command should run on
  @type cmd: string
231
232
  @param cmd: command to be executed in the node; if None or empty
      string, no command will be executed
233
234
  @type strict: boolean
  @param strict: whether to enable strict host key checking
235
236
  @type opts: list
  @param opts: list of additional options
Iustin Pop's avatar
Iustin Pop committed
237
238
  @type tty: boolean or None
  @param tty: if we should use tty; if None, will be auto-detected
239
240
  @type use_multiplexer: boolean
  @param use_multiplexer: if the multiplexer for the node should be used
241

242
  """
243
  args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-lroot"]
244

Iustin Pop's avatar
Iustin Pop committed
245
246
247
  if tty is None:
    tty = sys.stdout.isatty()

248
249
  if tty:
    args.append("-t")
250
251

  if strict:
Iustin Pop's avatar
Iustin Pop committed
252
    tmp = "yes"
253
  else:
Iustin Pop's avatar
Iustin Pop committed
254
255
256
257
    tmp = "no"
  args.append("-oStrictHostKeyChecking=%s" % tmp)
  args.append("-oClearAllForwardings=yes")
  args.append("-oForwardAgent=yes")
258
259
  if opts:
    args.extend(opts)
260
  if node in _MULTIPLEXERS and use_multiplexer:
261
    spath = _MULTIPLEXERS[node][0]
Iustin Pop's avatar
Iustin Pop committed
262
263
    args.append("-oControlPath=%s" % spath)
    args.append("-oControlMaster=no")
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282

  (vcluster_master, vcluster_basedir) = \
    qa_config.GetVclusterSettings()

  if vcluster_master:
    args.append(vcluster_master)
    args.append("%s/%s/cmd" % (vcluster_basedir, node))

    if cmd:
      # For virtual clusters the whole command must be wrapped using the "cmd"
      # script, as that script sets a number of environment variables. If the
      # command contains shell meta characters the whole command needs to be
      # quoted.
      args.append(utils.ShellQuote(cmd))
  else:
    args.append(node)

    if cmd:
      args.append(cmd)
283
284
285
286

  return args


287
def StartLocalCommand(cmd, _nolog_opts=False, log_cmd=True, **kwargs):
288
289
290
  """Starts a local command.

  """
291
292
293
294
295
  if log_cmd:
    if _nolog_opts:
      pcmd = [i for i in cmd if not i.startswith("-")]
    else:
      pcmd = cmd
296
297
    print "%s %s" % (colors.colorize("Command:", colors.CYAN),
                     utils.ShellQuoteArgs(pcmd))
298
299
300
  return subprocess.Popen(cmd, shell=False, **kwargs)


301
def StartSSH(node, cmd, strict=True, log_cmd=True):
302
303
304
  """Starts SSH.

  """
305
  return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict),
306
307
                           _nolog_opts=True, log_cmd=log_cmd,
                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
308
309


310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
def StartMultiplexer(node):
  """Starts a multiplexer command.

  @param node: the node for which to open the multiplexer

  """
  if node in _MULTIPLEXERS:
    return

  # Note: yes, we only need mktemp, since we'll remove the file anyway
  sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
  utils.RemoveFile(sname)
  opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
  print "Created socket at %s" % sname
  child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
  _MULTIPLEXERS[node] = (sname, child)


def CloseMultiplexers():
  """Closes all current multiplexers and cleans up.

  """
  for node in _MULTIPLEXERS.keys():
    (sname, child) = _MULTIPLEXERS.pop(node)
    utils.KillProcess(child.pid, timeout=10, waitpid=True)
    utils.RemoveFile(sname)


338
339
340
341
342
343
344
345
346
def _GetCommandStdout(proc):
  """Extract the stored standard error, print it and return it.

  """
  out = proc.stdout.read()
  sys.stdout.write(out)
  return out


347
def GetCommandOutput(node, cmd, tty=False, use_multiplexer=True, log_cmd=True,
348
                     fail=False):
349
350
  """Returns the output of a command executed on the given node.

351
352
353
354
355
356
  @type node: string
  @param node: node the command should run on
  @type cmd: string
  @param cmd: command to be executed in the node (cannot be empty or None)
  @type tty: bool or None
  @param tty: if we should use tty; if None, it will be auto-detected
357
358
359
  @type use_multiplexer: bool
  @param use_multiplexer: if the SSH multiplexer provided by the QA should be
                          used or not
360
361
  @type log_cmd: bool
  @param log_cmd: if the command should be logged
362
363
  @type fail: bool
  @param fail: whether the command is expected to fail
364
  """
365
  assert cmd
366
367
  p = StartLocalCommand(GetSSHCommand(node, cmd, tty=tty,
                                      use_multiplexer=use_multiplexer),
368
                        stdout=subprocess.PIPE, log_cmd=log_cmd)
369
  rcode = p.wait()
370
  out = _GetCommandStdout(p)
371
  _AssertRetCode(rcode, fail, cmd, node)
372
  return out
373
374


375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
def GetObjectInfo(infocmd):
  """Get and parse information about a Ganeti object.

  @type infocmd: list of strings
  @param infocmd: command to be executed, e.g. ["gnt-cluster", "info"]
  @return: the information parsed, appropriately stored in dictionaries,
      lists...

  """
  master = qa_config.GetMasterNode()
  cmdline = utils.ShellQuoteArgs(infocmd)
  info_out = GetCommandOutput(master.primary, cmdline)
  return yaml.load(info_out)


390
391
392
393
394
def UploadFile(node, src):
  """Uploads a file to a node and returns the filename.

  Caller needs to remove the returned file on the node when it's not needed
  anymore.
395

396
397
398
399
  """
  # Make sure nobody else has access to it while preserving local permissions
  mode = os.stat(src).st_mode & 0700

400
401
  cmd = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
         'chmod %o "${tmp}" && '
402
403
404
405
         '[[ -f "${tmp}" ]] && '
         'cat > "${tmp}" && '
         'echo "${tmp}"') % mode

Iustin Pop's avatar
Iustin Pop committed
406
  f = open(src, "r")
407
408
409
410
411
412
  try:
    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
                         stdout=subprocess.PIPE)
    AssertEqual(p.wait(), 0)

    # Return temporary filename
413
    return _GetCommandStdout(p).strip()
414
415
  finally:
    f.close()
Michael Hanselmann's avatar
Michael Hanselmann committed
416
417


418
419
420
421
422
423
424
425
426
427
def UploadData(node, data, mode=0600, filename=None):
  """Uploads data to a node and returns the filename.

  Caller needs to remove the returned file on the node when it's not needed
  anymore.

  """
  if filename:
    tmp = "tmp=%s" % utils.ShellQuote(filename)
  else:
428
429
    tmp = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
           'chmod %o "${tmp}"') % mode
430
431
432
433
434
435
436
437
438
439
440
441
  cmd = ("%s && "
         "[[ -f \"${tmp}\" ]] && "
         "cat > \"${tmp}\" && "
         "echo \"${tmp}\"") % tmp

  p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  p.stdin.write(data)
  p.stdin.close()
  AssertEqual(p.wait(), 0)

  # Return temporary filename
442
  return _GetCommandStdout(p).strip()
443
444


445
446
447
448
449
450
451
def BackupFile(node, path):
  """Creates a backup of a file on the node and returns the filename.

  Caller needs to remove the returned file on the node when it's not needed
  anymore.

  """
452
453
  vpath = MakeNodePath(node, path)

454
  cmd = ("tmp=$(mktemp .gnt.XXXXXX --tmpdir=$(dirname %s)) && "
455
456
         "[[ -f \"$tmp\" ]] && "
         "cp %s $tmp && "
457
         "echo $tmp") % (utils.ShellQuote(vpath), utils.ShellQuote(vpath))
458
459

  # Return temporary filename
460
461
462
463
464
  result = GetCommandOutput(node, cmd).strip()

  print "Backup filename: %s" % result

  return result
465
466


Michael Hanselmann's avatar
Michael Hanselmann committed
467
468
469
def ResolveInstanceName(instance):
  """Gets the full name of an instance.

470
471
472
  @type instance: string
  @param instance: Instance name

473
  """
474
475
  info = GetObjectInfo(["gnt-instance", "info", instance])
  return info[0]["Instance name"]
476
477
478
479
480
481


def ResolveNodeName(node):
  """Gets the full name of a node.

  """
482
483
  info = GetObjectInfo(["gnt-node", "info", node.primary])
  return info[0]["Node name"]
484
485
486
487
488


def GetNodeInstances(node, secondaries=False):
  """Gets a list of instances on a node.

Michael Hanselmann's avatar
Michael Hanselmann committed
489
490
  """
  master = qa_config.GetMasterNode()
491
  node_name = ResolveNodeName(node)
Michael Hanselmann's avatar
Michael Hanselmann committed
492

493
  # Get list of all instances
Iustin Pop's avatar
Iustin Pop committed
494
495
  cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
         "--output=name,pnode,snodes"]
496
  output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
497
498
499

  instances = []
  for line in output.splitlines():
Iustin Pop's avatar
Iustin Pop committed
500
    (name, pnode, snodes) = line.split(":", 2)
501
    if ((not secondaries and pnode == node_name) or
Iustin Pop's avatar
Iustin Pop committed
502
        (secondaries and node_name in snodes.split(","))):
503
      instances.append(name)
Michael Hanselmann's avatar
Michael Hanselmann committed
504

505
  return instances
Michael Hanselmann's avatar
Michael Hanselmann committed
506
507


508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
def _SelectQueryFields(rnd, fields):
  """Generates a list of fields for query tests.

  """
  # Create copy for shuffling
  fields = list(fields)
  rnd.shuffle(fields)

  # Check all fields
  yield fields
  yield sorted(fields)

  # Duplicate fields
  yield fields + fields

  # Check small groups of fields
  while fields:
    yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]


def _List(listcmd, fields, names):
  """Runs a list command.

  """
  master = qa_config.GetMasterNode()

534
  cmd = [listcmd, "list", "--separator=|", "--no-headers",
535
536
537
538
539
         "--output", ",".join(fields)]

  if names:
    cmd.extend(names)

540
  return GetCommandOutput(master.primary,
541
542
543
                          utils.ShellQuoteArgs(cmd)).splitlines()


544
def GenericQueryTest(cmd, fields, namefield="name", test_unknown=True):
545
546
547
548
549
550
551
552
  """Runs a number of tests on query commands.

  @param cmd: Command name
  @param fields: List of field names

  """
  rnd = random.Random(hash(cmd))

Iustin Pop's avatar
Iustin Pop committed
553
  fields = list(fields)
554
555
556
557
  rnd.shuffle(fields)

  # Test a number of field combinations
  for testfields in _SelectQueryFields(rnd, fields):
558
    AssertRedirectedCommand([cmd, "list", "--output", ",".join(testfields)])
559

560
561
  if namefield is not None:
    namelist_fn = compat.partial(_List, cmd, [namefield])
562

563
564
565
    # When no names were requested, the list must be sorted
    names = namelist_fn(None)
    AssertEqual(names, utils.NiceSort(names))
566

567
568
569
    # When requesting specific names, the order must be kept
    revnames = list(reversed(names))
    AssertEqual(namelist_fn(revnames), revnames)
570

571
572
573
    randnames = list(names)
    rnd.shuffle(randnames)
    AssertEqual(namelist_fn(randnames), randnames)
574

575
576
577
578
  if test_unknown:
    # Listing unknown items must fail
    AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"],
                  fail=True)
579
580

  # Check exit code for listing unknown field
581
582
583
584
  rcode, _, _ = AssertRedirectedCommand([cmd, "list",
                                         "--output=field/does/not/exist"],
                                        fail=True)
  AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
585
586
587
588
589
590


def GenericQueryFieldsTest(cmd, fields):
  master = qa_config.GetMasterNode()

  # Listing fields
591
592
  AssertRedirectedCommand([cmd, "list-fields"])
  AssertRedirectedCommand([cmd, "list-fields"] + fields)
593
594
595

  # Check listed fields (all, must be sorted)
  realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
596
  output = GetCommandOutput(master.primary,
597
598
                            utils.ShellQuoteArgs(realcmd)).splitlines()
  AssertEqual([line.split("|", 1)[0] for line in output],
599
              utils.NiceSort(fields))
600
601

  # Check exit code for listing unknown field
602
603
604
  rcode, _, _ = AssertCommand([cmd, "list-fields", "field/does/not/exist"],
                              fail=True)
  AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
605

606

607
608
609
610
611
612
613
def AddToEtcHosts(hostnames):
  """Adds hostnames to /etc/hosts.

  @param hostnames: List of hostnames first used A records, all other CNAMEs

  """
  master = qa_config.GetMasterNode()
614
  tmp_hosts = UploadData(master.primary, "", mode=0644)
615
616
617
618
619
620

  data = []
  for localhost in ("::1", "127.0.0.1"):
    data.append("%s %s" % (localhost, " ".join(hostnames)))

  try:
621
622
623
624
625
626
627
628
629
    AssertCommand("{ cat %s && echo -e '%s'; } > %s && mv %s %s" %
                  (utils.ShellQuote(pathutils.ETC_HOSTS),
                   "\\n".join(data),
                   utils.ShellQuote(tmp_hosts),
                   utils.ShellQuote(tmp_hosts),
                   utils.ShellQuote(pathutils.ETC_HOSTS)))
  except Exception:
    AssertCommand(["rm", "-f", tmp_hosts])
    raise
630
631
632
633
634
635
636
637
638


def RemoveFromEtcHosts(hostnames):
  """Remove hostnames from /etc/hosts.

  @param hostnames: List of hostnames first used A records, all other CNAMEs

  """
  master = qa_config.GetMasterNode()
639
  tmp_hosts = UploadData(master.primary, "", mode=0644)
640
641
642
643
  quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)

  sed_data = " ".join(hostnames)
  try:
644
645
    AssertCommand((r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
                   r" && mv %s %s") %
646
647
648
649
650
651
                   (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS),
                    quoted_tmp_hosts, quoted_tmp_hosts,
                    utils.ShellQuote(pathutils.ETC_HOSTS)))
  except Exception:
    AssertCommand(["rm", "-f", tmp_hosts])
    raise
652
653
654
655
656
657


def RunInstanceCheck(instance, running):
  """Check if instance is running or not.

  """
658
  instance_name = _GetName(instance, operator.attrgetter("name"))
659

660
661
662
663
664
665
666
  script = qa_config.GetInstanceCheckScript()
  if not script:
    return

  master_node = qa_config.GetMasterNode()

  # Build command to connect to master node
667
  master_ssh = GetSSHCommand(master_node.primary, "--")
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732

  if running:
    running_shellval = "1"
    running_text = ""
  else:
    running_shellval = ""
    running_text = "not "

  print FormatInfo("Checking if instance '%s' is %srunning" %
                   (instance_name, running_text))

  args = [script, instance_name]
  env = {
    "PATH": constants.HOOKS_PATH,
    "RUN_UUID": _RUN_UUID,
    "MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
    "INSTANCE_NAME": instance_name,
    "INSTANCE_RUNNING": running_shellval,
    }

  result = os.spawnve(os.P_WAIT, script, args, env)
  if result != 0:
    raise qa_error.Error("Instance check failed with result %s" % result)


def _InstanceCheckInner(expected, instarg, args, result):
  """Helper function used by L{InstanceCheck}.

  """
  if instarg == FIRST_ARG:
    instance = args[0]
  elif instarg == RETURN_VALUE:
    instance = result
  else:
    raise Exception("Invalid value '%s' for instance argument" % instarg)

  if expected in (INST_DOWN, INST_UP):
    RunInstanceCheck(instance, (expected == INST_UP))
  elif expected is not None:
    raise Exception("Invalid value '%s'" % expected)


def InstanceCheck(before, after, instarg):
  """Decorator to check instance status before and after test.

  @param before: L{INST_DOWN} if instance must be stopped before test,
    L{INST_UP} if instance must be running before test, L{None} to not check.
  @param after: L{INST_DOWN} if instance must be stopped after test,
    L{INST_UP} if instance must be running after test, L{None} to not check.
  @param instarg: L{FIRST_ARG} to use first argument to test as instance (a
    dictionary), L{RETURN_VALUE} to use return value (disallows pre-checks)

  """
  def decorator(fn):
    @functools.wraps(fn)
    def wrapper(*args, **kwargs):
      _InstanceCheckInner(before, instarg, args, NotImplemented)

      result = fn(*args, **kwargs)

      _InstanceCheckInner(after, instarg, args, result)

      return result
    return wrapper
  return decorator
733
734
735
736
737
738


def GetNonexistentGroups(count):
  """Gets group names which shouldn't exist on the cluster.

  @param count: Number of groups to get
Helga Velroyen's avatar
Helga Velroyen committed
739
  @rtype: integer
740
741

  """
Helga Velroyen's avatar
Helga Velroyen committed
742
  return GetNonexistentEntityNames(count, "groups", "group")
743

Helga Velroyen's avatar
Helga Velroyen committed
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765

def GetNonexistentEntityNames(count, name_config, name_prefix):
  """Gets entity names which shouldn't exist on the cluster.

  The actualy names can refer to arbitrary entities (for example
  groups, networks).

  @param count: Number of names to get
  @rtype: integer
  @param name_config: name of the leaf in the config containing
    this entity's configuration, including a 'inexistent-'
    element
  @rtype: string
  @param name_prefix: prefix of the entity's names, used to compose
    the default values; for example for groups, the prefix is
    'group' and the generated names are then group1, group2, ...
  @rtype: string

  """
  entities = qa_config.get(name_config, {})

  default = [name_prefix + str(i) for i in range(count)]
766
767
  assert count <= len(default)

Helga Velroyen's avatar
Helga Velroyen committed
768
769
  name_config_inexistent = "inexistent-" + name_config
  candidates = entities.get(name_config_inexistent, default)[:count]
770
771

  if len(candidates) < count:
Helga Velroyen's avatar
Helga Velroyen committed
772
773
    raise Exception("At least %s non-existent %s are needed" %
                    (count, name_config))
774
775

  return candidates
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798


def MakeNodePath(node, path):
  """Builds an absolute path for a virtual node.

  @type node: string or L{qa_config._QaNode}
  @param node: Node
  @type path: string
  @param path: Path without node-specific prefix

  """
  (_, basedir) = qa_config.GetVclusterSettings()

  if isinstance(node, basestring):
    name = node
  else:
    name = node.primary

  if basedir:
    assert path.startswith("/")
    return "%s%s" % (vcluster.MakeNodeRoot(basedir, name), path)
  else:
    return path
799
800


801
def _GetParameterOptions(specs):
802
  """Helper to build policy options."""
803
804
  values = ["%s=%s" % (par, val)
            for (par, val) in specs.items()]
805
806
807
  return ",".join(values)


808
809
def TestSetISpecs(new_specs=None, diff_specs=None, get_policy_fn=None,
                  build_cmd_fn=None, fail=False, old_values=None):
810
811
  """Change instance specs for an object.

812
813
814
815
816
817
  At most one of new_specs or diff_specs can be specified.

  @type new_specs: dict
  @param new_specs: new complete specs, in the same format returned by
      L{ParseIPolicy}.
  @type diff_specs: dict
818
819
820
  @param diff_specs: partial specs, it can be an incomplete specifications, but
      if min/max specs are specified, their number must match the number of the
      existing specs
821
822
  @type get_policy_fn: function
  @param get_policy_fn: function that returns the current policy as in
823
      L{ParseIPolicy}
824
825
826
827
828
829
830
  @type build_cmd_fn: function
  @param build_cmd_fn: function that return the full command line from the
      options alone
  @type fail: bool
  @param fail: if the change is expected to fail
  @type old_values: tuple
  @param old_values: (old_policy, old_specs), as returned by
831
832
     L{ParseIPolicy}
  @return: same as L{ParseIPolicy}
833
834
835
836

  """
  assert get_policy_fn is not None
  assert build_cmd_fn is not None
837
  assert new_specs is None or diff_specs is None
838
839
840
841
842

  if old_values:
    (old_policy, old_specs) = old_values
  else:
    (old_policy, old_specs) = get_policy_fn()
843
844
845

  if diff_specs:
    new_specs = copy.deepcopy(old_specs)
846
847
848
849
850
851
852
853
854
855
    if constants.ISPECS_MINMAX in diff_specs:
      AssertEqual(len(new_specs[constants.ISPECS_MINMAX]),
                  len(diff_specs[constants.ISPECS_MINMAX]))
      for (new_minmax, diff_minmax) in zip(new_specs[constants.ISPECS_MINMAX],
                                           diff_specs[constants.ISPECS_MINMAX]):
        for (key, parvals) in diff_minmax.items():
          for (par, val) in parvals.items():
            new_minmax[key][par] = val
    for (par, val) in diff_specs.get(constants.ISPECS_STD, {}).items():
      new_specs[constants.ISPECS_STD][par] = val
856

857
858
  if new_specs:
    cmd = []
859
    if (diff_specs is None or constants.ISPECS_MINMAX in diff_specs):
860
      minmax_opt_items = []
861
862
863
864
865
866
      for minmax in new_specs[constants.ISPECS_MINMAX]:
        minmax_opts = []
        for key in ["min", "max"]:
          keyopt = _GetParameterOptions(minmax[key])
          minmax_opts.append("%s:%s" % (key, keyopt))
        minmax_opt_items.append("/".join(minmax_opts))
867
868
      cmd.extend([
        "--ipolicy-bounds-specs",
869
        "//".join(minmax_opt_items)
870
        ])
871
    if diff_specs is None:
872
      std_source = new_specs
873
874
    else:
      std_source = diff_specs
875
    std_opt = _GetParameterOptions(std_source.get("std", {}))
876
877
878
879
    if std_opt:
      cmd.extend(["--ipolicy-std-specs", std_opt])
    AssertCommand(build_cmd_fn(cmd), fail=fail)

880
881
882
883
884
885
886
887
    # Check the new state
    (eff_policy, eff_specs) = get_policy_fn()
    AssertEqual(eff_policy, old_policy)
    if fail:
      AssertEqual(eff_specs, old_specs)
    else:
      AssertEqual(eff_specs, new_specs)

888
  else:
889
890
    (eff_policy, eff_specs) = (old_policy, old_specs)

891
892
893
894
895
896
897
898
899
900
901
  return (eff_policy, eff_specs)


def ParseIPolicy(policy):
  """Parse and split instance an instance policy.

  @type policy: dict
  @param policy: policy, as returned by L{GetObjectInfo}
  @rtype: tuple
  @return: (policy, specs), where:
      - policy is a dictionary of the policy values, instance specs excluded
902
903
      - specs is a dictionary containing only the specs, using the internal
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
904
905
906
907
908

  """
  ret_specs = {}
  ret_policy = {}
  for (key, val) in policy.items():
909
910
911
912
913
914
915
916
917
918
    if key == "bounds specs":
      ret_specs[constants.ISPECS_MINMAX] = []
      for minmax in val:
        ret_minmax = {}
        for key in minmax:
          keyparts = key.split("/", 1)
          assert len(keyparts) > 1
          ret_minmax[keyparts[0]] = minmax[key]
        ret_specs[constants.ISPECS_MINMAX].append(ret_minmax)
    elif key == constants.ISPECS_STD:
919
      ret_specs[key] = val
920
921
922
    else:
      ret_policy[key] = val
  return (ret_policy, ret_specs)
923
924
925
926
927
928
929


def UsesIPv6Connection(host, port):
  """Returns True if the connection to a given host/port could go through IPv6.

  """
  return any(t[0] == socket.AF_INET6 for t in socket.getaddrinfo(host, port))
930
931


932
933
934
935
936
937
938
939
940
941
942
943
944
945
def TimedeltaToTotalSeconds(td):
  """Returns the total seconds in a C{datetime.timedelta} object.

  This performs the same task as the C{datetime.timedelta.total_seconds()}
  method which is present in Python 2.7 onwards.

  @type td: datetime.timedelta
  @param td: timedelta object to convert
  @rtype float
  @return: total seconds in the timedelta object

  """
  return ((td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10 ** 6) /
          10 ** 6)