diff --git a/qa/qa_cluster.py b/qa/qa_cluster.py index 995bbf15bafabd4945470de0735f49985b246fb5..dc24b755d432b87f6b22d06e1006c67ffad84b2e 100644 --- a/qa/qa_cluster.py +++ b/qa/qa_cluster.py @@ -1,7 +1,7 @@ # # -# Copyright (C) 2007, 2010, 2011, 2012 Google Inc. +# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,6 +23,7 @@ """ +import re import tempfile import os.path @@ -59,6 +60,46 @@ def _CheckFileOnAllNodes(filename, content): AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content) +# Cluster-verify errors (date, "ERROR", then error code) +_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- ERROR:([A-Z0-9_-]+):") + + +def _GetCVErrorCodes(cvout): + ret = set() + for l in cvout.splitlines(): + m = _CVERROR_RE.match(l) + if m: + ecode = m.group(1) + ret.add(ecode) + return ret + + +def AssertClusterVerify(fail=False, errors=None): + """Run cluster-verify and check the result + + @type fail: bool + @param fail: if cluster-verify is expected to fail instead of succeeding + @type errors: list of tuples + @param errors: List of CV_XXX errors that are expected; if specified, all the + errors listed must appear in cluster-verify output. A non-empty value + implies C{fail=True}. + + """ + cvcmd = "gnt-cluster verify" + mnode = qa_config.GetMasterNode() + if errors: + cvout = GetCommandOutput(mnode["primary"], cvcmd + " --error-codes", + fail=True) + actual = _GetCVErrorCodes(cvout) + expected = compat.UniqueFrozenset(e for (_, e, _) in errors) + if not actual.issuperset(expected): + missing = expected.difference(actual) + raise qa_error.Error("Cluster-verify didn't return these expected" + " errors: %s" % utils.CommaJoin(missing)) + else: + AssertCommand(cvcmd, fail=fail, node=mnode) + + # data for testing failures due to bad keys/values for disk parameters _FAIL_PARAMS = ["nonexistent:resync-rate=1", "drbd:nonexistent=1", diff --git a/qa/qa_utils.py b/qa/qa_utils.py index 45baea6b4f896021b8b6c06f153087e898163c96..9a67a0a771cc0379a8bb1e10e8f0cc366d10935d 100644 --- a/qa/qa_utils.py +++ b/qa/qa_utils.py @@ -1,7 +1,7 @@ # # -# Copyright (C) 2007, 2011, 2012 Google Inc. +# Copyright (C) 2007, 2011, 2012, 2013 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -148,6 +148,18 @@ def _GetName(entity, key): return result +def _AssertRetCode(rcode, fail, cmdstr, nodename): + """Check the return value from a command and possibly raise an exception. + + """ + if fail and rcode == 0: + raise qa_error.Error("Command '%s' on node %s was expected to fail but" + " didn't" % (cmdstr, nodename)) + elif not fail and rcode != 0: + raise qa_error.Error("Command '%s' on node %s failed, exit code %s" % + (cmdstr, nodename, rcode)) + + def AssertCommand(cmd, fail=False, node=None): """Checks that a remote command succeeds. @@ -173,15 +185,7 @@ def AssertCommand(cmd, fail=False, node=None): cmdstr = utils.ShellQuoteArgs(cmd) rcode = StartSSH(nodename, cmdstr).wait() - - if fail: - if rcode == 0: - raise qa_error.Error("Command '%s' on node %s was expected to fail but" - " didn't" % (cmdstr, nodename)) - else: - if rcode != 0: - raise qa_error.Error("Command '%s' on node %s failed, exit code %s" % - (cmdstr, nodename, rcode)) + _AssertRetCode(rcode, fail, cmdstr, nodename) return rcode @@ -278,13 +282,23 @@ def CloseMultiplexers(): utils.RemoveFile(sname) -def GetCommandOutput(node, cmd, tty=None): +def GetCommandOutput(node, cmd, tty=None, fail=False): """Returns the output of a command executed on the given node. + @type node: string + @param node: node the command should run on + @type cmd: string + @param cmd: command to be executed in the node (cannot be empty or None) + @type tty: bool or None + @param tty: if we should use tty; if None, it will be auto-detected + @type fail: bool + @param fail: whether the command is expected to fail """ + assert cmd p = StartLocalCommand(GetSSHCommand(node, cmd, tty=tty), stdout=subprocess.PIPE) - AssertEqual(p.wait(), 0) + rcode = p.wait() + _AssertRetCode(rcode, fail, node, cmd) return p.stdout.read()