Merge branch 'stable-2.8' into stable-2.9

* stable-2.8
  Include qa/patch in Makefile
  Handle empty patches better
  Move message formatting functions to separate file
  Add optional ordering of QA patch files
  Allow multiple QA patches
  Refactor current patching code

Conflicts:
	Makefile.am (include all added files)
	qa/qa_cluster.py (trivial)
Signed-off-by: Thomas Thrainer <thomasth@google.com>
Reviewed-by: Helga Velroyen <helgav@google.com>

Makefile.am

@@ -130,6 +130,7 @@ DIRS = \
 	lib/watcher \
 	man \
 	qa \
+	qa/patch \
 	test \
 	test/data \
 	test/data/bdev-rbd \
@@ -849,6 +850,7 @@ qa_scripts = \
 	qa/qa_instance_utils.py \
 	qa/qa_job.py \
 	qa/qa_monitoring.py \
+	qa/qa_logging.py \
 	qa/qa_node.py \
 	qa/qa_os.py \
 	qa/qa_rapi.py \

qa/qa_cluster.py

@@ -34,9 +34,10 @@ from ganeti import pathutils
 
 import qa_config
 import qa_daemon
-import qa_utils
 import qa_error
 import qa_instance
+import qa_logging
+import qa_utils
 
 from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
 
@@ -265,7 +266,7 @@ def TestClusterRename():
   original_name = qa_config.get("name")
   rename_target = qa_config.get("rename", None)
   if rename_target is None:
-    print qa_utils.FormatError('"rename" entry is missing')
+    print qa_logging.FormatError('"rename" entry is missing')
     return
 
   for data in [

qa/qa_config.py

@@ -32,6 +32,7 @@ from ganeti import compat
 from ganeti import ht
 
 import qa_error
+import qa_logging
 
 
 _INSTANCE_CHECK_KEY = "instance-check"
@@ -40,9 +41,12 @@ _VCLUSTER_MASTER_KEY = "vcluster-master"
 _VCLUSTER_BASEDIR_KEY = "vcluster-basedir"
 _ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates"
 
-# The path of an optional JSON Patch file (as per RFC6902) that modifies QA's
+# The constants related to JSON patching (as per RFC6902) that modifies QA's
 # configuration.
-_PATCH_JSON = os.path.join(os.path.dirname(__file__), "qa-patch.json")
+_QA_BASE_PATH = os.path.dirname(__file__)
+_QA_DEFAULT_PATCH = "qa-patch.json"
+_QA_PATCH_DIR = "patch"
+_QA_PATCH_ORDER_FILE = "order"
 
 #: QA configuration (L{_QaConfig})
 _config = None
@@ -254,6 +258,114 @@ class _QaConfig(object):
     #: Cluster-wide run-time value of the exclusive storage flag
     self._exclusive_storage = None
 
+  @staticmethod
+  def LoadPatch(patch_dict, rel_path):
+    """ Loads a single patch.
+
+    @type patch_dict: dict of string to dict
+    @param patch_dict: A dictionary storing patches by relative path.
+    @type rel_path: string
+    @param rel_path: The relative path to the patch, might or might not exist.
+
+    """
+    try:
+      full_path = os.path.join(_QA_BASE_PATH, rel_path)
+      patch = serializer.LoadJson(utils.ReadFile(full_path))
+      patch_dict[rel_path] = patch
+    except IOError:
+      pass
+
+  @staticmethod
+  def LoadPatches():
+    """ Finds and loads all patches supported by the QA.
+
+    @rtype: dict of string to dict
+    @return: A dictionary of relative path to patch content.
+
+    """
+    patches = {}
+    _QaConfig.LoadPatch(patches, _QA_DEFAULT_PATCH)
+    patch_dir_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR)
+    if os.path.exists(patch_dir_path):
+      for filename in os.listdir(patch_dir_path):
+        if filename.endswith(".json"):
+          _QaConfig.LoadPatch(patches, os.path.join(_QA_PATCH_DIR, filename))
+    return patches
+
+  @staticmethod
+  def ApplyPatch(data, patch_module, patches, patch_path):
+    """Applies a single patch.
+
+    @type data: dict (deserialized json)
+    @param data: The QA configuration
+    @type patch_module: module
+    @param patch_module: The json patch module, loaded dynamically
+    @type patches: dict of string to dict
+    @param patches: The dictionary of patch path to content
+    @type patch_path: string
+    @param patch_path: The path to the patch, relative to the QA directory
+
+    @return: The modified configuration data.
+
+    """
+    patch_content = patches[patch_path]
+    print qa_logging.FormatInfo("Applying patch %s" % patch_path)
+    if not patch_content and patch_path != _QA_DEFAULT_PATCH:
+      print qa_logging.FormatWarning("The patch %s added by the user is empty" %
+                                     patch_path)
+    data = patch_module.apply_patch(data, patch_content)
+
+  @staticmethod
+  def ApplyPatches(data, patch_module, patches):
+    """Applies any patches present, and returns the modified QA configuration.
+
+    First, patches from the patch directory are applied. They are ordered
+    alphabetically, unless there is an ``order`` file present - any patches
+    listed within are applied in that order, and any remaining ones in
+    alphabetical order again. Finally, the default patch residing in the
+    top-level QA directory is applied.
+
+    @type data: dict (deserialized json)
+    @param data: The QA configuration
+    @type patch_module: module
+    @param patch_module: The json patch module, loaded dynamically
+    @type patches: dict of string to dict
+    @param patches: The dictionary of patch path to content
+
+    @return: The modified configuration data.
+
+    """
+    ordered_patches = []
+    order_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR,
+                              _QA_PATCH_ORDER_FILE)
+    if os.path.exists(order_path):
+      order_file = open(order_path, 'r')
+      ordered_patches = order_file.read().splitlines()
+      # Removes empty lines
+      ordered_patches = filter(None, ordered_patches)
+
+    # Add the patch dir
+    ordered_patches = map(lambda x: os.path.join(_QA_PATCH_DIR, x),
+                          ordered_patches)
+
+    # First the ordered patches
+    for patch in ordered_patches:
+      if patch not in patches:
+        raise qa_error.Error("Patch %s specified in the ordering file does not "
+                             "exist" % patch)
+      _QaConfig.ApplyPatch(data, patch_module, patches, patch)
+
+    # Then the other non-default ones
+    for patch in sorted(patches):
+      if patch != _QA_DEFAULT_PATCH and patch not in ordered_patches:
+        _QaConfig.ApplyPatch(data, patch_module, patches, patch)
+
+    # Finally the default one
+    if _QA_DEFAULT_PATCH in patches:
+      _QaConfig.ApplyPatch(data, patch_module, patches, _QA_DEFAULT_PATCH)
+
+    return data
+
   @classmethod
   def Load(cls, filename):
     """Loads a configuration file and produces a configuration object.
@@ -268,16 +380,17 @@ class _QaConfig(object):
     # Patch the document using JSON Patch (RFC6902) in file _PATCH_JSON, if
     # available
     try:
-      patch = serializer.LoadJson(utils.ReadFile(_PATCH_JSON))
-      if patch:
+      patches = _QaConfig.LoadPatches()
+      # Try to use the module only if there is a non-empty patch present
+      if any(patches.values()):
         mod = __import__("jsonpatch", fromlist=[])
-        data = mod.apply_patch(data, patch)
+        data = _QaConfig.ApplyPatches(data, mod, patches)
     except IOError:
       pass
     except ImportError:
-      raise qa_error.Error("If you want to use the QA JSON patching feature,"
-                           " you need to install Python modules"
-                           " 'jsonpatch' and 'jsonpointer'.")
+      raise qa_error.Error("For the QA JSON patching feature to work, you "
+                           "need to install Python modules 'jsonpatch' and "
+                           "'jsonpointer'.")
 
     result = cls(dict(map(_ConvertResources,
                           data.items()))) # pylint: disable=E1103

qa/qa_logging.py

+#
+#
+
+# Copyright (C) 2014 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+""" Handles the logging of messages with appropriate coloring.
+
+"""
+
+import sys
+
+
+_INFO_SEQ = None
+_WARNING_SEQ = None
+_ERROR_SEQ = None
+_RESET_SEQ = None
+
+
+def _SetupColours():
+  """Initializes the colour constants.
+
+  """
+  # pylint: disable=W0603
+  # due to global usage
+  global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
+
+  # Don't use colours if stdout isn't a terminal
+  if not sys.stdout.isatty():
+    return
+
+  try:
+    import curses
+  except ImportError:
+    # Don't use colours if curses module can't be imported
+    return
+
+  curses.setupterm()
+
+  _RESET_SEQ = curses.tigetstr("op")
+
+  setaf = curses.tigetstr("setaf")
+  _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
+  _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
+  _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
+
+
+_SetupColours()
+
+
+def _FormatWithColor(text, seq):
+  if not seq:
+    return text
+  return "%s%s%s" % (seq, text, _RESET_SEQ)
+
+
+FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
+FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
+FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)

qa/qa_rapi.py

@@ -44,8 +44,9 @@ import ganeti.rapi.client        # pylint: disable=W0611
 import ganeti.rapi.client_utils
 
 import qa_config
-import qa_utils
 import qa_error
+import qa_logging
+import qa_utils
 
 from qa_instance import IsFailoverSupported
 from qa_instance import IsMigrationSupported
@@ -91,7 +92,7 @@ def Setup(username, password):
 
   if qa_config.UseVirtualCluster():
     # TODO: Implement full support for RAPI on virtual clusters
-    print qa_utils.FormatWarning("RAPI tests are not yet supported on"
+    print qa_logging.FormatWarning("RAPI tests are not yet supported on"
                                    " virtual clusters and will be disabled")
 
     assert _rapi_client is None
@@ -715,7 +716,7 @@ def TestRapiInstanceRemove(instance, use_client):
 def TestRapiInstanceMigrate(instance):
   """Test migrating instance via RAPI"""
   if not IsMigrationSupported(instance):
-    print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
+    print qa_logging.FormatInfo("Instance doesn't support migration, skipping"
                                 " test")
     return
   # Move to secondary node
@@ -729,7 +730,7 @@ def TestRapiInstanceMigrate(instance):
 def TestRapiInstanceFailover(instance):
   """Test failing over instance via RAPI"""
   if not IsFailoverSupported(instance):
-    print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
+    print qa_logging.FormatInfo("Instance doesn't support failover, skipping"
                                 " test")
     return
   # Move to secondary node
@@ -770,7 +771,7 @@ def TestRapiInstanceRenameAndBack(rename_source, rename_target):
 def TestRapiInstanceReinstall(instance):
   """Test reinstalling an instance via RAPI"""
   if instance.disk_template == constants.DT_DISKLESS:
-    print qa_utils.FormatInfo("Test not supported for diskless instances")
+    print qa_logging.FormatInfo("Test not supported for diskless instances")
     return
 
   _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name))
@@ -786,7 +787,7 @@ def TestRapiInstanceReinstall(instance):
 def TestRapiInstanceReplaceDisks(instance):
   """Test replacing instance disks via RAPI"""
   if not IsDiskReplacingSupported(instance):
-    print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
+    print qa_logging.FormatInfo("Instance doesn't support disk replacing,"
                                 " skipping test")
     return
   fn = _rapi_client.ReplaceInstanceDisks

qa/qa_utils.py

@@ -48,11 +48,8 @@ from ganeti import vcluster
 import qa_config
 import qa_error
 
+from qa_logging import FormatInfo
 
-_INFO_SEQ = None
-_WARNING_SEQ = None
-_ERROR_SEQ = None
-_RESET_SEQ = None
 
 _MULTIPLEXERS = {}
 
@@ -70,37 +67,6 @@ _QA_OUTPUT = pathutils.GetLogFilename("qa-output")
  RETURN_VALUE) = range(1000, 1002)
 
 
-def _SetupColours():
-  """Initializes the colour constants.
-
-  """
-  # pylint: disable=W0603
-  # due to global usage
-  global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
-
-  # Don't use colours if stdout isn't a terminal
-  if not sys.stdout.isatty():
-    return
-
-  try:
-    import curses
-  except ImportError:
-    # Don't use colours if curses module can't be imported
-    return
-
-  curses.setupterm()
-
-  _RESET_SEQ = curses.tigetstr("op")
-
-  setaf = curses.tigetstr("setaf")
-  _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
-  _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
-  _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
-
-
-_SetupColours()
-
-
 def AssertIn(item, sequence):
   """Raises an error when item is not in sequence.
 
@@ -577,17 +543,6 @@ def GenericQueryFieldsTest(cmd, fields):
               constants.EXIT_UNKNOWN_FIELD)
 
 
-def _FormatWithColor(text, seq):
-  if not seq:
-    return text
-  return "%s%s%s" % (seq, text, _RESET_SEQ)
-
-
-FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
-FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
-FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
-
-
 def AddToEtcHosts(hostnames):
   """Adds hostnames to /etc/hosts.