Commit ade40547 authored by Thomas Thrainer's avatar Thomas Thrainer

Merge branch 'stable-2.8' into stable-2.9

* stable-2.8
  Include qa/patch in Makefile
  Handle empty patches better
  Move message formatting functions to separate file
  Add optional ordering of QA patch files
  Allow multiple QA patches
  Refactor current patching code

Conflicts:
	Makefile.am (include all added files)
	qa/qa_cluster.py (trivial)
Signed-off-by: default avatarThomas Thrainer <thomasth@google.com>
Reviewed-by: default avatarHelga Velroyen <helgav@google.com>
parents ddc64582 7d76de75
......@@ -130,6 +130,7 @@ DIRS = \
lib/watcher \
man \
qa \
qa/patch \
test \
test/data \
test/data/bdev-rbd \
......@@ -849,6 +850,7 @@ qa_scripts = \
qa/qa_instance_utils.py \
qa/qa_job.py \
qa/qa_monitoring.py \
qa/qa_logging.py \
qa/qa_node.py \
qa/qa_os.py \
qa/qa_rapi.py \
......
......@@ -34,9 +34,10 @@ from ganeti import pathutils
import qa_config
import qa_daemon
import qa_utils
import qa_error
import qa_instance
import qa_logging
import qa_utils
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
......@@ -265,7 +266,7 @@ def TestClusterRename():
original_name = qa_config.get("name")
rename_target = qa_config.get("rename", None)
if rename_target is None:
print qa_utils.FormatError('"rename" entry is missing')
print qa_logging.FormatError('"rename" entry is missing')
return
for data in [
......
......@@ -32,6 +32,7 @@ from ganeti import compat
from ganeti import ht
import qa_error
import qa_logging
_INSTANCE_CHECK_KEY = "instance-check"
......@@ -40,9 +41,12 @@ _VCLUSTER_MASTER_KEY = "vcluster-master"
_VCLUSTER_BASEDIR_KEY = "vcluster-basedir"
_ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates"
# The path of an optional JSON Patch file (as per RFC6902) that modifies QA's
# The constants related to JSON patching (as per RFC6902) that modifies QA's
# configuration.
_PATCH_JSON = os.path.join(os.path.dirname(__file__), "qa-patch.json")
_QA_BASE_PATH = os.path.dirname(__file__)
_QA_DEFAULT_PATCH = "qa-patch.json"
_QA_PATCH_DIR = "patch"
_QA_PATCH_ORDER_FILE = "order"
#: QA configuration (L{_QaConfig})
_config = None
......@@ -254,6 +258,114 @@ class _QaConfig(object):
#: Cluster-wide run-time value of the exclusive storage flag
self._exclusive_storage = None
@staticmethod
def LoadPatch(patch_dict, rel_path):
""" Loads a single patch.
@type patch_dict: dict of string to dict
@param patch_dict: A dictionary storing patches by relative path.
@type rel_path: string
@param rel_path: The relative path to the patch, might or might not exist.
"""
try:
full_path = os.path.join(_QA_BASE_PATH, rel_path)
patch = serializer.LoadJson(utils.ReadFile(full_path))
patch_dict[rel_path] = patch
except IOError:
pass
@staticmethod
def LoadPatches():
""" Finds and loads all patches supported by the QA.
@rtype: dict of string to dict
@return: A dictionary of relative path to patch content.
"""
patches = {}
_QaConfig.LoadPatch(patches, _QA_DEFAULT_PATCH)
patch_dir_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR)
if os.path.exists(patch_dir_path):
for filename in os.listdir(patch_dir_path):
if filename.endswith(".json"):
_QaConfig.LoadPatch(patches, os.path.join(_QA_PATCH_DIR, filename))
return patches
@staticmethod
def ApplyPatch(data, patch_module, patches, patch_path):
"""Applies a single patch.
@type data: dict (deserialized json)
@param data: The QA configuration
@type patch_module: module
@param patch_module: The json patch module, loaded dynamically
@type patches: dict of string to dict
@param patches: The dictionary of patch path to content
@type patch_path: string
@param patch_path: The path to the patch, relative to the QA directory
@return: The modified configuration data.
"""
patch_content = patches[patch_path]
print qa_logging.FormatInfo("Applying patch %s" % patch_path)
if not patch_content and patch_path != _QA_DEFAULT_PATCH:
print qa_logging.FormatWarning("The patch %s added by the user is empty" %
patch_path)
data = patch_module.apply_patch(data, patch_content)
@staticmethod
def ApplyPatches(data, patch_module, patches):
"""Applies any patches present, and returns the modified QA configuration.
First, patches from the patch directory are applied. They are ordered
alphabetically, unless there is an ``order`` file present - any patches
listed within are applied in that order, and any remaining ones in
alphabetical order again. Finally, the default patch residing in the
top-level QA directory is applied.
@type data: dict (deserialized json)
@param data: The QA configuration
@type patch_module: module
@param patch_module: The json patch module, loaded dynamically
@type patches: dict of string to dict
@param patches: The dictionary of patch path to content
@return: The modified configuration data.
"""
ordered_patches = []
order_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR,
_QA_PATCH_ORDER_FILE)
if os.path.exists(order_path):
order_file = open(order_path, 'r')
ordered_patches = order_file.read().splitlines()
# Removes empty lines
ordered_patches = filter(None, ordered_patches)
# Add the patch dir
ordered_patches = map(lambda x: os.path.join(_QA_PATCH_DIR, x),
ordered_patches)
# First the ordered patches
for patch in ordered_patches:
if patch not in patches:
raise qa_error.Error("Patch %s specified in the ordering file does not "
"exist" % patch)
_QaConfig.ApplyPatch(data, patch_module, patches, patch)
# Then the other non-default ones
for patch in sorted(patches):
if patch != _QA_DEFAULT_PATCH and patch not in ordered_patches:
_QaConfig.ApplyPatch(data, patch_module, patches, patch)
# Finally the default one
if _QA_DEFAULT_PATCH in patches:
_QaConfig.ApplyPatch(data, patch_module, patches, _QA_DEFAULT_PATCH)
return data
@classmethod
def Load(cls, filename):
"""Loads a configuration file and produces a configuration object.
......@@ -268,16 +380,17 @@ class _QaConfig(object):
# Patch the document using JSON Patch (RFC6902) in file _PATCH_JSON, if
# available
try:
patch = serializer.LoadJson(utils.ReadFile(_PATCH_JSON))
if patch:
patches = _QaConfig.LoadPatches()
# Try to use the module only if there is a non-empty patch present
if any(patches.values()):
mod = __import__("jsonpatch", fromlist=[])
data = mod.apply_patch(data, patch)
data = _QaConfig.ApplyPatches(data, mod, patches)
except IOError:
pass
except ImportError:
raise qa_error.Error("If you want to use the QA JSON patching feature,"
" you need to install Python modules"
" 'jsonpatch' and 'jsonpointer'.")
raise qa_error.Error("For the QA JSON patching feature to work, you "
"need to install Python modules 'jsonpatch' and "
"'jsonpointer'.")
result = cls(dict(map(_ConvertResources,
data.items()))) # pylint: disable=E1103
......
#
#
# Copyright (C) 2014 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
""" Handles the logging of messages with appropriate coloring.
"""
import sys
_INFO_SEQ = None
_WARNING_SEQ = None
_ERROR_SEQ = None
_RESET_SEQ = None
def _SetupColours():
"""Initializes the colour constants.
"""
# pylint: disable=W0603
# due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
if not sys.stdout.isatty():
return
try:
import curses
except ImportError:
# Don't use colours if curses module can't be imported
return
curses.setupterm()
_RESET_SEQ = curses.tigetstr("op")
setaf = curses.tigetstr("setaf")
_INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
_WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
_ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
_SetupColours()
def _FormatWithColor(text, seq):
if not seq:
return text
return "%s%s%s" % (seq, text, _RESET_SEQ)
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
......@@ -44,8 +44,9 @@ import ganeti.rapi.client # pylint: disable=W0611
import ganeti.rapi.client_utils
import qa_config
import qa_utils
import qa_error
import qa_logging
import qa_utils
from qa_instance import IsFailoverSupported
from qa_instance import IsMigrationSupported
......@@ -91,8 +92,8 @@ def Setup(username, password):
if qa_config.UseVirtualCluster():
# TODO: Implement full support for RAPI on virtual clusters
print qa_utils.FormatWarning("RAPI tests are not yet supported on"
" virtual clusters and will be disabled")
print qa_logging.FormatWarning("RAPI tests are not yet supported on"
" virtual clusters and will be disabled")
assert _rapi_client is None
else:
......@@ -715,8 +716,8 @@ def TestRapiInstanceRemove(instance, use_client):
def TestRapiInstanceMigrate(instance):
"""Test migrating instance via RAPI"""
if not IsMigrationSupported(instance):
print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
" test")
print qa_logging.FormatInfo("Instance doesn't support migration, skipping"
" test")
return
# Move to secondary node
_WaitForRapiJob(_rapi_client.MigrateInstance(instance.name))
......@@ -729,8 +730,8 @@ def TestRapiInstanceMigrate(instance):
def TestRapiInstanceFailover(instance):
"""Test failing over instance via RAPI"""
if not IsFailoverSupported(instance):
print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
" test")
print qa_logging.FormatInfo("Instance doesn't support failover, skipping"
" test")
return
# Move to secondary node
_WaitForRapiJob(_rapi_client.FailoverInstance(instance.name))
......@@ -770,7 +771,7 @@ def TestRapiInstanceRenameAndBack(rename_source, rename_target):
def TestRapiInstanceReinstall(instance):
"""Test reinstalling an instance via RAPI"""
if instance.disk_template == constants.DT_DISKLESS:
print qa_utils.FormatInfo("Test not supported for diskless instances")
print qa_logging.FormatInfo("Test not supported for diskless instances")
return
_WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name))
......@@ -786,8 +787,8 @@ def TestRapiInstanceReinstall(instance):
def TestRapiInstanceReplaceDisks(instance):
"""Test replacing instance disks via RAPI"""
if not IsDiskReplacingSupported(instance):
print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
" skipping test")
print qa_logging.FormatInfo("Instance doesn't support disk replacing,"
" skipping test")
return
fn = _rapi_client.ReplaceInstanceDisks
_WaitForRapiJob(fn(instance.name,
......
......@@ -48,11 +48,8 @@ from ganeti import vcluster
import qa_config
import qa_error
from qa_logging import FormatInfo
_INFO_SEQ = None
_WARNING_SEQ = None
_ERROR_SEQ = None
_RESET_SEQ = None
_MULTIPLEXERS = {}
......@@ -70,37 +67,6 @@ _QA_OUTPUT = pathutils.GetLogFilename("qa-output")
RETURN_VALUE) = range(1000, 1002)
def _SetupColours():
"""Initializes the colour constants.
"""
# pylint: disable=W0603
# due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
if not sys.stdout.isatty():
return
try:
import curses
except ImportError:
# Don't use colours if curses module can't be imported
return
curses.setupterm()
_RESET_SEQ = curses.tigetstr("op")
setaf = curses.tigetstr("setaf")
_INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
_WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
_ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
_SetupColours()
def AssertIn(item, sequence):
"""Raises an error when item is not in sequence.
......@@ -577,17 +543,6 @@ def GenericQueryFieldsTest(cmd, fields):
constants.EXIT_UNKNOWN_FIELD)
def _FormatWithColor(text, seq):
if not seq:
return text
return "%s%s%s" % (seq, text, _RESET_SEQ)
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
def AddToEtcHosts(hostnames):
"""Adds hostnames to /etc/hosts.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment