Commit ece21e0b authored by Klaus Aehlig's avatar Klaus Aehlig
Browse files

Merge branch 'stable-2.11' into master



* stable-2.11
  Rearranging entries in NEWS file
  Prepare NEWS file for 2.11.0 rc1
  Bump version to 2.11~rc1 in configure.ac

* stable-2.10
  Bump version to 2.10.2
  Prepare NEWS file for 2.10.2
  Allow releases scheduled 5 days in advance

* stable-2.9
  Make watcher submit queries low priority

* stable-2.8
  Include qa/patch in Makefile
  Handle empty patches better
  Move message formatting functions to separate file
  Add optional ordering of QA patch files
  Allow multiple QA patches
  Refactor current patching code

Conflicts:
	NEWS: take all additions
	configure.ac: ignore revision/suffix bump
	qa/qa_rapi.py: trivial
	qa/qa_utils.py: trivial
Signed-off-by: default avatarKlaus Aehlig <aehlig@google.com>
Reviewed-by: default avatarHrvoje Ribicic <riba@google.com>
parents 937ff984 df376ffa
......@@ -192,6 +192,7 @@ DIRS = \
lib/watcher \
man \
qa \
qa/patch \
test \
test/data \
test/data/bdev-rbd \
......@@ -1056,6 +1057,7 @@ qa_scripts = \
qa/qa_job.py \
qa/qa_job_utils.py \
qa/qa_monitoring.py \
qa/qa_logging.py \
qa/qa_node.py \
qa/qa_os.py \
qa/qa_rapi.py \
......
......@@ -33,10 +33,10 @@ New features
opportunistic locking when an iallocator is used.
Version 2.11.0 beta1
--------------------
Version 2.11.0 rc1
------------------
*(Released Wed, 5 Mar 2014)*
*(Released Thu, 20 Mar 2014)*
Incompatible/important changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
......@@ -82,6 +82,7 @@ New features
New dependencies
~~~~~~~~~~~~~~~~
The following new dependencies have been added:
For Haskell:
......@@ -95,6 +96,48 @@ For Haskell:
- ``lens`` library (http://hackage.haskell.org/package/lens)
Since 2.11.0 beta1
~~~~~~~~~~~~~~~~~~
This was the first RC release of the 2.11 series. Since 2.11.0 beta1:
- Convert int to float when checking config. consistency
- Rename compression option in gnt-backup export
Inherited from the 2.9 branch:
- Fix error introduced during merge
- gnt-cluster copyfile: accept relative paths
Inherited from the 2.8 branch:
- Improve RAPI detection of the watcher
- Add patching QA configuration files on buildbots
- Enable a timeout for instance shutdown
- Allow KVM commands to have a timeout
- Allow xen commands to have a timeout
- Fix wrong docstring
Version 2.11.0 beta1
--------------------
*(Released Wed, 5 Mar 2014)*
This was the first beta release of the 2.11 series. All important changes
are listed in the latest 2.11 entry.
Version 2.10.2
--------------
*(Released Mon, 24 Mar 2014)*
- Fix conflict between virtio + spice or soundhw (issue 757)
- accept relative paths in gnt-cluster copyfile (issue 754)
- Introduce shutdown timeout for 'xm shutdown' command
- Improve RAPI detection of the watcher (issue 752)
Version 2.10.1
--------------
......
......@@ -42,7 +42,7 @@ UNRELEASED_RE = re.compile(r"^\*\(unreleased\)\*$")
VERSION_RE = re.compile(r"^Version (\d+(\.\d+)+( (alpha|beta|rc)\d+)?)$")
#: How many days release timestamps may be in the future
TIMESTAMP_FUTURE_DAYS_MAX = 3
TIMESTAMP_FUTURE_DAYS_MAX = 5
errors = []
......
......@@ -140,29 +140,36 @@ def _GenerateDeviceKVMId(dev_type, dev):
return "%s-%s-pci-%d" % (dev_type.lower(), dev.uuid.split("-")[0], dev.pci)
def _UpdatePCISlots(dev, pci_reservations):
"""Update pci configuration for a stopped instance
def _GetFreeSlot(slots, slot=None, reserve=False):
"""Helper method to get first available slot in a bitarray
@type slots: bitarray
@param slots: the bitarray to operate on
@type slot: integer
@param slot: if given we check whether the slot is free
@type reserve: boolean
@param reserve: whether to reserve the first available slot or not
@return: the idx of the (first) available slot
@raise errors.HotplugError: If all slots in a bitarray are occupied
or the given slot is not free.
If dev has a pci slot then reserve it, else find first available
in pci_reservations bitarray. It acts on the same objects passed
as params so there is no need to return anything.
"""
if slot is not None:
assert slot < len(slots)
if slots[slot]:
raise errors.HypervisorError("Slots %d occupied" % slot)
@type dev: L{objects.Disk} or L{objects.NIC}
@param dev: the device object for which we update its pci slot
@type pci_reservations: bitarray
@param pci_reservations: existing pci reservations for an instance
@raise errors.HotplugError: in case an instance has all its slot occupied
else:
avail = slots.search(_AVAILABLE_PCI_SLOT, 1)
if not avail:
raise errors.HypervisorError("All slots occupied")
"""
if dev.pci:
free = dev.pci
else: # pylint: disable=E1103
[free] = pci_reservations.search(_AVAILABLE_PCI_SLOT, 1)
if not free:
raise errors.HypervisorError("All PCI slots occupied")
dev.pci = int(free)
slot = int(avail[0])
pci_reservations[free] = True
if reserve:
slots[slot] = True
return slot
def _GetExistingDeviceInfo(dev_type, device, runtime):
......@@ -800,7 +807,9 @@ class KVMHypervisor(hv_base.BaseHypervisor):
re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
_INFO_VERSION_CMD = "info version"
_DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
# Slot 0 for Host bridge, Slot 1 for ISA bridge, Slot 2 for VGA controller
_DEFAULT_PCI_RESERVATIONS = "11100000000000000000000000000000"
_SOUNDHW_WITH_PCI_SLOT = ["ac97", "es1370", "hda"]
ANCILLARY_FILES = [
_KVM_NETWORK_SCRIPT,
......@@ -1463,7 +1472,23 @@ class KVMHypervisor(hv_base.BaseHypervisor):
kvm_cmd.extend(["-smp", ",".join(smp_list)])
kvm_cmd.extend(["-pidfile", pidfile])
kvm_cmd.extend(["-balloon", "virtio"])
pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
# As requested by music lovers
if hvp[constants.HV_SOUNDHW]:
soundhw = hvp[constants.HV_SOUNDHW]
# For some reason only few sound devices require a PCI slot
# while the Audio controller *must* be in slot 3.
# That's why we bridge this option early in command line
if soundhw in self._SOUNDHW_WITH_PCI_SLOT:
_ = _GetFreeSlot(pci_reservations, reserve=True)
kvm_cmd.extend(["-soundhw", soundhw])
# Add id to ballon and place to the first available slot (3 or 4)
addr = _GetFreeSlot(pci_reservations, reserve=True)
pci_info = ",bus=pci.0,addr=%s" % hex(addr)
kvm_cmd.extend(["-balloon", "virtio,id=balloon%s" % pci_info])
kvm_cmd.extend(["-daemonize"])
if not instance.hvparams[constants.HV_ACPI]:
kvm_cmd.extend(["-no-acpi"])
......@@ -1699,7 +1724,9 @@ class KVMHypervisor(hv_base.BaseHypervisor):
else:
# Enable the spice agent communication channel between the host and the
# agent.
kvm_cmd.extend(["-device", "virtio-serial-pci"])
addr = _GetFreeSlot(pci_reservations, reserve=True)
pci_info = ",bus=pci.0,addr=%s" % hex(addr)
kvm_cmd.extend(["-device", "virtio-serial-pci,id=spice%s" % pci_info])
kvm_cmd.extend([
"-device",
"virtserialport,chardev=spicechannel0,name=com.redhat.spice.0",
......@@ -1727,10 +1754,6 @@ class KVMHypervisor(hv_base.BaseHypervisor):
if hvp[constants.HV_CPU_TYPE]:
kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]])
# As requested by music lovers
if hvp[constants.HV_SOUNDHW]:
kvm_cmd.extend(["-soundhw", hvp[constants.HV_SOUNDHW]])
# Pass a -vga option if requested, or if spice is used, for backwards
# compatibility.
if hvp[constants.HV_VGA]:
......@@ -1750,15 +1773,14 @@ class KVMHypervisor(hv_base.BaseHypervisor):
if hvp[constants.HV_KVM_EXTRA]:
kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
kvm_disks = []
for disk, link_name, uri in block_devices:
_UpdatePCISlots(disk, pci_reservations)
disk.pci = _GetFreeSlot(pci_reservations, disk.pci, True)
kvm_disks.append((disk, link_name, uri))
kvm_nics = []
for nic in instance.nics:
_UpdatePCISlots(nic, pci_reservations)
nic.pci = _GetFreeSlot(pci_reservations, nic.pci, True)
kvm_nics.append(nic)
hvparams = hvp
......@@ -2179,11 +2201,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
slot = int(match.group(1))
slots[slot] = True
[free] = slots.search(_AVAILABLE_PCI_SLOT, 1) # pylint: disable=E1101
if not free:
raise errors.HypervisorError("All PCI slots occupied")
dev.pci = int(free)
dev.pci = _GetFreeSlot(slots)
def VerifyHotplugSupport(self, instance, action, dev_type):
"""Verifies that hotplug is supported.
......
......@@ -313,7 +313,8 @@ def _VerifyDisks(cl, uuid, nodes, instances):
"""Run a per-group "gnt-cluster verify-disks".
"""
job_id = cl.SubmitJob([opcodes.OpGroupVerifyDisks(group_name=uuid)])
job_id = cl.SubmitJob([opcodes.OpGroupVerifyDisks(
group_name=uuid, priority=constants.OP_PRIO_LOW)])
((_, offline_disk_instances, _), ) = \
cli.PollJob(job_id, cl=cl, feedback_fn=logging.debug)
cl.ArchiveJob(job_id)
......
......@@ -35,9 +35,10 @@ from ganeti import pathutils
import qa_config
import qa_daemon
import qa_utils
import qa_error
import qa_instance
import qa_logging
import qa_utils
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
......@@ -287,7 +288,7 @@ def TestClusterRename():
original_name = qa_config.get("name")
rename_target = qa_config.get("rename", None)
if rename_target is None:
print qa_utils.FormatError('"rename" entry is missing')
print qa_logging.FormatError('"rename" entry is missing')
return
for data in [
......
......@@ -32,6 +32,7 @@ from ganeti import compat
from ganeti import ht
import qa_error
import qa_logging
_INSTANCE_CHECK_KEY = "instance-check"
......@@ -40,9 +41,12 @@ _VCLUSTER_MASTER_KEY = "vcluster-master"
_VCLUSTER_BASEDIR_KEY = "vcluster-basedir"
_ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates"
# The path of an optional JSON Patch file (as per RFC6902) that modifies QA's
# The constants related to JSON patching (as per RFC6902) that modifies QA's
# configuration.
_PATCH_JSON = os.path.join(os.path.dirname(__file__), "qa-patch.json")
_QA_BASE_PATH = os.path.dirname(__file__)
_QA_DEFAULT_PATCH = "qa-patch.json"
_QA_PATCH_DIR = "patch"
_QA_PATCH_ORDER_FILE = "order"
#: QA configuration (L{_QaConfig})
_config = None
......@@ -254,6 +258,114 @@ class _QaConfig(object):
#: Cluster-wide run-time value of the exclusive storage flag
self._exclusive_storage = None
@staticmethod
def LoadPatch(patch_dict, rel_path):
""" Loads a single patch.
@type patch_dict: dict of string to dict
@param patch_dict: A dictionary storing patches by relative path.
@type rel_path: string
@param rel_path: The relative path to the patch, might or might not exist.
"""
try:
full_path = os.path.join(_QA_BASE_PATH, rel_path)
patch = serializer.LoadJson(utils.ReadFile(full_path))
patch_dict[rel_path] = patch
except IOError:
pass
@staticmethod
def LoadPatches():
""" Finds and loads all patches supported by the QA.
@rtype: dict of string to dict
@return: A dictionary of relative path to patch content.
"""
patches = {}
_QaConfig.LoadPatch(patches, _QA_DEFAULT_PATCH)
patch_dir_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR)
if os.path.exists(patch_dir_path):
for filename in os.listdir(patch_dir_path):
if filename.endswith(".json"):
_QaConfig.LoadPatch(patches, os.path.join(_QA_PATCH_DIR, filename))
return patches
@staticmethod
def ApplyPatch(data, patch_module, patches, patch_path):
"""Applies a single patch.
@type data: dict (deserialized json)
@param data: The QA configuration
@type patch_module: module
@param patch_module: The json patch module, loaded dynamically
@type patches: dict of string to dict
@param patches: The dictionary of patch path to content
@type patch_path: string
@param patch_path: The path to the patch, relative to the QA directory
@return: The modified configuration data.
"""
patch_content = patches[patch_path]
print qa_logging.FormatInfo("Applying patch %s" % patch_path)
if not patch_content and patch_path != _QA_DEFAULT_PATCH:
print qa_logging.FormatWarning("The patch %s added by the user is empty" %
patch_path)
data = patch_module.apply_patch(data, patch_content)
@staticmethod
def ApplyPatches(data, patch_module, patches):
"""Applies any patches present, and returns the modified QA configuration.
First, patches from the patch directory are applied. They are ordered
alphabetically, unless there is an ``order`` file present - any patches
listed within are applied in that order, and any remaining ones in
alphabetical order again. Finally, the default patch residing in the
top-level QA directory is applied.
@type data: dict (deserialized json)
@param data: The QA configuration
@type patch_module: module
@param patch_module: The json patch module, loaded dynamically
@type patches: dict of string to dict
@param patches: The dictionary of patch path to content
@return: The modified configuration data.
"""
ordered_patches = []
order_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR,
_QA_PATCH_ORDER_FILE)
if os.path.exists(order_path):
order_file = open(order_path, 'r')
ordered_patches = order_file.read().splitlines()
# Removes empty lines
ordered_patches = filter(None, ordered_patches)
# Add the patch dir
ordered_patches = map(lambda x: os.path.join(_QA_PATCH_DIR, x),
ordered_patches)
# First the ordered patches
for patch in ordered_patches:
if patch not in patches:
raise qa_error.Error("Patch %s specified in the ordering file does not "
"exist" % patch)
_QaConfig.ApplyPatch(data, patch_module, patches, patch)
# Then the other non-default ones
for patch in sorted(patches):
if patch != _QA_DEFAULT_PATCH and patch not in ordered_patches:
_QaConfig.ApplyPatch(data, patch_module, patches, patch)
# Finally the default one
if _QA_DEFAULT_PATCH in patches:
_QaConfig.ApplyPatch(data, patch_module, patches, _QA_DEFAULT_PATCH)
return data
@classmethod
def Load(cls, filename):
"""Loads a configuration file and produces a configuration object.
......@@ -268,16 +380,17 @@ class _QaConfig(object):
# Patch the document using JSON Patch (RFC6902) in file _PATCH_JSON, if
# available
try:
patch = serializer.LoadJson(utils.ReadFile(_PATCH_JSON))
if patch:
patches = _QaConfig.LoadPatches()
# Try to use the module only if there is a non-empty patch present
if any(patches.values()):
mod = __import__("jsonpatch", fromlist=[])
data = mod.apply_patch(data, patch)
data = _QaConfig.ApplyPatches(data, mod, patches)
except IOError:
pass
except ImportError:
raise qa_error.Error("If you want to use the QA JSON patching feature,"
" you need to install Python modules"
" 'jsonpatch' and 'jsonpointer'.")
raise qa_error.Error("For the QA JSON patching feature to work, you "
"need to install Python modules 'jsonpatch' and "
"'jsonpointer'.")
result = cls(dict(map(_ConvertResources,
data.items()))) # pylint: disable=E1103
......
#
#
# Copyright (C) 2014 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
""" Handles the logging of messages with appropriate coloring.
"""
import sys
_INFO_SEQ = None
_WARNING_SEQ = None
_ERROR_SEQ = None
_RESET_SEQ = None
def _SetupColours():
"""Initializes the colour constants.
"""
# pylint: disable=W0603
# due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
if not sys.stdout.isatty():
return
try:
import curses
except ImportError:
# Don't use colours if curses module can't be imported
return
curses.setupterm()
_RESET_SEQ = curses.tigetstr("op")
setaf = curses.tigetstr("setaf")
_INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
_WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
_ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
_SetupColours()
def _FormatWithColor(text, seq):
if not seq:
return text
return "%s%s%s" % (seq, text, _RESET_SEQ)
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
......@@ -47,6 +47,7 @@ import ganeti.rapi.client_utils
import qa_config
import qa_error
import qa_logging
import qa_utils
from qa_instance import IsDiskReplacingSupported
......@@ -94,8 +95,8 @@ def Setup(username, password):
if qa_config.UseVirtualCluster():
# TODO: Implement full support for RAPI on virtual clusters
print qa_utils.FormatWarning("RAPI tests are not yet supported on"
" virtual clusters and will be disabled")
print qa_logging.FormatWarning("RAPI tests are not yet supported on"
" virtual clusters and will be disabled")
assert _rapi_client is None
else:
......@@ -751,8 +752,8 @@ def TestRapiInstanceRemove(instance, use_client):
def TestRapiInstanceMigrate(instance):
"""Test migrating instance via RAPI"""
if not IsMigrationSupported(instance):
print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
" test")
print qa_logging.FormatInfo("Instance doesn't support migration, skipping"
" test")
return
# Move to secondary node
_WaitForRapiJob(_rapi_client.MigrateInstance(instance.name))
......@@ -765,8 +766,8 @@ def TestRapiInstanceMigrate(instance):
def TestRapiInstanceFailover(instance):
"""Test failing over instance via RAPI"""
if not IsFailoverSupported(instance):
print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
" test")
print qa_logging.FormatInfo("Instance doesn't support failover, skipping"
" test")
return
# Move to secondary node
_WaitForRapiJob(_rapi_client.FailoverInstance(instance.name))
......@@ -806,7 +807,7 @@ def TestRapiInstanceRenameAndBack(rename_source, rename_target):
def TestRapiInstanceReinstall(instance):
"""Test reinstalling an instance via RAPI"""
if instance.disk_template == constants.DT_DISKLESS:
print qa_utils.FormatInfo("Test not supported for diskless instances")
print qa_logging.FormatInfo("Test not supported for diskless instances")
return
_WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name))
......@@ -822,8 +823,8 @@ def TestRapiInstanceReinstall(instance):
def TestRapiInstanceReplaceDisks(instance):
"""Test replacing instance disks via RAPI"""
if not IsDiskReplacingSupported(instance):
print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
" skipping test")
print qa_logging.FormatInfo("Instance doesn't support disk replacing,"
" skipping test")
return
fn = _rapi_client.ReplaceInstanceDisks
_WaitForRapiJob(fn(instance.name,
......
......@@ -50,11 +50,8 @@ import colors
import qa_config
import qa_error
from qa_logging import FormatInfo
_INFO_SEQ = None
_WARNING_SEQ = None
_ERROR_SEQ = None
_RESET_SEQ = None
_MULTIPLEXERS = {}
......@@ -72,41 +69,6 @@ _QA_OUTPUT = pathutils.GetLogFilename("qa-output")
RETURN_VALUE) = range(1000, 1002)
def _SetupColours():
"""Initializes the colour constants.
"""
# pylint: disable=W0603
# due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
if not sys.stdout.isatty():
return
try:
import curses
except ImportError:
# Don't use colours if curses module can't be imported
return
try:
curses.setupterm()
except curses.error:
# Probably a non-standard terminal, don't use colours then
return
_RESET_SEQ = curses.tigetstr("op")
setaf = curses.tigetstr("setaf")
_INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
_WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
_ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
_SetupColours()
def AssertIn(item, sequence, msg=""):
"""Raises an error when item is not in sequence.
......@@ -600,17 +562,6 @@ def GenericQueryFieldsTest(cmd, fields):
constants.EXIT_UNKNOWN_FIELD)
def _FormatWithColor(text, seq):
if not seq:
return text
return "%s%s%s" % (seq, text, _RESET_SEQ)
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)