Commit e69c93fb authored by Hrvoje Ribicic's avatar Hrvoje Ribicic
Browse files

Merge branch 'stable-2.10' into stable-2.11

* stable-2.10
  Fix specification of TIDiskParams
  Add unittests for instance modify parameter renaming
  Add renaming of instance custom params
  Add RAPI symmetry tests for groups
  Extend RAPI symmetry tests with RAPI-only aliases
  Add test for group custom parameter renaming
  Add renaming of group custom ndparams, ipolicy, diskparams
  Add the RAPI symmetry test for nodes
  Add aliases for nodes
  Allow choice of HTTP method for modification
  Add cluster RAPI symmetry test
  Fix failing cluster query test
  Add aliases for cluster parameters
  Add support for value aliases to RAPI
  Provide tests for GET/PUT symmetry
  Sort imports
  Also consider filter fields for deciding if using live data
  Document the python-fdsend dependency

* stable-2.9
  Fix failing gnt-node list-drbd command

* stable-2.8
  Add reason parameter to RAPI client functions
Signed-off-by: default avatarHrvoje Ribicic <>
Reviewed-by: default avatarKlaus Aehlig <>

	lib/rapi/ Add both params
	lib/rapi/ Remove reference to missing LU, add aliases
	src/Ganeti/Query/Query.hs: Take more imports
parents df376ffa 7bb22823
......@@ -43,6 +43,8 @@ Before installing, please verify that you have the following programs:
- `affinity Python module <>`_,
optional python package for supporting CPU pinning under KVM
- `fdsend Python module <>`_,
optional Python package for supporting NIC hotplugging under KVM
- `qemu-img <>`_, if you want to use ``ovfconverter``
- `fping <>`_
- `Python IP address manipulation library
......@@ -168,6 +168,7 @@ New features
- one can not (yet) hotplug a disk using userspace access mode for RBD
- in case of a downgrade instances should suffer a reboot in order to
be migratable (due to core change of runtime files)
- ``python-fdsend`` is required for NIC hotplugging.
Misc changes
......@@ -192,6 +193,8 @@ Python
- The version requirements for ``python-mock`` have increased to at least
version 1.0.1. It is still used for testing only.
- ``python-fdsend`` ( is optional
but required for KVM NIC hotplugging to work.
Since 2.10.0 rc3
......@@ -474,7 +474,7 @@ def _GetHandlerMethods(handler):
return sorted(method
for (method, op_attr, _, _) in rapi.baserlib.OPCODE_ATTRS
for (method, op_attr, _, _, _) in rapi.baserlib.OPCODE_ATTRS
# Only if handler supports method
if hasattr(handler, method) or hasattr(handler, op_attr))
......@@ -57,7 +57,7 @@ def _BuildOpcodeAttributes():
return [(method, "%s_OPCODE" % method, "%s_RENAME" % method,
"Get%sOpInput" % method.capitalize())
"%s_ALIASES" % method, "Get%sOpInput" % method.capitalize())
for method in _SUPPORTED_METHODS]
......@@ -407,7 +407,7 @@ def GetResourceOpcodes(cls):
return frozenset(filter(None, (getattr(cls, op_attr, None)
for (_, op_attr, _, _) in OPCODE_ATTRS)))
for (_, op_attr, _, _, _) in OPCODE_ATTRS)))
def GetHandlerAccess(handler, method):
......@@ -421,6 +421,22 @@ def GetHandlerAccess(handler, method):
return getattr(handler, "%s_ACCESS" % method, None)
def GetHandler(get_fn, aliases):
result = get_fn()
if not isinstance(result, dict) or aliases is None:
return result
for (param, alias) in aliases.items():
if param in result:
if alias in result:
raise http.HttpBadRequest("Parameter '%s' has an alias of '%s', but"
" both values are present in response" %
(param, alias))
result[alias] = result[param]
return result
class _MetaOpcodeResource(type):
"""Meta class for RAPI resources.
......@@ -432,13 +448,23 @@ class _MetaOpcodeResource(type):
# Access to private attributes of a client class, pylint: disable=W0212
obj = type.__call__(mcs, *args, **kwargs)
for (method, op_attr, rename_attr, fn_attr) in OPCODE_ATTRS:
for (method, op_attr, rename_attr, aliases_attr, fn_attr) in OPCODE_ATTRS:
if hasattr(obj, method):
# If the method handler is already defined, "*_RENAME" or "Get*OpInput"
# shouldn't be (they're only used by the automatically generated
# handler)
# If the method handler is already defined, "*_RENAME" or
# "Get*OpInput" shouldn't be (they're only used by the automatically
# generated handler)
assert not hasattr(obj, rename_attr)
assert not hasattr(obj, fn_attr)
# The aliases are allowed only on GET calls
assert not hasattr(obj, aliases_attr) or method == http.HTTP_GET
# GET methods can add aliases of values they return under a different
# name
if method == http.HTTP_GET and hasattr(obj, aliases_attr):
setattr(obj, method,
compat.partial(GetHandler, getattr(obj, method),
getattr(obj, aliases_attr)))
# Try to generate handler method on handler instance
......@@ -474,6 +500,8 @@ class OpcodeResource(ResourceBase):
automatically generate a GET handler submitting the opcode
@cvar GET_RENAME: Set this to rename parameters in the GET handler (see
@cvar GET_ALIASES: Set this to duplicate return values in GET results (see
@ivar GetGetOpInput: Define this to override the default method for
getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData})
This diff is collapsed.
......@@ -220,6 +220,10 @@ class R_2_info(baserlib.OpcodeResource):
GET_OPCODE = opcodes.OpClusterQuery
"volume_group_name": "vg_name",
"drbd_usermode_helper": "drbd_helper",
def GET(self):
"""Returns cluster information.
......@@ -413,6 +417,9 @@ class R_2_nodes_name(baserlib.OpcodeResource):
"""/2/nodes/[node_name] resource.
"sip": "secondary_ip",
def GET(self):
"""Send information about a node.
......@@ -836,6 +843,11 @@ class R_2_groups_name_modify(baserlib.OpcodeResource):
PUT_OPCODE = opcodes.OpGroupSetParams
"custom_ndparams": "ndparams",
"custom_ipolicy": "ipolicy",
"custom_diskparams": "diskparams",
def GetPutOpInput(self):
"""Changes some parameters of node group.
......@@ -1347,6 +1359,11 @@ class R_2_instances_name_modify(baserlib.OpcodeResource):
PUT_OPCODE = opcodes.OpInstanceSetParams
"custom_beparams": "beparams",
"custom_hvparams": "hvparams",
"custom_nicparams": "nicparams",
def GetPutOpInput(self):
"""Changes parameters of an instance.
......@@ -790,7 +790,8 @@ def RunInstanceTests():
RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
for node in inodes:
RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node,
templ == constants.DT_DRBD8)
if len(inodes) > 1:
RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
......@@ -32,7 +32,7 @@ import qa_config
import qa_error
import qa_utils
from qa_utils import AssertCommand, AssertEqual
from qa_utils import AssertCommand, AssertEqual, AssertIn, GetCommandOutput
def NodeAdd(node, readd=False, group=None):
......@@ -452,9 +452,21 @@ def TestNodeListFields():
qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
def TestNodeListDrbd(node):
def TestNodeListDrbd(node, is_drbd):
"""gnt-node list-drbd"""
AssertCommand(["gnt-node", "list-drbd", node.primary])
master = qa_config.GetMasterNode()
result_output = GetCommandOutput(master.primary,
"gnt-node list-drbd --no-header %s" %
# Meaningful to note: there is but one instance, and the node is either the
# primary or one of the secondaries
if is_drbd:
# Invoked for both primary and secondary
drbd_node, _, _, _, _, drbd_peer = result_output.split()
AssertIn(node.primary, [drbd_node, drbd_peer])
# Output should be empty, barring newlines
AssertEqual(result_output.strip(), "")
def _BuildSetESCmd(action, value, node_name):
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -23,22 +23,23 @@
import tempfile
import functools
import itertools
import random
import re
import itertools
import functools
import tempfile
from ganeti import utils
from ganeti import cli
from ganeti import compat
from ganeti import constants
from ganeti import errors
from ganeti import cli
from ganeti import rapi
from ganeti import pathutils
from ganeti import objects
from ganeti import opcodes
from ganeti import query
from ganeti import compat
from ganeti import qlang
from ganeti import pathutils
from ganeti import rapi
from ganeti import utils
from ganeti.http.auth import ParsePasswordFile
import ganeti.rapi.client # pylint: disable=W0611
......@@ -190,6 +191,74 @@ def _DoTests(uris):
return results
# pylint: disable=W0212
# Due to _SendRequest usage
def _DoGetPutTests(get_uri, modify_uri, opcode_params, rapi_only_aliases=None,
modify_method="PUT", exceptions=None, set_exceptions=None):
""" Test if all params of an object can be retrieved, and set as well.
@type get_uri: string
@param get_uri: The URI from which information about the object can be
@type modify_uri: string
@param modify_uri: The URI which can be used to modify the object.
@type opcode_params: list of tuple
@param opcode_params: The parameters of the underlying opcode, used to
determine which parameters are actually present.
@type rapi_only_aliases: list of string or None
@param rapi_only_aliases: Aliases for parameters which differ from the opcode,
and become renamed before opcode submission.
@type modify_method: string
@param modify_method: The method to be used in the modification.
@type exceptions: list of string or None
@param exceptions: The parameters which have not been exposed and should not
be tested at all.
@type set_exceptions: list of string or None
@param set_exceptions: The parameters whose setting should not be tested as a
part of this test.
assert get_uri.startswith("/")
assert modify_uri.startswith("/")
if exceptions is None:
exceptions = []
if set_exceptions is None:
set_exceptions = []
print "Testing get/modify symmetry of %s and %s" % (get_uri, modify_uri)
# First we see if all parameters of the opcode are returned through RAPI
params_of_interest = map(lambda x: x[0], opcode_params)
# The RAPI-specific aliases are to be checked as well
if rapi_only_aliases is not None:
info = _rapi_client._SendRequest("GET", get_uri, None, {})
missing_params = filter(lambda x: x not in info and x not in exceptions,
if missing_params:
raise qa_error.Error("The parameters %s which can be set through the "
"appropriate opcode are not present in the response "
"from %s" % (','.join(missing_params), get_uri))
print "GET successful at %s" % get_uri
# Then if we can perform a set with the same values as received
put_payload = {}
for param in params_of_interest:
if param not in exceptions and param not in set_exceptions:
put_payload[param] = info[param]
_rapi_client._SendRequest(modify_method, modify_uri, None, put_payload)
print "%s successful at %s" % (modify_method, modify_uri)
# pylint: enable=W0212
def _VerifyReturnsJob(data):
if not isinstance(data, int):
AssertMatch(data, r"^\d+$")
......@@ -271,6 +340,21 @@ def TestEmptyCluster():
raise qa_error.Error("Non-implemented method didn't fail")
# Test GET/PUT symmetry
"force", # Standard option
"add_uids", # Modifies UID pool, is not a param itself
"remove_uids", # Same as above
NOT_EXPOSED_YET = ["hv_state", "disk_state", "modify_etc_hosts"]
# The nicparams are returned under the default entry, yet accepted as they
# are - this is a TODO to fix!
DEFAULT_ISSUES = ["nicparams"]
_DoGetPutTests("/2/info", "/2/modify", opcodes.OpClusterSetParams.OP_PARAMS,
def TestRapiQuery():
"""Testing resource queries via remote API.
......@@ -475,6 +559,20 @@ def TestNode(node):
("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
# Not parameters of the node, but controlling opcode behavior
LEGITIMATELY_MISSING = ["force", "powered"]
# Identifying the node - RAPI provides these itself
IDENTIFIERS = ["node_name", "node_uuid"]
# As the name states, these can be set but not retrieved yet
NOT_EXPOSED_YET = ["hv_state", "disk_state", "auto_promote"]
_DoGetPutTests("/2/nodes/%s" % node.primary,
"/2/nodes/%s/modify" % node.primary,
def _FilterTags(seq):
"""Removes unwanted tags from a sequence.
......@@ -596,6 +694,27 @@ def TestRapiNodeGroups():
# Test for get/set symmetry
# Identifying the node - RAPI provides these itself
IDENTIFIERS = ["group_name"]
# As the name states, not exposed yet
NOT_EXPOSED_YET = ["hv_state", "disk_state"]
# The parameters we do not want to get and set (as that sets the
# group-specific params to the filled ones)
FILLED_PARAMS = ["ndparams", "ipolicy", "diskparams"]
# The aliases that we can use to perform this test with the group-specific
# params
CUSTOM_PARAMS = ["custom_ndparams", "custom_ipolicy", "custom_diskparams"]
_DoGetPutTests("/2/groups/%s" % group3, "/2/groups/%s/modify" % group3,
# Delete groups
for group in [group1, group3]:
(job_id, ) = _DoTests([
......@@ -29,6 +29,7 @@ module Ganeti.Confd.Server
, prepMain
) where
import Control.Applicative((<$>))
import Control.Concurrent
import Control.Monad (forever, liftM)
import Data.IORef
......@@ -112,6 +113,20 @@ getNodePipByInstanceIp cfg linkipmap link instip =
Bad _ -> queryUnknownEntry -- either instance or node not found
Ok node -> (ReplyStatusOk, J.showJSON (nodePrimaryIp node))
-- | Returns a node name for a given UUID
uuidToNodeName :: ConfigData -> String -> Result String
uuidToNodeName cfg uuid = gntErrorToResult $ nodeName <$> getNode cfg uuid
-- | Encodes a list of minors into a JSON representation, converting UUIDs to
-- names in the process
encodeMinors :: ConfigData -> (String, Int, String, String, String, String)
-> Result J.JSValue
encodeMinors cfg (node_uuid, a, b, c, d, peer_uuid) = do
node_name <- uuidToNodeName cfg node_uuid
peer_name <- uuidToNodeName cfg peer_uuid
return . J.JSArray $ [J.showJSON node_name, J.showJSON a, J.showJSON b,
J.showJSON c, J.showJSON d, J.showJSON peer_name]
-- | Builds the response to a given query.
buildResponse :: (ConfigData, LinkIpMap) -> ConfdRequest -> Result StatusAnswer
buildResponse (cfg, _) (ConfdRequest { confdRqType = ReqPing }) =
......@@ -186,11 +201,9 @@ buildResponse cdata req@(ConfdRequest { confdRqType = ReqNodeDrbd }) = do
PlainQuery str -> return str
_ -> fail $ "Invalid query type " ++ show (confdRqQuery req)
node <- gntErrorToResult $ getNode cfg node_name
let minors = concatMap (getInstMinorsForNode (nodeName node)) .
let minors = concatMap (getInstMinorsForNode (nodeUuid node)) .
M.elems . fromContainer . configInstances $ cfg
encoded = [J.JSArray [J.showJSON a, J.showJSON b, J.showJSON c,
J.showJSON d, J.showJSON e, J.showJSON f] |
(a, b, c, d, e, f) <- minors]
encoded <- mapM (encodeMinors cfg) minors
return (ReplyStatusOk, J.showJSON encoded)
-- | Return the list of instances for a node (as ([primary], [secondary])) given
......@@ -334,7 +334,8 @@ roleSecondary = "secondary"
-- | Gets the list of DRBD minors for an instance that are related to
-- a given node.
getInstMinorsForNode :: String -> Instance
getInstMinorsForNode :: String -- ^ The UUID of a node.
-> Instance
-> [(String, Int, String, String, String, String)]
getInstMinorsForNode node inst =
let role = if node == instPrimaryNode inst
......@@ -107,7 +107,7 @@ class TestOpcodeResource(unittest.TestCase):
def _GetMethodAttributes(method):
attrs = ["%s_OPCODE" % method, "%s_RENAME" % method,
"Get%sOpInput" % method.capitalize()]
"%s_ALIASES" % method, "Get%sOpInput" % method.capitalize()]
assert attrs == dict((opattrs[0], list(opattrs[1:]))
for opattrs in baserlib.OPCODE_ATTRS)[method]
return attrs
......@@ -1271,13 +1271,15 @@ class GanetiRapiClientTests(testutils.GanetiTestCase):
for mnh in [None, False, True]:
data = serializer.LoadJson(self.rapi.GetLastRequestData())
self.assertEqual(len(data), 1)
self.assertEqual(data["maintain_node_health"], mnh)
self.assertEqual(self.rapi.CountPending(), 0)
self.assertQuery("reason", ["PinkBunniesInvasion"])
def testRedistributeConfig(self):
......@@ -558,6 +558,42 @@ class TestInstanceDiskGrow(unittest.TestCase):
self.assertRaises(IndexError, cl.GetNextSubmittedJob)
class TestInstanceModify():
def testCustomParamRename(self):
clfactory = _FakeClientFactory(_FakeClient)
name = "instant_instance"
data = {
"custom_beparams": {},
"custom_hvparams": {},
"custom_nicparams": {},
handler = _CreateHandler(rlib2.R_2_instances_name_modify, [name], {}, data,
job_id = handler.PUT()
cl = clfactory.GetNextClient()
self.assertRaises(IndexError, clfactory.GetNextClient)
(exp_job_id, (op, )) = cl.GetNextSubmittedJob()
self.assertEqual(job_id, exp_job_id)
self.assertTrue(isinstance(op, opcodes.OpInstanceSetParams))
self.assertEqual(op.beparams, {})
self.assertEqual(op.hvparams, {})
self.assertEqual(op.nicparams, {})
self.assertRaises(IndexError, cl.GetNextSubmittedJob)
# Define both
data["beparams"] = {}
assert "beparams" in data and "custom_beparams" in data
handler = _CreateHandler(rlib2.R_2_instances_name_modify, [name], {}, data,
self.assertRaises(http.HttpBadRequest, handler.PUT)
class TestBackupPrepare(unittest.TestCase):
def test(self):
clfactory = _FakeClientFactory(_FakeClient)
......@@ -1588,6 +1624,40 @@ class TestGroupModify(unittest.TestCase):
self.assertRaises(IndexError, cl.GetNextSubmittedJob)
def testCustomParamRename(self):
clfactory = _FakeClientFactory(_FakeClient)
name = "groupie"
data = {
"custom_diskparams": {},
"custom_ipolicy": {},
"custom_ndparams": {},
handler = _CreateHandler(rlib2.R_2_groups_name_modify, [name], {}, data,
job_id = handler.PUT()
cl = clfactory.GetNextClient()
self.assertRaises(IndexError, clfactory.GetNextClient)
(exp_job_id, (op, )) = cl.GetNextSubmittedJob()
self.assertEqual(job_id, exp_job_id)
self.assertTrue(isinstance(op, opcodes.OpGroupSetParams))
self.assertEqual(op.diskparams, {})
self.assertEqual(op.ipolicy, {})
self.assertEqual(op.ndparams, {})
self.assertRaises(IndexError, cl.GetNextSubmittedJob)
# Define both
data["diskparams"] = {}
assert "diskparams" in data and "custom_diskparams" in data
handler = _CreateHandler(rlib2.R_2_groups_name_modify, [name], {}, data,
self.assertRaises(http.HttpBadRequest, handler.PUT)
class TestGroupAdd(unittest.TestCase):
def test(self):
......@@ -1746,7 +1816,7 @@ class TestClusterInfo(unittest.TestCase):
def QueryClusterInfo(self):
assert self.cluster_info is None
self.cluster_info = object()
self.cluster_info = {}
return self.cluster_info
def test(self):
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment