Commit 8235fe04 authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Add node query definition



This includes a bunch of helper functions which can be helpful for other
queries, too. Unittests are included.
Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent a123dc19
......@@ -21,6 +21,7 @@
"""Module for query operations"""
import logging
import operator
import re
......@@ -32,6 +33,12 @@ from ganeti import objects
from ganeti import ht
(NQ_CONFIG,
NQ_INST,
NQ_LIVE,
NQ_GROUP) = range(1, 5)
FIELD_NAME_RE = re.compile(r"^[a-z0-9/._]+$")
TITLE_RE = re.compile(r"^[^\s]+$")
......@@ -231,3 +238,203 @@ def _MakeField(name, title, kind):
"""
return objects.QueryFieldDefinition(name=name, title=title, kind=kind)
def _GetNodeRole(node, master_name):
"""Determine node role.
@type node: L{objects.Node}
@param node: Node object
@type master_name: string
@param master_name: Master node name
"""
if node.name == master_name:
return "M"
elif node.master_candidate:
return "C"
elif node.drained:
return "D"
elif node.offline:
return "O"
else:
return "R"
def _GetItemAttr(attr):
"""Returns a field function to return an attribute of the item.
@param attr: Attribute name
"""
getter = operator.attrgetter(attr)
return lambda _, item: (constants.QRFS_NORMAL, getter(item))
class NodeQueryData:
"""Data container for node data queries.
"""
def __init__(self, nodes, live_data, master_name, node_to_primary,
node_to_secondary, groups):
"""Initializes this class.
"""
self.nodes = nodes
self.live_data = live_data
self.master_name = master_name
self.node_to_primary = node_to_primary
self.node_to_secondary = node_to_secondary
self.groups = groups
# Used for individual rows
self.curlive_data = None
def __iter__(self):
"""Iterate over all nodes.
This function has side-effects and only one instance of the resulting
generator should be used at a time.
"""
for node in self.nodes:
if self.live_data:
self.curlive_data = self.live_data.get(node.name, None)
else:
self.curlive_data = None
yield node
#: Fields that are direct attributes of an L{objects.Node} object
_NODE_SIMPLE_FIELDS = {
"ctime": ("CTime", constants.QFT_TIMESTAMP),
"drained": ("Drained", constants.QFT_BOOL),
"master_candidate": ("MasterC", constants.QFT_BOOL),
"master_capable": ("MasterCapable", constants.QFT_BOOL),
"mtime": ("MTime", constants.QFT_TIMESTAMP),
"name": ("Node", constants.QFT_TEXT),
"offline": ("Offline", constants.QFT_BOOL),
"serial_no": ("SerialNo", constants.QFT_NUMBER),
"uuid": ("UUID", constants.QFT_TEXT),
"vm_capable": ("VMCapable", constants.QFT_BOOL),
}
#: Fields requiring talking to the node
_NODE_LIVE_FIELDS = {
"bootid": ("BootID", constants.QFT_TEXT, "bootid"),
"cnodes": ("CNodes", constants.QFT_NUMBER, "cpu_nodes"),
"csockets": ("CSockets", constants.QFT_NUMBER, "cpu_sockets"),
"ctotal": ("CTotal", constants.QFT_NUMBER, "cpu_total"),
"dfree": ("DFree", constants.QFT_UNIT, "vg_free"),
"dtotal": ("DTotal", constants.QFT_UNIT, "vg_size"),
"mfree": ("MFree", constants.QFT_UNIT, "memory_free"),
"mnode": ("MNode", constants.QFT_UNIT, "memory_dom0"),
"mtotal": ("MTotal", constants.QFT_UNIT, "memory_total"),
}
def _GetNodeGroup(ctx, node):
"""Returns the name of a node's group.
@type ctx: L{NodeQueryData}
@type node: L{objects.Node}
@param node: Node object
"""
ng = ctx.groups.get(node.group, None)
if ng is None:
# Nodes always have a group, or the configuration is corrupt
return (constants.QRFS_UNAVAIL, None)
return (constants.QRFS_NORMAL, ng.name)
def _GetLiveNodeField(field, kind, ctx, _):
"""Gets the value of a "live" field from L{NodeQueryData}.
@param field: Live field name
@param kind: Data kind, one of L{constants.QFT_ALL}
@type ctx: L{NodeQueryData}
"""
if not ctx.curlive_data:
return (constants.QRFS_NODATA, None)
try:
value = ctx.curlive_data[field]
except KeyError:
return (constants.QRFS_UNAVAIL, None)
if kind == constants.QFT_TEXT:
return (constants.QRFS_NORMAL, value)
assert kind in (constants.QFT_NUMBER, constants.QFT_UNIT)
# Try to convert into number
try:
return (constants.QRFS_NORMAL, int(value))
except (ValueError, TypeError):
logging.exception("Failed to convert node field '%s' (value %r) to int",
value, field)
return (constants.QRFS_UNAVAIL, None)
def _BuildNodeFields():
"""Builds list of fields for node queries.
"""
fields = [
(_MakeField("pip", "PrimaryIP", constants.QFT_TEXT), NQ_CONFIG,
lambda ctx, node: (constants.QRFS_NORMAL, node.primary_ip)),
(_MakeField("sip", "SecondaryIP", constants.QFT_TEXT), NQ_CONFIG,
lambda ctx, node: (constants.QRFS_NORMAL, node.secondary_ip)),
(_MakeField("tags", "Tags", constants.QFT_OTHER), NQ_CONFIG,
lambda ctx, node: (constants.QRFS_NORMAL, list(node.GetTags()))),
(_MakeField("master", "IsMaster", constants.QFT_BOOL), NQ_CONFIG,
lambda ctx, node: (constants.QRFS_NORMAL, node.name == ctx.master_name)),
(_MakeField("role", "Role", constants.QFT_TEXT), NQ_CONFIG,
lambda ctx, node: (constants.QRFS_NORMAL,
_GetNodeRole(node, ctx.master_name))),
(_MakeField("group", "Group", constants.QFT_TEXT), NQ_GROUP, _GetNodeGroup),
(_MakeField("group.uuid", "GroupUUID", constants.QFT_TEXT),
NQ_CONFIG, lambda ctx, node: (constants.QRFS_NORMAL, node.group)),
]
def _GetLength(getter):
return lambda ctx, node: (constants.QRFS_NORMAL,
len(getter(ctx)[node.name]))
def _GetList(getter):
return lambda ctx, node: (constants.QRFS_NORMAL,
list(getter(ctx)[node.name]))
# Add fields operating on instance lists
for prefix, titleprefix, getter in \
[("p", "Pri", operator.attrgetter("node_to_primary")),
("s", "Sec", operator.attrgetter("node_to_secondary"))]:
fields.extend([
(_MakeField("%sinst_cnt" % prefix, "%sinst" % prefix.upper(),
constants.QFT_NUMBER),
NQ_INST, _GetLength(getter)),
(_MakeField("%sinst_list" % prefix, "%sInstances" % titleprefix,
constants.QFT_OTHER),
NQ_INST, _GetList(getter)),
])
# Add simple fields
fields.extend([(_MakeField(name, title, kind), NQ_CONFIG, _GetItemAttr(name))
for (name, (title, kind)) in _NODE_SIMPLE_FIELDS.items()])
# Add fields requiring live data
fields.extend([
(_MakeField(name, title, kind), NQ_LIVE,
compat.partial(_GetLiveNodeField, nfield, kind))
for (name, (title, kind, nfield)) in _NODE_LIVE_FIELDS.items()
])
return _PrepareFieldList(fields)
#: Fields available for node queries
NODE_FIELDS = _BuildNodeFields()
......@@ -254,5 +254,209 @@ class TestQuery(unittest.TestCase):
for i in range(1, 10)])
class TestGetNodeRole(unittest.TestCase):
def testMaster(self):
node = objects.Node(name="node1")
self.assertEqual(query._GetNodeRole(node, "node1"), "M")
def testMasterCandidate(self):
node = objects.Node(name="node1", master_candidate=True)
self.assertEqual(query._GetNodeRole(node, "master"), "C")
def testRegular(self):
node = objects.Node(name="node1")
self.assertEqual(query._GetNodeRole(node, "master"), "R")
def testDrained(self):
node = objects.Node(name="node1", drained=True)
self.assertEqual(query._GetNodeRole(node, "master"), "D")
def testOffline(self):
node = objects.Node(name="node1", offline=True)
self.assertEqual(query._GetNodeRole(node, "master"), "O")
class TestNodeQuery(unittest.TestCase):
def _Create(self, selected):
return query.Query(query.NODE_FIELDS, selected)
def testSimple(self):
nodes = [
objects.Node(name="node1", drained=False),
objects.Node(name="node2", drained=True),
objects.Node(name="node3", drained=False),
]
for live_data in [None, dict.fromkeys([node.name for node in nodes], {})]:
nqd = query.NodeQueryData(nodes, live_data, None, None, None, None)
q = self._Create(["name", "drained"])
self.assertEqual(q.RequestedData(), set([query.NQ_CONFIG]))
self.assertEqual(q.Query(nqd),
[[(constants.QRFS_NORMAL, "node1"),
(constants.QRFS_NORMAL, False)],
[(constants.QRFS_NORMAL, "node2"),
(constants.QRFS_NORMAL, True)],
[(constants.QRFS_NORMAL, "node3"),
(constants.QRFS_NORMAL, False)],
])
self.assertEqual(q.OldStyleQuery(nqd),
[["node1", False],
["node2", True],
["node3", False]])
def test(self):
selected = query.NODE_FIELDS.keys()
field_index = dict((field, idx) for idx, field in enumerate(selected))
q = self._Create(selected)
self.assertEqual(q.RequestedData(),
set([query.NQ_CONFIG, query.NQ_LIVE, query.NQ_INST,
query.NQ_GROUP]))
node_names = ["node%s" % i for i in range(20)]
master_name = node_names[3]
nodes = [
objects.Node(name=name,
primary_ip="192.0.2.%s" % idx,
secondary_ip="192.0.100.%s" % idx,
serial_no=7789 * idx,
master_candidate=(name != master_name and idx % 3 == 0),
offline=False,
drained=False,
vm_capable=False,
master_capable=False,
group="default",
ctime=1290006900,
mtime=1290006913,
uuid="fd9ccebe-6339-43c9-a82e-94bbe575%04d" % idx)
for idx, name in enumerate(node_names)
]
master_node = nodes[3]
master_node.AddTag("masternode")
master_node.AddTag("another")
master_node.AddTag("tag")
assert master_node.name == master_name
live_data_name = node_names[4]
assert live_data_name != master_name
fake_live_data = {
"bootid": "a2504766-498e-4b25-b21e-d23098dc3af4",
"cnodes": 4,
"csockets": 4,
"ctotal": 8,
"mnode": 128,
"mfree": 100,
"mtotal": 4096,
"dfree": 5 * 1024 * 1024,
"dtotal": 100 * 1024 * 1024,
}
assert (sorted(query._NODE_LIVE_FIELDS.keys()) ==
sorted(fake_live_data.keys()))
live_data = dict.fromkeys(node_names, {})
live_data[live_data_name] = \
dict((query._NODE_LIVE_FIELDS[name][2], value)
for name, value in fake_live_data.items())
node_to_primary = dict((name, set()) for name in node_names)
node_to_primary[master_name].update(["inst1", "inst2"])
node_to_secondary = dict((name, set()) for name in node_names)
node_to_secondary[live_data_name].update(["instX", "instY", "instZ"])
ng_uuid = "492b4b74-8670-478a-b98d-4c53a76238e6"
groups = {
ng_uuid: objects.NodeGroup(name="ng1", uuid=ng_uuid),
}
master_node.group = ng_uuid
nqd = query.NodeQueryData(nodes, live_data, master_name,
node_to_primary, node_to_secondary, groups)
result = q.Query(nqd)
self.assert_(compat.all(len(row) == len(selected) for row in result))
self.assertEqual([row[field_index["name"]] for row in result],
[(constants.QRFS_NORMAL, name) for name in node_names])
node_to_row = dict((row[field_index["name"]][1], idx)
for idx, row in enumerate(result))
master_row = result[node_to_row[master_name]]
self.assert_(master_row[field_index["master"]])
self.assert_(master_row[field_index["role"]], "M")
self.assertEqual(master_row[field_index["group"]],
(constants.QRFS_NORMAL, "ng1"))
self.assertEqual(master_row[field_index["group.uuid"]],
(constants.QRFS_NORMAL, ng_uuid))
self.assert_(row[field_index["pip"]] == node.primary_ip and
row[field_index["sip"]] == node.secondary_ip and
set(row[field_index["tags"]]) == node.GetTags() and
row[field_index["serial_no"]] == node.serial_no and
row[field_index["role"]] == query._GetNodeRole(node,
master_name) and
(node.name == master_name or
(row[field_index["group"]] == "<unknown>" and
row[field_index["group.uuid"]] is None))
for row, node in zip(result, nodes))
live_data_row = result[node_to_row[live_data_name]]
for (field, value) in fake_live_data.items():
self.assertEqual(live_data_row[field_index[field]],
(constants.QRFS_NORMAL, value))
self.assertEqual(master_row[field_index["pinst_cnt"]],
(constants.QRFS_NORMAL, 2))
self.assertEqual(live_data_row[field_index["sinst_cnt"]],
(constants.QRFS_NORMAL, 3))
self.assertEqual(master_row[field_index["pinst_list"]],
(constants.QRFS_NORMAL,
list(node_to_primary[master_name])))
self.assertEqual(live_data_row[field_index["sinst_list"]],
(constants.QRFS_NORMAL,
list(node_to_secondary[live_data_name])))
def testGetLiveNodeField(self):
nodes = [
objects.Node(name="node1", drained=False),
objects.Node(name="node2", drained=True),
objects.Node(name="node3", drained=False),
]
live_data = dict.fromkeys([node.name for node in nodes], {})
# No data
nqd = query.NodeQueryData(None, None, None, None, None, None)
self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
nqd, None),
(constants.QRFS_NODATA, None))
# Missing field
ctx = _QueryData(None, curlive_data={
"some": 1,
"other": 2,
})
self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
ctx, None),
(constants.QRFS_UNAVAIL, None))
# Wrong format/datatype
ctx = _QueryData(None, curlive_data={
"hello": ["Hello World"],
"other": 2,
})
self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
ctx, None),
(constants.QRFS_UNAVAIL, None))
# Wrong field type
ctx = _QueryData(None, curlive_data={"hello": 123})
self.assertRaises(AssertionError, query._GetLiveNodeField,
"hello", constants.QFT_BOOL, ctx, None)
if __name__ == "__main__":
testutils.GanetiTestProgram()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment