From 8235fe04125877e72bd51f2488a311d72699a9da Mon Sep 17 00:00:00 2001
From: Michael Hanselmann <hansmi@google.com>
Date: Fri, 12 Nov 2010 16:17:43 +0100
Subject: [PATCH] Add node query definition
This includes a bunch of helper functions which can be helpful for other
queries, too. Unittests are included.
Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>
---
lib/query.py | 207 ++++++++++++++++++++++++++++++++++
test/ganeti.query_unittest.py | 204 +++++++++++++++++++++++++++++++++
2 files changed, 411 insertions(+)
diff --git a/lib/query.py b/lib/query.py
index 9be8c5452..11f1025a2 100644
--- a/lib/query.py
+++ b/lib/query.py
@@ -21,6 +21,7 @@
"""Module for query operations"""
+import logging
import operator
import re
@@ -32,6 +33,12 @@ from ganeti import objects
from ganeti import ht
+(NQ_CONFIG,
+ NQ_INST,
+ NQ_LIVE,
+ NQ_GROUP) = range(1, 5)
+
+
FIELD_NAME_RE = re.compile(r"^[a-z0-9/._]+$")
TITLE_RE = re.compile(r"^[^\s]+$")
@@ -231,3 +238,203 @@ def _MakeField(name, title, kind):
"""
return objects.QueryFieldDefinition(name=name, title=title, kind=kind)
+
+
+def _GetNodeRole(node, master_name):
+ """Determine node role.
+
+ @type node: L{objects.Node}
+ @param node: Node object
+ @type master_name: string
+ @param master_name: Master node name
+
+ """
+ if node.name == master_name:
+ return "M"
+ elif node.master_candidate:
+ return "C"
+ elif node.drained:
+ return "D"
+ elif node.offline:
+ return "O"
+ else:
+ return "R"
+
+
+def _GetItemAttr(attr):
+ """Returns a field function to return an attribute of the item.
+
+ @param attr: Attribute name
+
+ """
+ getter = operator.attrgetter(attr)
+ return lambda _, item: (constants.QRFS_NORMAL, getter(item))
+
+
+class NodeQueryData:
+ """Data container for node data queries.
+
+ """
+ def __init__(self, nodes, live_data, master_name, node_to_primary,
+ node_to_secondary, groups):
+ """Initializes this class.
+
+ """
+ self.nodes = nodes
+ self.live_data = live_data
+ self.master_name = master_name
+ self.node_to_primary = node_to_primary
+ self.node_to_secondary = node_to_secondary
+ self.groups = groups
+
+ # Used for individual rows
+ self.curlive_data = None
+
+ def __iter__(self):
+ """Iterate over all nodes.
+
+ This function has side-effects and only one instance of the resulting
+ generator should be used at a time.
+
+ """
+ for node in self.nodes:
+ if self.live_data:
+ self.curlive_data = self.live_data.get(node.name, None)
+ else:
+ self.curlive_data = None
+ yield node
+
+
+#: Fields that are direct attributes of an L{objects.Node} object
+_NODE_SIMPLE_FIELDS = {
+ "ctime": ("CTime", constants.QFT_TIMESTAMP),
+ "drained": ("Drained", constants.QFT_BOOL),
+ "master_candidate": ("MasterC", constants.QFT_BOOL),
+ "master_capable": ("MasterCapable", constants.QFT_BOOL),
+ "mtime": ("MTime", constants.QFT_TIMESTAMP),
+ "name": ("Node", constants.QFT_TEXT),
+ "offline": ("Offline", constants.QFT_BOOL),
+ "serial_no": ("SerialNo", constants.QFT_NUMBER),
+ "uuid": ("UUID", constants.QFT_TEXT),
+ "vm_capable": ("VMCapable", constants.QFT_BOOL),
+ }
+
+
+#: Fields requiring talking to the node
+_NODE_LIVE_FIELDS = {
+ "bootid": ("BootID", constants.QFT_TEXT, "bootid"),
+ "cnodes": ("CNodes", constants.QFT_NUMBER, "cpu_nodes"),
+ "csockets": ("CSockets", constants.QFT_NUMBER, "cpu_sockets"),
+ "ctotal": ("CTotal", constants.QFT_NUMBER, "cpu_total"),
+ "dfree": ("DFree", constants.QFT_UNIT, "vg_free"),
+ "dtotal": ("DTotal", constants.QFT_UNIT, "vg_size"),
+ "mfree": ("MFree", constants.QFT_UNIT, "memory_free"),
+ "mnode": ("MNode", constants.QFT_UNIT, "memory_dom0"),
+ "mtotal": ("MTotal", constants.QFT_UNIT, "memory_total"),
+ }
+
+
+def _GetNodeGroup(ctx, node):
+ """Returns the name of a node's group.
+
+ @type ctx: L{NodeQueryData}
+ @type node: L{objects.Node}
+ @param node: Node object
+
+ """
+ ng = ctx.groups.get(node.group, None)
+ if ng is None:
+ # Nodes always have a group, or the configuration is corrupt
+ return (constants.QRFS_UNAVAIL, None)
+
+ return (constants.QRFS_NORMAL, ng.name)
+
+
+def _GetLiveNodeField(field, kind, ctx, _):
+ """Gets the value of a "live" field from L{NodeQueryData}.
+
+ @param field: Live field name
+ @param kind: Data kind, one of L{constants.QFT_ALL}
+ @type ctx: L{NodeQueryData}
+
+ """
+ if not ctx.curlive_data:
+ return (constants.QRFS_NODATA, None)
+
+ try:
+ value = ctx.curlive_data[field]
+ except KeyError:
+ return (constants.QRFS_UNAVAIL, None)
+
+ if kind == constants.QFT_TEXT:
+ return (constants.QRFS_NORMAL, value)
+
+ assert kind in (constants.QFT_NUMBER, constants.QFT_UNIT)
+
+ # Try to convert into number
+ try:
+ return (constants.QRFS_NORMAL, int(value))
+ except (ValueError, TypeError):
+ logging.exception("Failed to convert node field '%s' (value %r) to int",
+ value, field)
+ return (constants.QRFS_UNAVAIL, None)
+
+
+def _BuildNodeFields():
+ """Builds list of fields for node queries.
+
+ """
+ fields = [
+ (_MakeField("pip", "PrimaryIP", constants.QFT_TEXT), NQ_CONFIG,
+ lambda ctx, node: (constants.QRFS_NORMAL, node.primary_ip)),
+ (_MakeField("sip", "SecondaryIP", constants.QFT_TEXT), NQ_CONFIG,
+ lambda ctx, node: (constants.QRFS_NORMAL, node.secondary_ip)),
+ (_MakeField("tags", "Tags", constants.QFT_OTHER), NQ_CONFIG,
+ lambda ctx, node: (constants.QRFS_NORMAL, list(node.GetTags()))),
+ (_MakeField("master", "IsMaster", constants.QFT_BOOL), NQ_CONFIG,
+ lambda ctx, node: (constants.QRFS_NORMAL, node.name == ctx.master_name)),
+ (_MakeField("role", "Role", constants.QFT_TEXT), NQ_CONFIG,
+ lambda ctx, node: (constants.QRFS_NORMAL,
+ _GetNodeRole(node, ctx.master_name))),
+ (_MakeField("group", "Group", constants.QFT_TEXT), NQ_GROUP, _GetNodeGroup),
+ (_MakeField("group.uuid", "GroupUUID", constants.QFT_TEXT),
+ NQ_CONFIG, lambda ctx, node: (constants.QRFS_NORMAL, node.group)),
+ ]
+
+ def _GetLength(getter):
+ return lambda ctx, node: (constants.QRFS_NORMAL,
+ len(getter(ctx)[node.name]))
+
+ def _GetList(getter):
+ return lambda ctx, node: (constants.QRFS_NORMAL,
+ list(getter(ctx)[node.name]))
+
+ # Add fields operating on instance lists
+ for prefix, titleprefix, getter in \
+ [("p", "Pri", operator.attrgetter("node_to_primary")),
+ ("s", "Sec", operator.attrgetter("node_to_secondary"))]:
+ fields.extend([
+ (_MakeField("%sinst_cnt" % prefix, "%sinst" % prefix.upper(),
+ constants.QFT_NUMBER),
+ NQ_INST, _GetLength(getter)),
+ (_MakeField("%sinst_list" % prefix, "%sInstances" % titleprefix,
+ constants.QFT_OTHER),
+ NQ_INST, _GetList(getter)),
+ ])
+
+ # Add simple fields
+ fields.extend([(_MakeField(name, title, kind), NQ_CONFIG, _GetItemAttr(name))
+ for (name, (title, kind)) in _NODE_SIMPLE_FIELDS.items()])
+
+ # Add fields requiring live data
+ fields.extend([
+ (_MakeField(name, title, kind), NQ_LIVE,
+ compat.partial(_GetLiveNodeField, nfield, kind))
+ for (name, (title, kind, nfield)) in _NODE_LIVE_FIELDS.items()
+ ])
+
+ return _PrepareFieldList(fields)
+
+
+#: Fields available for node queries
+NODE_FIELDS = _BuildNodeFields()
diff --git a/test/ganeti.query_unittest.py b/test/ganeti.query_unittest.py
index 098645756..2e37fd82f 100755
--- a/test/ganeti.query_unittest.py
+++ b/test/ganeti.query_unittest.py
@@ -254,5 +254,209 @@ class TestQuery(unittest.TestCase):
for i in range(1, 10)])
+class TestGetNodeRole(unittest.TestCase):
+ def testMaster(self):
+ node = objects.Node(name="node1")
+ self.assertEqual(query._GetNodeRole(node, "node1"), "M")
+
+ def testMasterCandidate(self):
+ node = objects.Node(name="node1", master_candidate=True)
+ self.assertEqual(query._GetNodeRole(node, "master"), "C")
+
+ def testRegular(self):
+ node = objects.Node(name="node1")
+ self.assertEqual(query._GetNodeRole(node, "master"), "R")
+
+ def testDrained(self):
+ node = objects.Node(name="node1", drained=True)
+ self.assertEqual(query._GetNodeRole(node, "master"), "D")
+
+ def testOffline(self):
+ node = objects.Node(name="node1", offline=True)
+ self.assertEqual(query._GetNodeRole(node, "master"), "O")
+
+
+class TestNodeQuery(unittest.TestCase):
+ def _Create(self, selected):
+ return query.Query(query.NODE_FIELDS, selected)
+
+ def testSimple(self):
+ nodes = [
+ objects.Node(name="node1", drained=False),
+ objects.Node(name="node2", drained=True),
+ objects.Node(name="node3", drained=False),
+ ]
+ for live_data in [None, dict.fromkeys([node.name for node in nodes], {})]:
+ nqd = query.NodeQueryData(nodes, live_data, None, None, None, None)
+
+ q = self._Create(["name", "drained"])
+ self.assertEqual(q.RequestedData(), set([query.NQ_CONFIG]))
+ self.assertEqual(q.Query(nqd),
+ [[(constants.QRFS_NORMAL, "node1"),
+ (constants.QRFS_NORMAL, False)],
+ [(constants.QRFS_NORMAL, "node2"),
+ (constants.QRFS_NORMAL, True)],
+ [(constants.QRFS_NORMAL, "node3"),
+ (constants.QRFS_NORMAL, False)],
+ ])
+ self.assertEqual(q.OldStyleQuery(nqd),
+ [["node1", False],
+ ["node2", True],
+ ["node3", False]])
+
+ def test(self):
+ selected = query.NODE_FIELDS.keys()
+ field_index = dict((field, idx) for idx, field in enumerate(selected))
+
+ q = self._Create(selected)
+ self.assertEqual(q.RequestedData(),
+ set([query.NQ_CONFIG, query.NQ_LIVE, query.NQ_INST,
+ query.NQ_GROUP]))
+
+ node_names = ["node%s" % i for i in range(20)]
+ master_name = node_names[3]
+ nodes = [
+ objects.Node(name=name,
+ primary_ip="192.0.2.%s" % idx,
+ secondary_ip="192.0.100.%s" % idx,
+ serial_no=7789 * idx,
+ master_candidate=(name != master_name and idx % 3 == 0),
+ offline=False,
+ drained=False,
+ vm_capable=False,
+ master_capable=False,
+ group="default",
+ ctime=1290006900,
+ mtime=1290006913,
+ uuid="fd9ccebe-6339-43c9-a82e-94bbe575%04d" % idx)
+ for idx, name in enumerate(node_names)
+ ]
+
+ master_node = nodes[3]
+ master_node.AddTag("masternode")
+ master_node.AddTag("another")
+ master_node.AddTag("tag")
+ assert master_node.name == master_name
+
+ live_data_name = node_names[4]
+ assert live_data_name != master_name
+
+ fake_live_data = {
+ "bootid": "a2504766-498e-4b25-b21e-d23098dc3af4",
+ "cnodes": 4,
+ "csockets": 4,
+ "ctotal": 8,
+ "mnode": 128,
+ "mfree": 100,
+ "mtotal": 4096,
+ "dfree": 5 * 1024 * 1024,
+ "dtotal": 100 * 1024 * 1024,
+ }
+
+ assert (sorted(query._NODE_LIVE_FIELDS.keys()) ==
+ sorted(fake_live_data.keys()))
+
+ live_data = dict.fromkeys(node_names, {})
+ live_data[live_data_name] = \
+ dict((query._NODE_LIVE_FIELDS[name][2], value)
+ for name, value in fake_live_data.items())
+
+ node_to_primary = dict((name, set()) for name in node_names)
+ node_to_primary[master_name].update(["inst1", "inst2"])
+
+ node_to_secondary = dict((name, set()) for name in node_names)
+ node_to_secondary[live_data_name].update(["instX", "instY", "instZ"])
+
+ ng_uuid = "492b4b74-8670-478a-b98d-4c53a76238e6"
+ groups = {
+ ng_uuid: objects.NodeGroup(name="ng1", uuid=ng_uuid),
+ }
+
+ master_node.group = ng_uuid
+
+ nqd = query.NodeQueryData(nodes, live_data, master_name,
+ node_to_primary, node_to_secondary, groups)
+ result = q.Query(nqd)
+ self.assert_(compat.all(len(row) == len(selected) for row in result))
+ self.assertEqual([row[field_index["name"]] for row in result],
+ [(constants.QRFS_NORMAL, name) for name in node_names])
+
+ node_to_row = dict((row[field_index["name"]][1], idx)
+ for idx, row in enumerate(result))
+
+ master_row = result[node_to_row[master_name]]
+ self.assert_(master_row[field_index["master"]])
+ self.assert_(master_row[field_index["role"]], "M")
+ self.assertEqual(master_row[field_index["group"]],
+ (constants.QRFS_NORMAL, "ng1"))
+ self.assertEqual(master_row[field_index["group.uuid"]],
+ (constants.QRFS_NORMAL, ng_uuid))
+
+ self.assert_(row[field_index["pip"]] == node.primary_ip and
+ row[field_index["sip"]] == node.secondary_ip and
+ set(row[field_index["tags"]]) == node.GetTags() and
+ row[field_index["serial_no"]] == node.serial_no and
+ row[field_index["role"]] == query._GetNodeRole(node,
+ master_name) and
+ (node.name == master_name or
+ (row[field_index["group"]] == "<unknown>" and
+ row[field_index["group.uuid"]] is None))
+ for row, node in zip(result, nodes))
+
+ live_data_row = result[node_to_row[live_data_name]]
+
+ for (field, value) in fake_live_data.items():
+ self.assertEqual(live_data_row[field_index[field]],
+ (constants.QRFS_NORMAL, value))
+
+ self.assertEqual(master_row[field_index["pinst_cnt"]],
+ (constants.QRFS_NORMAL, 2))
+ self.assertEqual(live_data_row[field_index["sinst_cnt"]],
+ (constants.QRFS_NORMAL, 3))
+ self.assertEqual(master_row[field_index["pinst_list"]],
+ (constants.QRFS_NORMAL,
+ list(node_to_primary[master_name])))
+ self.assertEqual(live_data_row[field_index["sinst_list"]],
+ (constants.QRFS_NORMAL,
+ list(node_to_secondary[live_data_name])))
+
+ def testGetLiveNodeField(self):
+ nodes = [
+ objects.Node(name="node1", drained=False),
+ objects.Node(name="node2", drained=True),
+ objects.Node(name="node3", drained=False),
+ ]
+ live_data = dict.fromkeys([node.name for node in nodes], {})
+
+ # No data
+ nqd = query.NodeQueryData(None, None, None, None, None, None)
+ self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
+ nqd, None),
+ (constants.QRFS_NODATA, None))
+
+ # Missing field
+ ctx = _QueryData(None, curlive_data={
+ "some": 1,
+ "other": 2,
+ })
+ self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
+ ctx, None),
+ (constants.QRFS_UNAVAIL, None))
+
+ # Wrong format/datatype
+ ctx = _QueryData(None, curlive_data={
+ "hello": ["Hello World"],
+ "other": 2,
+ })
+ self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
+ ctx, None),
+ (constants.QRFS_UNAVAIL, None))
+
+ # Wrong field type
+ ctx = _QueryData(None, curlive_data={"hello": 123})
+ self.assertRaises(AssertionError, query._GetLiveNodeField,
+ "hello", constants.QFT_BOOL, ctx, None)
+
+
if __name__ == "__main__":
testutils.GanetiTestProgram()
--
GitLab