qlang: Add parser for query filter language

With this parser, command line utilities will be able to provide filters
through query2 in a simplistic language. Example filters:

  name == "node3.example.com"
  master or (name == "node4.example.com")
  be/memory == 128 and name =~ /^web/i
  "inst1.example.com" in sinst_list
  status != "up"
  not master

Parts of the syntax came from Python, others from Perl. Documentation
will be added in follow-up patches.

Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>

INSTALL

@@ -28,7 +28,8 @@ Before installing, please verify that you have the following programs:
 - `Python <http://www.python.org/>`_, version 2.4 or above, not 3.0
 - `Python OpenSSL bindings <http://pyopenssl.sourceforge.net/>`_
 - `simplejson Python module <http://code.google.com/p/simplejson/>`_
-- `pyparsing Python module <http://pyparsing.wikispaces.com/>`_
+- `pyparsing Python module <http://pyparsing.wikispaces.com/>`_, version
+  1.4.6 or above
 - `pyinotify Python module <http://trac.dbzteam.org/pyinotify/>`_
 - `PycURL Python module <http://pycurl.sourceforge.net/>`_
 - `ctypes Python module

NEWS

@@ -16,6 +16,8 @@ Version 2.5.0 beta1
 - Support for the undocumented and deprecated RAPI instance creation
   request format version 0 has been dropped. Use version 1, supported
   since Ganeti 2.1.3 and :doc:`documented <rapi>`, instead.
+- Pyparsing 1.4.6 or above is required, see :doc:`installation
+  documentation <install>`
 
 
 Version 2.4.1

lib/errors.py

@@ -391,6 +391,24 @@ class LuxiError(GenericError):
   """
 
 
+class QueryFilterParseError(ParseError):
+  """Error while parsing query filter.
+
+  """
+  def GetDetails(self):
+    """Returns a list of strings with details about the error.
+
+    """
+    try:
+      (_, inner) = self.args
+    except IndexError:
+      return None
+
+    return [str(inner.line),
+            (" " * (inner.column - 1)) + "^",
+            str(inner)]
+
+
 # errors should be added above
 
 

lib/qlang.py

@@ -31,6 +31,13 @@ converted to callable functions by L{query._CompileFilter}.
 
 """
 
+import re
+
+import pyparsing as pyp
+
+from ganeti import errors
+
+
 # Logic operators with one or more operands, each of which is a filter on its
 # own
 OP_OR = "|"
@@ -61,3 +68,155 @@ def MakeSimpleFilter(namefield, values):
     return [OP_OR] + [[OP_EQUAL, namefield, i] for i in values]
 
   return None
+
+
+def _ConvertLogicOp(op):
+  """Creates parsing action function for logic operator.
+
+  @type op: string
+  @param op: Operator for data structure, e.g. L{OP_AND}
+
+  """
+  def fn(toks):
+    """Converts parser tokens to query operator structure.
+
+    @rtype: list
+    @return: Query operator structure, e.g. C{[OP_AND, ["=", "foo", "bar"]]}
+
+    """
+    operands = toks[0]
+
+    if len(operands) == 1:
+      return operands[0]
+
+    # Build query operator structure
+    return [[op] + operands.asList()]
+
+  return fn
+
+
+_KNOWN_REGEXP_DELIM = "/#^|"
+_KNOWN_REGEXP_FLAGS = frozenset("si")
+
+
+def _ConvertRegexpValue(_, loc, toks):
+  """Regular expression value for condition.
+
+  """
+  (regexp, flags) = toks[0]
+
+  # Ensure only whitelisted flags are used
+  unknown_flags = (frozenset(flags) - _KNOWN_REGEXP_FLAGS)
+  if unknown_flags:
+    raise pyp.ParseFatalException("Unknown regular expression flags: '%s'" %
+                                  "".join(unknown_flags), loc)
+
+  if flags:
+    re_flags = "(?%s)" % "".join(sorted(flags))
+  else:
+    re_flags = ""
+
+  re_cond = re_flags + regexp
+
+  # Test if valid
+  try:
+    re.compile(re_cond)
+  except re.error, err:
+    raise pyp.ParseFatalException("Invalid regular expression (%s)" % err, loc)
+
+  return [re_cond]
+
+
+def BuildFilterParser():
+  """Builds a parser for query filter strings.
+
+  @rtype: pyparsing.ParserElement
+
+  """
+  field_name = pyp.Word(pyp.alphas, pyp.alphanums + "_/.")
+
+  # Integer
+  num_sign = pyp.Word("-+", exact=1)
+  number = pyp.Combine(pyp.Optional(num_sign) + pyp.Word(pyp.nums))
+  number.setParseAction(lambda toks: int(toks[0]))
+
+  # Right-hand-side value
+  rval = (number | pyp.quotedString.setParseAction(pyp.removeQuotes))
+
+  # Boolean condition
+  bool_cond = field_name.copy()
+  bool_cond.setParseAction(lambda (fname, ): [[OP_TRUE, fname]])
+
+  # Simple binary conditions
+  binopstbl = {
+    "==": OP_EQUAL,
+    "!=": OP_NOT_EQUAL,
+    }
+
+  binary_cond = (field_name + pyp.oneOf(binopstbl.keys()) + rval)
+  binary_cond.setParseAction(lambda (lhs, op, rhs): [[binopstbl[op], lhs, rhs]])
+
+  # "in" condition
+  in_cond = (rval + pyp.Suppress("in") + field_name)
+  in_cond.setParseAction(lambda (value, field): [[OP_CONTAINS, field, value]])
+
+  # "not in" condition
+  not_in_cond = (rval + pyp.Suppress("not") + pyp.Suppress("in") + field_name)
+  not_in_cond.setParseAction(lambda (value, field): [[OP_NOT, [OP_CONTAINS,
+                                                               field, value]]])
+
+  # Regular expression, e.g. m/foobar/i
+  regexp_val = pyp.Group(pyp.Optional("m").suppress() +
+                         pyp.MatchFirst([pyp.QuotedString(i, escChar="\\")
+                                         for i in _KNOWN_REGEXP_DELIM]) +
+                         pyp.Optional(pyp.Word(pyp.alphas), default=""))
+  regexp_val.setParseAction(_ConvertRegexpValue)
+  regexp_cond = (field_name + pyp.Suppress("=~") + regexp_val)
+  regexp_cond.setParseAction(lambda (field, value): [[OP_REGEXP, field, value]])
+
+  not_regexp_cond = (field_name + pyp.Suppress("!~") + regexp_val)
+  not_regexp_cond.setParseAction(lambda (field, value):
+                                 [[OP_NOT, [OP_REGEXP, field, value]]])
+
+  # All possible conditions
+  condition = (binary_cond ^ bool_cond ^
+               in_cond ^ not_in_cond ^
+               regexp_cond ^ not_regexp_cond)
+
+  # Associativity operators
+  filter_expr = pyp.operatorPrecedence(condition, [
+    (pyp.Keyword("not").suppress(), 1, pyp.opAssoc.RIGHT,
+     lambda toks: [[OP_NOT, toks[0][0]]]),
+    (pyp.Keyword("and").suppress(), 2, pyp.opAssoc.LEFT,
+     _ConvertLogicOp(OP_AND)),
+    (pyp.Keyword("or").suppress(), 2, pyp.opAssoc.LEFT,
+     _ConvertLogicOp(OP_OR)),
+    ])
+
+  parser = pyp.StringStart() + filter_expr + pyp.StringEnd()
+  parser.parseWithTabs()
+
+  # Originally C{parser.validate} was called here, but there seems to be some
+  # issue causing it to fail whenever the "not" operator is included above.
+
+  return parser
+
+
+def ParseFilter(text, parser=None):
+  """Parses a query filter.
+
+  @type text: string
+  @param text: Query filter
+  @type parser: pyparsing.ParserElement
+  @param parser: Pyparsing object
+  @rtype: list
+
+  """
+  if parser is None:
+    parser = BuildFilterParser()
+
+  try:
+    return parser.parseString(text)[0]
+  except pyp.ParseBaseException, err:
+    raise errors.QueryFilterParseError("Failed to parse query filter"
+                                       " '%s': %s" % (text, err), err)

test/ganeti.qlang_unittest.py

@@ -26,6 +26,7 @@ import unittest
 from ganeti import utils
 from ganeti import errors
 from ganeti import qlang
+from ganeti import query
 
 import testutils
 
@@ -47,5 +48,123 @@ class TestMakeSimpleFilter(unittest.TestCase):
                ["|", ["=", "xyz", "a"], ["=", "xyz", "b"], ["=", "xyz", "c"]])
 
 
+class TestParseFilter(unittest.TestCase):
+  def setUp(self):
+    self.parser = qlang.BuildFilterParser()
+
+  def _Test(self, filter_, expected):
+    self.assertEqual(qlang.ParseFilter(filter_, parser=self.parser), expected)
+
+  def test(self):
+    self._Test("name==\"foobar\"", [qlang.OP_EQUAL, "name", "foobar"])
+    self._Test("name=='foobar'", [qlang.OP_EQUAL, "name", "foobar"])
+
+    self._Test("valA==1 and valB==2 or valC==3",
+               [qlang.OP_OR,
+                [qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1],
+                               [qlang.OP_EQUAL, "valB", 2]],
+                [qlang.OP_EQUAL, "valC", 3]])
+
+    self._Test(("(name\n==\"foobar\") and (xyz==\"va)ue\" and k == 256 or"
+                " x ==\t\"y\"\n) and mc"),
+               [qlang.OP_AND,
+                [qlang.OP_EQUAL, "name", "foobar"],
+                [qlang.OP_OR,
+                 [qlang.OP_AND, [qlang.OP_EQUAL, "xyz", "va)ue"],
+                                [qlang.OP_EQUAL, "k", 256]],
+                 [qlang.OP_EQUAL, "x", "y"]],
+                [qlang.OP_TRUE, "mc"]])
+
+    self._Test("(xyz==\"v\" or k == 256 and x == \"y\")",
+               [qlang.OP_OR,
+                [qlang.OP_EQUAL, "xyz", "v"],
+                [qlang.OP_AND, [qlang.OP_EQUAL, "k", 256],
+                               [qlang.OP_EQUAL, "x", "y"]]])
+
+    self._Test("valA==1 and valB==2 and valC==3",
+               [qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1],
+                              [qlang.OP_EQUAL, "valB", 2],
+                              [qlang.OP_EQUAL, "valC", 3]])
+    self._Test("master or field",
+               [qlang.OP_OR, [qlang.OP_TRUE, "master"],
+                             [qlang.OP_TRUE, "field"]])
+    self._Test("mem == 128", [qlang.OP_EQUAL, "mem", 128])
+    self._Test("negfield != -1", [qlang.OP_NOT_EQUAL, "negfield", -1])
+    self._Test("master", [qlang.OP_TRUE, "master"])
+    self._Test("not master", [qlang.OP_NOT, [qlang.OP_TRUE, "master"]])
+    for op in ["not", "and", "or"]:
+      self._Test("%sxyz" % op, [qlang.OP_TRUE, "%sxyz" % op])
+      self._Test("not %sxyz" % op,
+                 [qlang.OP_NOT, [qlang.OP_TRUE, "%sxyz" % op]])
+      self._Test("  not \t%sfoo" % op,
+                 [qlang.OP_NOT, [qlang.OP_TRUE, "%sfoo" % op]])
+      self._Test("%sname =~ m/abc/" % op,
+                 [qlang.OP_REGEXP, "%sname" % op, "abc"])
+    self._Test("master and not other",
+               [qlang.OP_AND, [qlang.OP_TRUE, "master"],
+                              [qlang.OP_NOT, [qlang.OP_TRUE, "other"]]])
+    self._Test("not (master or other == 4)",
+               [qlang.OP_NOT,
+                [qlang.OP_OR, [qlang.OP_TRUE, "master"],
+                              [qlang.OP_EQUAL, "other", 4]]])
+    self._Test("some==\"val\\\"ue\"", [qlang.OP_EQUAL, "some", "val\\\"ue"])
+    self._Test("123 in ips", [qlang.OP_CONTAINS, "ips", 123])
+    self._Test("99 not in ips", [qlang.OP_NOT, [qlang.OP_CONTAINS, "ips", 99]])
+    self._Test("\"a\" in valA and \"b\" not in valB",
+               [qlang.OP_AND, [qlang.OP_CONTAINS, "valA", "a"],
+                              [qlang.OP_NOT, [qlang.OP_CONTAINS, "valB", "b"]]])
+
+    self._Test("name =~ m/test/", [qlang.OP_REGEXP, "name", "test"])
+    self._Test("name =~ m/^node.*example.com$/i",
+               [qlang.OP_REGEXP, "name", "(?i)^node.*example.com$"])
+    self._Test("(name =~ m/^node.*example.com$/s and master) or pip =~ |^3.*|",
+               [qlang.OP_OR,
+                [qlang.OP_AND,
+                 [qlang.OP_REGEXP, "name", "(?s)^node.*example.com$"],
+                 [qlang.OP_TRUE, "master"]],
+                [qlang.OP_REGEXP, "pip", "^3.*"]])
+    for flags in ["si", "is", "ssss", "iiiisiii"]:
+      self._Test("name =~ m/gi/%s" % flags,
+                 [qlang.OP_REGEXP, "name", "(?%s)gi" % "".join(sorted(flags))])
+
+    for i in qlang._KNOWN_REGEXP_DELIM:
+      self._Test("name =~ m%stest%s" % (i, i),
+                 [qlang.OP_REGEXP, "name", "test"])
+      self._Test("name !~ m%stest%s" % (i, i),
+                 [qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]])
+      self._Test("not\tname =~ m%stest%s" % (i, i),
+                 [qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]])
+      self._Test("notname =~ m%stest%s" % (i, i),
+                 [qlang.OP_REGEXP, "notname", "test"])
+
+  def testAllFields(self):
+    for name in frozenset(i for d in query.ALL_FIELD_LISTS for i in d.keys()):
+      self._Test("%s == \"value\"" % name, [qlang.OP_EQUAL, name, "value"])
+
+  def testError(self):
+    # Invalid field names, meaning no boolean check is done
+    tests = ["#invalid!filter#", "m/x/,"]
+
+    # Unknown regexp flag
+    tests.append("name=~m#a#g")
+
+    # Incomplete regexp group
+    tests.append("name=~^[^")
+
+    # Valid flag, but in uppercase
+    tests.append("asdf =~ m|abc|I")
+
+    # Non-matching regexp delimiters
+    tests.append("name =~ /foobarbaz#")
+
+    for filter_ in tests:
+      try:
+        qlang.ParseFilter(filter_, parser=self.parser)
+      except errors.QueryFilterParseError, err:
+        self.assertEqual(len(err.GetDetails()), 3)
+      else:
+        self.fail("Invalid filter '%s' did not raise exception" % filter_)
+
+
 if __name__ == "__main__":
   testutils.GanetiTestProgram()