Commit 7578ab0a authored by Michael Hanselmann's avatar Michael Hanselmann

qlang: Add parser for query filter language

With this parser, command line utilities will be able to provide filters
through query2 in a simplistic language. Example filters:

  name == "node3.example.com"
  master or (name == "node4.example.com")
  be/memory == 128 and name =~ /^web/i
  "inst1.example.com" in sinst_list
  status != "up"
  not master

Parts of the syntax came from Python, others from Perl. Documentation
will be added in follow-up patches.
Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent 6a062ff9
...@@ -28,7 +28,8 @@ Before installing, please verify that you have the following programs: ...@@ -28,7 +28,8 @@ Before installing, please verify that you have the following programs:
- `Python <http://www.python.org/>`_, version 2.4 or above, not 3.0 - `Python <http://www.python.org/>`_, version 2.4 or above, not 3.0
- `Python OpenSSL bindings <http://pyopenssl.sourceforge.net/>`_ - `Python OpenSSL bindings <http://pyopenssl.sourceforge.net/>`_
- `simplejson Python module <http://code.google.com/p/simplejson/>`_ - `simplejson Python module <http://code.google.com/p/simplejson/>`_
- `pyparsing Python module <http://pyparsing.wikispaces.com/>`_ - `pyparsing Python module <http://pyparsing.wikispaces.com/>`_, version
1.4.6 or above
- `pyinotify Python module <http://trac.dbzteam.org/pyinotify/>`_ - `pyinotify Python module <http://trac.dbzteam.org/pyinotify/>`_
- `PycURL Python module <http://pycurl.sourceforge.net/>`_ - `PycURL Python module <http://pycurl.sourceforge.net/>`_
- `ctypes Python module - `ctypes Python module
......
...@@ -16,6 +16,8 @@ Version 2.5.0 beta1 ...@@ -16,6 +16,8 @@ Version 2.5.0 beta1
- Support for the undocumented and deprecated RAPI instance creation - Support for the undocumented and deprecated RAPI instance creation
request format version 0 has been dropped. Use version 1, supported request format version 0 has been dropped. Use version 1, supported
since Ganeti 2.1.3 and :doc:`documented <rapi>`, instead. since Ganeti 2.1.3 and :doc:`documented <rapi>`, instead.
- Pyparsing 1.4.6 or above is required, see :doc:`installation
documentation <install>`
Version 2.4.1 Version 2.4.1
......
...@@ -391,6 +391,24 @@ class LuxiError(GenericError): ...@@ -391,6 +391,24 @@ class LuxiError(GenericError):
""" """
class QueryFilterParseError(ParseError):
"""Error while parsing query filter.
"""
def GetDetails(self):
"""Returns a list of strings with details about the error.
"""
try:
(_, inner) = self.args
except IndexError:
return None
return [str(inner.line),
(" " * (inner.column - 1)) + "^",
str(inner)]
# errors should be added above # errors should be added above
......
...@@ -31,6 +31,13 @@ converted to callable functions by L{query._CompileFilter}. ...@@ -31,6 +31,13 @@ converted to callable functions by L{query._CompileFilter}.
""" """
import re
import pyparsing as pyp
from ganeti import errors
# Logic operators with one or more operands, each of which is a filter on its # Logic operators with one or more operands, each of which is a filter on its
# own # own
OP_OR = "|" OP_OR = "|"
...@@ -61,3 +68,155 @@ def MakeSimpleFilter(namefield, values): ...@@ -61,3 +68,155 @@ def MakeSimpleFilter(namefield, values):
return [OP_OR] + [[OP_EQUAL, namefield, i] for i in values] return [OP_OR] + [[OP_EQUAL, namefield, i] for i in values]
return None return None
def _ConvertLogicOp(op):
"""Creates parsing action function for logic operator.
@type op: string
@param op: Operator for data structure, e.g. L{OP_AND}
"""
def fn(toks):
"""Converts parser tokens to query operator structure.
@rtype: list
@return: Query operator structure, e.g. C{[OP_AND, ["=", "foo", "bar"]]}
"""
operands = toks[0]
if len(operands) == 1:
return operands[0]
# Build query operator structure
return [[op] + operands.asList()]
return fn
_KNOWN_REGEXP_DELIM = "/#^|"
_KNOWN_REGEXP_FLAGS = frozenset("si")
def _ConvertRegexpValue(_, loc, toks):
"""Regular expression value for condition.
"""
(regexp, flags) = toks[0]
# Ensure only whitelisted flags are used
unknown_flags = (frozenset(flags) - _KNOWN_REGEXP_FLAGS)
if unknown_flags:
raise pyp.ParseFatalException("Unknown regular expression flags: '%s'" %
"".join(unknown_flags), loc)
if flags:
re_flags = "(?%s)" % "".join(sorted(flags))
else:
re_flags = ""
re_cond = re_flags + regexp
# Test if valid
try:
re.compile(re_cond)
except re.error, err:
raise pyp.ParseFatalException("Invalid regular expression (%s)" % err, loc)
return [re_cond]
def BuildFilterParser():
"""Builds a parser for query filter strings.
@rtype: pyparsing.ParserElement
"""
field_name = pyp.Word(pyp.alphas, pyp.alphanums + "_/.")
# Integer
num_sign = pyp.Word("-+", exact=1)
number = pyp.Combine(pyp.Optional(num_sign) + pyp.Word(pyp.nums))
number.setParseAction(lambda toks: int(toks[0]))
# Right-hand-side value
rval = (number | pyp.quotedString.setParseAction(pyp.removeQuotes))
# Boolean condition
bool_cond = field_name.copy()
bool_cond.setParseAction(lambda (fname, ): [[OP_TRUE, fname]])
# Simple binary conditions
binopstbl = {
"==": OP_EQUAL,
"!=": OP_NOT_EQUAL,
}
binary_cond = (field_name + pyp.oneOf(binopstbl.keys()) + rval)
binary_cond.setParseAction(lambda (lhs, op, rhs): [[binopstbl[op], lhs, rhs]])
# "in" condition
in_cond = (rval + pyp.Suppress("in") + field_name)
in_cond.setParseAction(lambda (value, field): [[OP_CONTAINS, field, value]])
# "not in" condition
not_in_cond = (rval + pyp.Suppress("not") + pyp.Suppress("in") + field_name)
not_in_cond.setParseAction(lambda (value, field): [[OP_NOT, [OP_CONTAINS,
field, value]]])
# Regular expression, e.g. m/foobar/i
regexp_val = pyp.Group(pyp.Optional("m").suppress() +
pyp.MatchFirst([pyp.QuotedString(i, escChar="\\")
for i in _KNOWN_REGEXP_DELIM]) +
pyp.Optional(pyp.Word(pyp.alphas), default=""))
regexp_val.setParseAction(_ConvertRegexpValue)
regexp_cond = (field_name + pyp.Suppress("=~") + regexp_val)
regexp_cond.setParseAction(lambda (field, value): [[OP_REGEXP, field, value]])
not_regexp_cond = (field_name + pyp.Suppress("!~") + regexp_val)
not_regexp_cond.setParseAction(lambda (field, value):
[[OP_NOT, [OP_REGEXP, field, value]]])
# All possible conditions
condition = (binary_cond ^ bool_cond ^
in_cond ^ not_in_cond ^
regexp_cond ^ not_regexp_cond)
# Associativity operators
filter_expr = pyp.operatorPrecedence(condition, [
(pyp.Keyword("not").suppress(), 1, pyp.opAssoc.RIGHT,
lambda toks: [[OP_NOT, toks[0][0]]]),
(pyp.Keyword("and").suppress(), 2, pyp.opAssoc.LEFT,
_ConvertLogicOp(OP_AND)),
(pyp.Keyword("or").suppress(), 2, pyp.opAssoc.LEFT,
_ConvertLogicOp(OP_OR)),
])
parser = pyp.StringStart() + filter_expr + pyp.StringEnd()
parser.parseWithTabs()
# Originally C{parser.validate} was called here, but there seems to be some
# issue causing it to fail whenever the "not" operator is included above.
return parser
def ParseFilter(text, parser=None):
"""Parses a query filter.
@type text: string
@param text: Query filter
@type parser: pyparsing.ParserElement
@param parser: Pyparsing object
@rtype: list
"""
if parser is None:
parser = BuildFilterParser()
try:
return parser.parseString(text)[0]
except pyp.ParseBaseException, err:
raise errors.QueryFilterParseError("Failed to parse query filter"
" '%s': %s" % (text, err), err)
...@@ -26,6 +26,7 @@ import unittest ...@@ -26,6 +26,7 @@ import unittest
from ganeti import utils from ganeti import utils
from ganeti import errors from ganeti import errors
from ganeti import qlang from ganeti import qlang
from ganeti import query
import testutils import testutils
...@@ -47,5 +48,123 @@ class TestMakeSimpleFilter(unittest.TestCase): ...@@ -47,5 +48,123 @@ class TestMakeSimpleFilter(unittest.TestCase):
["|", ["=", "xyz", "a"], ["=", "xyz", "b"], ["=", "xyz", "c"]]) ["|", ["=", "xyz", "a"], ["=", "xyz", "b"], ["=", "xyz", "c"]])
class TestParseFilter(unittest.TestCase):
def setUp(self):
self.parser = qlang.BuildFilterParser()
def _Test(self, filter_, expected):
self.assertEqual(qlang.ParseFilter(filter_, parser=self.parser), expected)
def test(self):
self._Test("name==\"foobar\"", [qlang.OP_EQUAL, "name", "foobar"])
self._Test("name=='foobar'", [qlang.OP_EQUAL, "name", "foobar"])
self._Test("valA==1 and valB==2 or valC==3",
[qlang.OP_OR,
[qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1],
[qlang.OP_EQUAL, "valB", 2]],
[qlang.OP_EQUAL, "valC", 3]])
self._Test(("(name\n==\"foobar\") and (xyz==\"va)ue\" and k == 256 or"
" x ==\t\"y\"\n) and mc"),
[qlang.OP_AND,
[qlang.OP_EQUAL, "name", "foobar"],
[qlang.OP_OR,
[qlang.OP_AND, [qlang.OP_EQUAL, "xyz", "va)ue"],
[qlang.OP_EQUAL, "k", 256]],
[qlang.OP_EQUAL, "x", "y"]],
[qlang.OP_TRUE, "mc"]])
self._Test("(xyz==\"v\" or k == 256 and x == \"y\")",
[qlang.OP_OR,
[qlang.OP_EQUAL, "xyz", "v"],
[qlang.OP_AND, [qlang.OP_EQUAL, "k", 256],
[qlang.OP_EQUAL, "x", "y"]]])
self._Test("valA==1 and valB==2 and valC==3",
[qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1],
[qlang.OP_EQUAL, "valB", 2],
[qlang.OP_EQUAL, "valC", 3]])
self._Test("master or field",
[qlang.OP_OR, [qlang.OP_TRUE, "master"],
[qlang.OP_TRUE, "field"]])
self._Test("mem == 128", [qlang.OP_EQUAL, "mem", 128])
self._Test("negfield != -1", [qlang.OP_NOT_EQUAL, "negfield", -1])
self._Test("master", [qlang.OP_TRUE, "master"])
self._Test("not master", [qlang.OP_NOT, [qlang.OP_TRUE, "master"]])
for op in ["not", "and", "or"]:
self._Test("%sxyz" % op, [qlang.OP_TRUE, "%sxyz" % op])
self._Test("not %sxyz" % op,
[qlang.OP_NOT, [qlang.OP_TRUE, "%sxyz" % op]])
self._Test(" not \t%sfoo" % op,
[qlang.OP_NOT, [qlang.OP_TRUE, "%sfoo" % op]])
self._Test("%sname =~ m/abc/" % op,
[qlang.OP_REGEXP, "%sname" % op, "abc"])
self._Test("master and not other",
[qlang.OP_AND, [qlang.OP_TRUE, "master"],
[qlang.OP_NOT, [qlang.OP_TRUE, "other"]]])
self._Test("not (master or other == 4)",
[qlang.OP_NOT,
[qlang.OP_OR, [qlang.OP_TRUE, "master"],
[qlang.OP_EQUAL, "other", 4]]])
self._Test("some==\"val\\\"ue\"", [qlang.OP_EQUAL, "some", "val\\\"ue"])
self._Test("123 in ips", [qlang.OP_CONTAINS, "ips", 123])
self._Test("99 not in ips", [qlang.OP_NOT, [qlang.OP_CONTAINS, "ips", 99]])
self._Test("\"a\" in valA and \"b\" not in valB",
[qlang.OP_AND, [qlang.OP_CONTAINS, "valA", "a"],
[qlang.OP_NOT, [qlang.OP_CONTAINS, "valB", "b"]]])
self._Test("name =~ m/test/", [qlang.OP_REGEXP, "name", "test"])
self._Test("name =~ m/^node.*example.com$/i",
[qlang.OP_REGEXP, "name", "(?i)^node.*example.com$"])
self._Test("(name =~ m/^node.*example.com$/s and master) or pip =~ |^3.*|",
[qlang.OP_OR,
[qlang.OP_AND,
[qlang.OP_REGEXP, "name", "(?s)^node.*example.com$"],
[qlang.OP_TRUE, "master"]],
[qlang.OP_REGEXP, "pip", "^3.*"]])
for flags in ["si", "is", "ssss", "iiiisiii"]:
self._Test("name =~ m/gi/%s" % flags,
[qlang.OP_REGEXP, "name", "(?%s)gi" % "".join(sorted(flags))])
for i in qlang._KNOWN_REGEXP_DELIM:
self._Test("name =~ m%stest%s" % (i, i),
[qlang.OP_REGEXP, "name", "test"])
self._Test("name !~ m%stest%s" % (i, i),
[qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]])
self._Test("not\tname =~ m%stest%s" % (i, i),
[qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]])
self._Test("notname =~ m%stest%s" % (i, i),
[qlang.OP_REGEXP, "notname", "test"])
def testAllFields(self):
for name in frozenset(i for d in query.ALL_FIELD_LISTS for i in d.keys()):
self._Test("%s == \"value\"" % name, [qlang.OP_EQUAL, name, "value"])
def testError(self):
# Invalid field names, meaning no boolean check is done
tests = ["#invalid!filter#", "m/x/,"]
# Unknown regexp flag
tests.append("name=~m#a#g")
# Incomplete regexp group
tests.append("name=~^[^")
# Valid flag, but in uppercase
tests.append("asdf =~ m|abc|I")
# Non-matching regexp delimiters
tests.append("name =~ /foobarbaz#")
for filter_ in tests:
try:
qlang.ParseFilter(filter_, parser=self.parser)
except errors.QueryFilterParseError, err:
self.assertEqual(len(err.GetDetails()), 3)
else:
self.fail("Invalid filter '%s' did not raise exception" % filter_)
if __name__ == "__main__": if __name__ == "__main__":
testutils.GanetiTestProgram() testutils.GanetiTestProgram()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment