Commit d8f5462a authored by Niklas Hambuechen's avatar Niklas Hambuechen

Add filter management to Luxid (RAPI and CLI interfaces)

This implements the management (add, remove, replace) of job filter rules
as specified by `doc/design-optables.rst`; it does not yet implement
the filtering logic.

This commit also includes the implementation of filters in the RAPI client.py
and its tests because the client tests check that if all available RAPI
resources are being used by the client, and the only way to make them
be used is to write tests for them.
Signed-off-by: default avatarNiklas Hambuechen <niklash@google.com>
Reviewed-by: default avatarKlaus Aehlig <aehlig@google.com>
parent df9cabde
......@@ -462,7 +462,8 @@ client_PYTHON = \
lib/client/gnt_node.py \
lib/client/gnt_network.py \
lib/client/gnt_os.py \
lib/client/gnt_storage.py
lib/client/gnt_storage.py \
lib/client/gnt_filter.py
cmdlib_PYTHON = \
lib/cmdlib/__init__.py \
......@@ -892,6 +893,7 @@ HS_LIB_SRCS = \
src/Ganeti/Query/Exec.hs \
src/Ganeti/Query/Export.hs \
src/Ganeti/Query/Filter.hs \
src/Ganeti/Query/FilterRules.hs \
src/Ganeti/Query/Group.hs \
src/Ganeti/Query/Instance.hs \
src/Ganeti/Query/Job.hs \
......@@ -1156,7 +1158,8 @@ gnt_scripts = \
scripts/gnt-network \
scripts/gnt-node \
scripts/gnt-os \
scripts/gnt-storage
scripts/gnt-storage \
scripts/gnt-filter
gnt_scripts_basenames = \
$(patsubst scripts/%,%,$(patsubst daemons/%,%,$(gnt_scripts) $(gnt_python_sbin_SCRIPTS)))
......@@ -1457,6 +1460,7 @@ man_MANS = \
man/gnt-node.8 \
man/gnt-os.8 \
man/gnt-storage.8 \
man/gnt-filter.8 \
man/hail.1 \
man/harep.1 \
man/hbal.1 \
......
......@@ -404,6 +404,8 @@ class CompletionWriter(object):
WriteCompReply(sw, "-W \"$(_ganeti_os)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_EXTSTORAGE:
WriteCompReply(sw, "-W \"$(_ganeti_extstorage)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_FILTER:
WriteCompReply(sw, "-W \"$(_ganeti_filter)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_IALLOCATOR:
WriteCompReply(sw, "-W \"$(_ganeti_iallocator)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_NODEGROUP:
......@@ -516,6 +518,8 @@ class CompletionWriter(object):
choices = "$(_ganeti_os)"
elif isinstance(arg, cli.ArgExtStorage):
choices = "$(_ganeti_extstorage)"
elif isinstance(arg, cli.ArgFilter):
choices = "$(_ganeti_filter)"
elif isinstance(arg, cli.ArgFile):
choices = ""
compgenargs.append("-f")
......
......@@ -474,6 +474,161 @@ features:
a new-style result (see resource description)
.. _rapi-res-filters:
``/2/filters``
+++++++++++++++
The filters resource.
.. rapi_resource_details:: /2/filters
.. _rapi-res-filters+get:
``GET``
~~~~~~~
Returns a list of all existing filters.
Example::
[
{
"id": "8b53f7de-f8e2-4470-99bd-1efe746e434f",
"uri": "/2/filters/8b53f7de-f8e2-4470-99bd-1efe746e434f"
},
{
"id": "b296f0c9-4809-46a8-b928-5ccf7720fa8c",
"uri": "/2/filters/b296f0c9-4809-46a8-b928-5ccf7720fa8c"
}
]
If the optional bool *bulk* argument is provided and set to a true value
(i.e ``?bulk=1``), the output contains detailed information about filters
as a list.
Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.FILTER_RULE_FIELDS))`.
Example::
[
{
"uuid": "8b53f7de-f8e2-4470-99bd-1efe746e434f",
"watermark": 12534,
"reason_trail": [
["luxid", "someFilterReason", 1409249801259897000]
],
"priority": 0,
"action": "REJECT",
"predicates": [
["jobid", [">", "id", "watermark"]]
]
},
{
"uuid": "b296f0c9-4809-46a8-b928-5ccf7720fa8c",
"watermark": 12534,
"reason_trail": [
["luxid", "someFilterReason", 1409249917268978000]
],
"priority": 1,
"action": "REJECT",
"predicates": [
["opcode", ["=", "OP_ID", "OP_INSTANCE_CREATE"]]
]
}
]
.. _rapi-res-filters+post:
``POST``
~~~~~~~~
Creates a filter.
Body parameters:
``priority`` (int, defaults to ``0``)
Must be non-negative. Lower numbers mean higher filter priority.
``predicates`` (list, defaults to ``[]``)
The first element is the name (``str``) of the predicate and the
rest are parameters suitable for that predicate.
Most predicates take a single parameter: A boolean expression
in the Ganeti query language.
``action`` (defaults to ``"CONTINUE"``)
The effect of the filter. Can be one of ``"ACCEPT"``, ``"PAUSE"``,
``"REJECT"``, ``"CONTINUE"`` and ``["RATE_LIMIT", n]``, where ``n``
is a positive integer.
``reason`` (list, defaults to ``[]``)
An initial reason trail for this filter. Each element in this list
is a list with 3 elements: ``[source, reason, timestamp]``, where
``source`` and ``reason`` are strings and ``timestamp`` is a time
since the UNIX epoch in nanoseconds as an integer.
Returns:
A filter UUID (``str``) that can be used for accessing the filter later.
.. _rapi-res-filters-filter_uuid:
``/2/filters/[filter_uuid]``
++++++++++++++++++++++++++++++
Returns information about a filter.
.. rapi_resource_details:: /2/filters/[filter_uuid]
.. _rapi-res-filters-filter_uuid+get:
``GET``
~~~~~~~
Returns information about a filter, similar to the bulk output from
the filter list.
Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.FILTER_RULE_FIELDS))`.
.. _rapi-res-filters-filter_uuid+put:
``PUT``
~~~~~~~
Replaces a filter with given UUID, or creates it with the given UUID
if it doesn't already exist.
Body parameters:
All parameters for adding a new filter via ``POST``, plus the following:
``uuid``: (string)
The UUID of the filter to replace or create.
Returns:
The filter UUID (``str``) of the replaced or created filter.
This will be the ``uuid`` body parameter if given, and a freshly generated
UUID otherwise.
.. _rapi-res-filters-filter_uuid+delete:
``DELETE``
~~~~~~~~~~
Deletes a filter.
Returns:
``None``
.. _rapi-res-modify:
``/2/modify``
......
......@@ -834,6 +834,7 @@ def InitConfig(version, cluster_config, master_node_config,
instances={},
networks={},
disks={},
filters={},
serial_no=1,
ctime=now, mtime=now)
utils.WriteFile(cfg_file,
......
......@@ -518,7 +518,8 @@ class _RapiHandlersForDocsHelper(object):
resources = \
rapi.connector.GetHandlers("[node_name]", "[instance_name]",
"[group_name]", "[network_name]", "[job_id]",
"[disk_index]", "[resource]",
"[disk_index]", "[filter_uuid]",
"[resource]",
translate=cls._TranslateResourceUri)
return resources
......
......@@ -34,10 +34,12 @@
import sys
import textwrap
import os.path
import re
import time
import logging
import errno
import itertools
import simplejson
import shlex
from cStringIO import StringIO
......@@ -297,12 +299,14 @@ __all__ = [
"ARGS_MANY_NODES",
"ARGS_MANY_GROUPS",
"ARGS_MANY_NETWORKS",
"ARGS_MANY_FILTERS",
"ARGS_NONE",
"ARGS_ONE_INSTANCE",
"ARGS_ONE_NODE",
"ARGS_ONE_GROUP",
"ARGS_ONE_OS",
"ARGS_ONE_NETWORK",
"ARGS_ONE_FILTER",
"ArgChoice",
"ArgCommand",
"ArgFile",
......@@ -314,6 +318,7 @@ __all__ = [
"ArgNode",
"ArgOs",
"ArgExtStorage",
"ArgFilter",
"ArgSuggest",
"ArgUnknown",
"OPT_COMPL_INST_ADD_NODES",
......@@ -325,6 +330,7 @@ __all__ = [
"OPT_COMPL_ONE_NETWORK",
"OPT_COMPL_ONE_OS",
"OPT_COMPL_ONE_EXTSTORAGE",
"OPT_COMPL_ONE_FILTER",
"cli_option",
"FixHvParams",
"SplitNodeOption",
......@@ -483,16 +489,24 @@ class ArgExtStorage(_Argument):
"""
class ArgFilter(_Argument):
"""Filter UUID argument.
"""
ARGS_NONE = []
ARGS_MANY_INSTANCES = [ArgInstance()]
ARGS_MANY_NETWORKS = [ArgNetwork()]
ARGS_MANY_NODES = [ArgNode()]
ARGS_MANY_GROUPS = [ArgGroup()]
ARGS_MANY_FILTERS = [ArgFilter()]
ARGS_ONE_INSTANCE = [ArgInstance(min=1, max=1)]
ARGS_ONE_NETWORK = [ArgNetwork(min=1, max=1)]
ARGS_ONE_NODE = [ArgNode(min=1, max=1)]
ARGS_ONE_GROUP = [ArgGroup(min=1, max=1)]
ARGS_ONE_OS = [ArgOs(min=1, max=1)]
ARGS_ONE_FILTER = [ArgFilter(min=1, max=1)]
def _ExtractTagsObject(opts, args):
......@@ -785,6 +799,32 @@ def check_maybefloat(option, opt, value): # pylint: disable=W0613
return float(value)
def check_json(option, opt, value): # pylint: disable=W0613
"""Custom parser for JSON arguments.
Takes a string containing JSON, returns a Python object.
"""
return simplejson.loads(value)
def check_filteraction(option, opt, value): # pylint: disable=W0613
"""Custom parser for filter rule actions.
Takes a string, returns an action as a Python object (list or string).
The string "RATE_LIMIT n" becomes `["RATE_LIMIT", n]`.
All other strings stay as they are.
"""
match = re.match(r"RATE_LIMIT\s+(\d+)", value)
if match:
n = int(match.group(1))
return ["RATE_LIMIT", n]
else:
return value
# completion_suggestion is normally a list. Using numeric values not evaluating
# to False for dynamic completion.
(OPT_COMPL_MANY_NODES,
......@@ -792,10 +832,11 @@ def check_maybefloat(option, opt, value): # pylint: disable=W0613
OPT_COMPL_ONE_INSTANCE,
OPT_COMPL_ONE_OS,
OPT_COMPL_ONE_EXTSTORAGE,
OPT_COMPL_ONE_FILTER,
OPT_COMPL_ONE_IALLOCATOR,
OPT_COMPL_ONE_NETWORK,
OPT_COMPL_INST_ADD_NODES,
OPT_COMPL_ONE_NODEGROUP) = range(100, 109)
OPT_COMPL_ONE_NODEGROUP) = range(100, 110)
OPT_COMPL_ALL = compat.UniqueFrozenset([
OPT_COMPL_MANY_NODES,
......@@ -803,6 +844,7 @@ OPT_COMPL_ALL = compat.UniqueFrozenset([
OPT_COMPL_ONE_INSTANCE,
OPT_COMPL_ONE_OS,
OPT_COMPL_ONE_EXTSTORAGE,
OPT_COMPL_ONE_FILTER,
OPT_COMPL_ONE_IALLOCATOR,
OPT_COMPL_ONE_NETWORK,
OPT_COMPL_INST_ADD_NODES,
......@@ -826,6 +868,8 @@ class CliOption(Option):
"bool",
"list",
"maybefloat",
"json",
"filteraction",
)
TYPE_CHECKER = Option.TYPE_CHECKER.copy()
TYPE_CHECKER["multilistidentkeyval"] = check_multilist_ident_key_val
......@@ -836,6 +880,8 @@ class CliOption(Option):
TYPE_CHECKER["bool"] = check_bool
TYPE_CHECKER["list"] = check_list
TYPE_CHECKER["maybefloat"] = check_maybefloat
TYPE_CHECKER["json"] = check_json
TYPE_CHECKER["filteraction"] = check_filteraction
# optparse.py sets make_option, so we do it for our own option class, too
......@@ -1623,7 +1669,7 @@ FAILURE_ONLY_OPT = cli_option("--failure-only", default=False,
help=("Hide successful results and show failures"
" only (determined by the exit code)"))
REASON_OPT = cli_option("--reason", default=None,
REASON_OPT = cli_option("--reason", default=[],
help="The reason for executing the command")
......
#
#
# Copyright (C) 2014 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Job filter rule commands"""
# pylint: disable=W0401,W0614
# W0401: Wildcard import ganeti.cli
# W0614: Unused import %s from wildcard import (since we need cli)
from ganeti.cli import *
from ganeti import constants
from ganeti import utils
#: default list of fields for L{ListFilters}
_LIST_DEF_FIELDS = ["uuid", "watermark", "priority",
"predicates", "action", "reason_trail"]
def AddFilter(opts, args):
"""Add a job filter rule.
@param opts: the command line options selected by the user
@type args: list
@param args: should be an empty list
@rtype: int
@return: the desired exit code
"""
assert args == []
reason = []
if opts.reason:
reason = [(constants.OPCODE_REASON_SRC_USER,
opts.reason,
utils.EpochNano())]
cl = GetClient()
result = cl.ReplaceFilter(None, opts.priority, opts.predicates, opts.action,
reason)
print result # Prints the UUID of the replaced/created filter
def ListFilters(opts, args):
"""List job filter rules and their properties.
@param opts: the command line options selected by the user
@type args: list
@param args: filters to list, or empty for all
@rtype: int
@return: the desired exit code
"""
desired_fields = ParseFields(opts.output, _LIST_DEF_FIELDS)
cl = GetClient()
return GenericList(constants.QR_FILTER, desired_fields, args, None,
opts.separator, not opts.no_headers,
verbose=opts.verbose, cl=cl, namefield="uuid")
def ListFilterFields(opts, args):
"""List filter rule fields.
@param opts: the command line options selected by the user
@type args: list
@param args: fields to list, or empty for all
@rtype: int
@return: the desired exit code
"""
cl = GetClient()
return GenericListFields(constants.QR_FILTER, args, opts.separator,
not opts.no_headers, cl=cl)
def ReplaceFilter(opts, args):
"""Replaces a job filter rule with the given UUID, or creates it, if it
doesn't exist already.
@param opts: the command line options selected by the user
@type args: list
@param args: should contain only one element, the UUID of the filter
@rtype: int
@return: the desired exit code
"""
(uuid,) = args
reason = []
if opts.reason:
reason = [(constants.OPCODE_REASON_SRC_USER,
opts.reason,
utils.EpochNano())]
cl = GetClient()
result = cl.ReplaceFilter(uuid,
priority=opts.priority,
predicates=opts.predicates,
action=opts.action,
reason=reason)
print result # Prints the UUID of the replaced/created filter
return 0
def ShowFilter(_, args):
"""Show filter rule details.
@type args: list
@param args: should either be an empty list, in which case
we show information about all filters, or should contain
a list of filter UUIDs to be queried for information
@rtype: int
@return: the desired exit code
"""
cl = GetClient()
result = cl.QueryFilters(fields=["uuid", "watermark", "priority",
"predicates", "action", "reason_trail"],
uuids=args)
for (uuid, watermark, priority, predicates, action, reason_trail) in result:
ToStdout("UUID: %s", uuid)
ToStdout(" Watermark: %s", watermark)
ToStdout(" Priority: %s", priority)
ToStdout(" Predicates: %s", predicates)
ToStdout(" Action: %s", action)
ToStdout(" Reason trail: %s", reason_trail)
return 0
def DeleteFilter(_, args):
"""Remove a job filter rule.
@type args: list
@param args: a list of length 1 with the UUID of the filter to remove
@rtype: int
@return: the desired exit code
"""
(uuid,) = args
cl = GetClient()
result = cl.DeleteFilter(uuid)
assert result is None
return 0
FILTER_PRIORITY_OPT = \
cli_option("--priority",
dest="priority", action="store", default=0, type="int",
help="Priority for filter processing")
FILTER_PREDICATES_OPT = \
cli_option("--predicates",
dest="predicates", action="store", default=[], type="json",
help="List of predicates in the Ganeti query language,"
" given as a JSON list.")
FILTER_ACTION_OPT = \
cli_option("--action",
dest="action", action="store", default="CONTINUE",
type="filteraction",
help="The effect of the filter. Can be one of 'ACCEPT',"
" 'PAUSE', 'REJECT', 'CONTINUE' and '[RATE_LIMIT, n]',"
" where n is a positive integer.")
commands = {
"add": (
AddFilter, ARGS_NONE,
[FILTER_PRIORITY_OPT, FILTER_PREDICATES_OPT, FILTER_ACTION_OPT],
"",
"Adds a new filter rule"),
"list": (
ListFilters, ARGS_MANY_FILTERS,
[NOHDR_OPT, SEP_OPT, FIELDS_OPT, VERBOSE_OPT],
"[<filter_uuid>...]",
"Lists the job filter rules. The available fields can be shown"
" using the \"list-fields\" command (see the man page for details)."
" The default list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS)),
"list-fields": (
ListFilterFields, [ArgUnknown()],
[NOHDR_OPT, SEP_OPT],
"[fields...]",
"Lists all available fields for filters"),
"info": (
ShowFilter, ARGS_MANY_FILTERS,
[],
"[<filter_uuid>...]",
"Shows information about the filter(s)"),
"replace": (
ReplaceFilter, ARGS_ONE_FILTER,
[FILTER_PRIORITY_OPT, FILTER_PREDICATES_OPT, FILTER_ACTION_OPT],
"<filter_uuid>",
"Replaces a filter"),
"delete": (
DeleteFilter, ARGS_ONE_FILTER,
[],
"<filter_uuid>",
"Removes a filter"),
}
def Main():
return GenericMain(commands)
......@@ -62,6 +62,9 @@ REQ_AUTO_ARCHIVE_JOBS = constants.LUXI_REQ_AUTO_ARCHIVE_JOBS
REQ_QUERY = constants.LUXI_REQ_QUERY
REQ_QUERY_FIELDS = constants.LUXI_REQ_QUERY_FIELDS
REQ_QUERY_JOBS = constants.LUXI_REQ_QUERY_JOBS
REQ_QUERY_FILTERS = constants.LUXI_REQ_QUERY_FILTERS
REQ_REPLACE_FILTER = constants.LUXI_REQ_REPLACE_FILTER
REQ_DELETE_FILTER = constants.LUXI_REQ_DELETE_FILTER
REQ_QUERY_INSTANCES = constants.LUXI_REQ_QUERY_INSTANCES
REQ_QUERY_NODES = constants.LUXI_REQ_QUERY_NODES
REQ_QUERY_GROUPS = constants.LUXI_REQ_QUERY_GROUPS
......@@ -214,6 +217,16 @@ class Client(cl.AbstractClient):
def QueryJobs(self, job_ids, fields):
return self.CallMethod(REQ_QUERY_JOBS, (job_ids, fields))
def QueryFilters(self, uuids, fields):
return self.CallMethod(REQ_QUERY_FILTERS, (uuids, fields))
def ReplaceFilter(self, uuid, priority, predicates, action, reason):
return self.CallMethod(REQ_REPLACE_FILTER,
(uuid, priority, predicates, action, reason))
def DeleteFilter(self, uuid):
return self.CallMethod(REQ_DELETE_FILTER, (uuid, ))
def QueryInstances(self, names, fields, use_locking):
return self.CallMethod(REQ_QUERY_INSTANCES, (names, fields, use_locking))
......
......@@ -62,7 +62,8 @@ from socket import AF_INET
__all__ = ["ConfigObject", "ConfigData", "NIC", "Disk", "Instance",
"OS", "Node", "NodeGroup", "Cluster", "FillDict", "Network"]
"OS", "Node", "NodeGroup", "Cluster", "FillDict", "Network",
"Filter"]
_TIMESTAMPS = ["ctime", "mtime"]
_UUID = ["uuid"]
......@@ -415,6 +416,7 @@ class ConfigData(ConfigObject):
"instances",
"networks",
"disks",
"filters",
"serial_no",
] + _TIMESTAMPS
......@@ -427,7 +429,8 @@ class ConfigData(ConfigObject):
"""