From 10b207d43a03d4fb800910a1185cf2635d7f5386 Mon Sep 17 00:00:00 2001 From: Oleksiy Mishchenko <oleksiy@google.com> Date: Tue, 22 Jul 2008 13:33:13 +0000 Subject: [PATCH] Split RAPI resources to pieces Reviewed-by: iustinp --- Makefile.am | 7 +- doc/build-rapi-resources-doc | 8 +- lib/rapi/RESTHTTPServer.py | 8 +- lib/rapi/baseresources.py | 0 lib/rapi/connector.py | 141 ++++++++++++ lib/rapi/{resources.py => rlib1.py} | 286 +++---------------------- lib/rapi/rlib2.py | 154 +++++++++++++ test/ganeti.rapi.resources_unittest.py | 18 +- 8 files changed, 346 insertions(+), 276 deletions(-) create mode 100644 lib/rapi/baseresources.py create mode 100644 lib/rapi/connector.py rename lib/rapi/{resources.py => rlib1.py} (55%) create mode 100644 lib/rapi/rlib2.py diff --git a/Makefile.am b/Makefile.am index 97dd2328b..07332a8e0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -94,7 +94,10 @@ rapi_PYTHON = \ lib/rapi/__init__.py \ lib/rapi/RESTHTTPServer.py \ lib/rapi/httperror.py \ - lib/rapi/resources.py + lib/rapi/baserlib.py \ + lib/rapi/connector.py \ + lib/rapi/rlib1.py \ + lib/rapi/rlib2.py docsgml = \ @@ -226,7 +229,7 @@ doc/%.html: doc/%.in $(DOCBOOK_WRAPPER) doc/rapi.pdf doc/rapi.html: doc/rapi-resources.sgml -doc/rapi-resources.sgml: $(BUILD_RAPI_RESOURCE_DOC) lib/rapi/resources.py +doc/rapi-resources.sgml: $(BUILD_RAPI_RESOURCE_DOC) lib/rapi/connector.py PYTHONPATH=.:$(top_builddir) $(BUILD_RAPI_RESOURCE_DOC) > $@ || rm -f $@ man/%.7: man/%.in man/footer.sgml $(DOCBOOK_WRAPPER) diff --git a/doc/build-rapi-resources-doc b/doc/build-rapi-resources-doc index c73e51883..32d1a463d 100755 --- a/doc/build-rapi-resources-doc +++ b/doc/build-rapi-resources-doc @@ -26,7 +26,9 @@ import re import cgi import inspect -from ganeti.rapi import resources +from ganeti.rapi import rlib1 +from ganeti.rapi import rlib2 +from ganeti.rapi import connector CHECKED_COMMANDS = ["GET", "POST", "PUT", "DELETE"] @@ -34,9 +36,9 @@ CHECKED_COMMANDS = ["GET", "POST", "PUT", "DELETE"] def main(): # Get list of all resources - all = list(resources._CONNECTOR.itervalues()) + all = list(connector.CONNECTOR.itervalues()) - # Sort resources by URI + # Sort rlib1 by URI all.sort(cmp=lambda a, b: cmp(a.DOC_URI, b.DOC_URI)) print "<!-- Automatically generated, do not edit -->" diff --git a/lib/rapi/RESTHTTPServer.py b/lib/rapi/RESTHTTPServer.py index c480ff877..bbea1a446 100644 --- a/lib/rapi/RESTHTTPServer.py +++ b/lib/rapi/RESTHTTPServer.py @@ -20,9 +20,10 @@ """ -import socket import BaseHTTPServer import OpenSSL +import re +import socket import time from ganeti import constants @@ -30,7 +31,8 @@ from ganeti import errors from ganeti import logger from ganeti import rpc from ganeti import serializer -from ganeti.rapi import resources + +from ganeti.rapi import connector from ganeti.rapi import httperror @@ -158,7 +160,7 @@ class RESTRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.connection = self.request self.rfile = socket._fileobject(self.request, "rb", self.rbufsize) self.wfile = socket._fileobject(self.request, "wb", self.wbufsize) - self._resmap = resources.Mapper() + self._resmap = connector.Mapper() def handle_one_request(self): """Handle a single REST request. diff --git a/lib/rapi/baseresources.py b/lib/rapi/baseresources.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/rapi/connector.py b/lib/rapi/connector.py new file mode 100644 index 000000000..782065469 --- /dev/null +++ b/lib/rapi/connector.py @@ -0,0 +1,141 @@ +# +# + +# Copyright (C) 2006, 2007, 2008 Google Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +"""Remote API connection map. + +""" + +import cgi +import re + +from ganeti import constants + +from ganeti.rapi import baserlib +from ganeti.rapi import httperror +from ganeti.rapi import rlib1 +from ganeti.rapi import rlib2 + +# the connection map created at the end of this file +CONNECTOR = {} + + +class Mapper: + """Map resource to method. + + """ + def __init__(self, connector=CONNECTOR): + """Resource mapper constructor. + + Args: + con: a dictionary, mapping method name with URL path regexp + + """ + self._connector = connector + + def getController(self, uri): + """Find method for a given URI. + + Args: + uri: string with URI + + Returns: + None if no method is found or a tuple containing the following fields: + methd: name of method mapped to URI + items: a list of variable intems in the path + args: a dictionary with additional parameters from URL + + """ + if '?' in uri: + (path, query) = uri.split('?', 1) + args = cgi.parse_qs(query) + else: + path = uri + query = None + args = {} + + result = None + + for key, handler in self._connector.iteritems(): + # Regex objects + if hasattr(key, "match"): + m = key.match(path) + if m: + result = (handler, list(m.groups()), args) + break + + # String objects + elif key == path: + result = (handler, [], args) + break + + if result is not None: + return result + else: + raise httperror.HTTPNotFound() + + +class R_root(baserlib.R_Generic): + """/ resource. + + """ + DOC_URI = "/" + + def GET(self): + """Show the list of mapped resources. + + Returns: + A dictionary with 'name' and 'uri' keys for each of them. + + """ + root_pattern = re.compile('^R_([a-zA-Z0-9]+)$') + + rootlist = [] + for handler in CONNECTOR.values(): + m = root_pattern.match(handler.__name__) + if m: + name = m.group(1) + if name != 'root': + rootlist.append(name) + + return baserlib.BuildUriList(rootlist, "/%s") + + +CONNECTOR.update({ + "/": R_root, + + "/version": rlib1.R_version, + + "/tags": rlib1.R_tags, + "/info": rlib1.R_info, + + "/nodes": rlib1.R_nodes, + re.compile(r'^/nodes/([\w\._-]+)$'): rlib1.R_nodes_name, + re.compile(r'^/nodes/([\w\._-]+)/tags$'): rlib1.R_nodes_name_tags, + + "/instances": rlib1.R_instances, + re.compile(r'^/instances/([\w\._-]+)$'): rlib1.R_instances_name, + re.compile(r'^/instances/([\w\._-]+)/tags$'): rlib1.R_instances_name_tags, + + "/os": rlib1.R_os, + + "/2/jobs": rlib2.R_2_jobs, + "/2/nodes": rlib2.R_2_nodes, + re.compile(r'/2/jobs/(%s)$' % constants.JOB_ID_TEMPLATE): rlib2.R_2_jobs_id, + }) diff --git a/lib/rapi/resources.py b/lib/rapi/rlib1.py similarity index 55% rename from lib/rapi/resources.py rename to lib/rapi/rlib1.py index f055e72c3..ad9565c8c 100644 --- a/lib/rapi/resources.py +++ b/lib/rapi/rlib1.py @@ -19,189 +19,24 @@ # 02110-1301, USA. -"""Remote API resources. +"""Remote API version 1 resources library. """ -import cgi import re -import ganeti.opcodes -import ganeti.errors import ganeti.cli +import ganeti.errors +import ganeti.opcodes from ganeti import constants -from ganeti import luxi from ganeti import utils -from ganeti.rapi import httperror - - -# Initialized at the end of this file. -_CONNECTOR = {} - - -def BuildUriList(ids, uri_format, uri_fields=("name", "uri")): - """Builds a URI list as used by index resources. - - Args: - - ids: List of ids as strings - - uri_format: Format to be applied for URI - - uri_fields: Optional parameter for field ids - - """ - (field_id, field_uri) = uri_fields - - def _MapId(m_id): - return { field_id: m_id, field_uri: uri_format % m_id, } - - # Make sure the result is sorted, makes it nicer to look at and simplifies - # unittests. - ids.sort() - - return map(_MapId, ids) - - -def ExtractField(sequence, index): - """Creates a list containing one column out of a list of lists. - - Args: - - sequence: Sequence of lists - - index: Index of field - - """ - return map(lambda item: item[index], sequence) - - -def MapFields(names, data): - """Maps two lists into one dictionary. - - Args: - - names: Field names (list of strings) - - data: Field data (list) - - Example: - >>> MapFields(["a", "b"], ["foo", 123]) - {'a': 'foo', 'b': 123} - - """ - if len(names) != len(data): - raise AttributeError("Names and data must have the same length") - return dict([(names[i], data[i]) for i in range(len(names))]) - - -def _Tags_GET(kind, name=None): - """Helper function to retrieve tags. - - """ - if name is None: - # Do not cause "missing parameter" error, which happens if a parameter - # is None. - name = "" - op = ganeti.opcodes.OpGetTags(kind=kind, name=name) - tags = ganeti.cli.SubmitOpCode(op) - return list(tags) - -class Mapper: - """Map resource to method. - - """ - def __init__(self, connector=_CONNECTOR): - """Resource mapper constructor. - - Args: - con: a dictionary, mapping method name with URL path regexp - - """ - self._connector = connector +from ganeti.rapi import baserlib +from ganeti.rapi import httperror - def getController(self, uri): - """Find method for a given URI. - Args: - uri: string with URI - - Returns: - None if no method is found or a tuple containing the following fields: - methd: name of method mapped to URI - items: a list of variable intems in the path - args: a dictionary with additional parameters from URL - - """ - if '?' in uri: - (path, query) = uri.split('?', 1) - args = cgi.parse_qs(query) - else: - path = uri - query = None - args = {} - - result = None - - for key, handler in self._connector.iteritems(): - # Regex objects - if hasattr(key, "match"): - m = key.match(path) - if m: - result = (handler, list(m.groups()), args) - break - - # String objects - elif key == path: - result = (handler, [], args) - break - - if result is not None: - return result - else: - raise httperror.HTTPNotFound() - - -class R_Generic(object): - """Generic class for resources. - - """ - def __init__(self, request, items, queryargs): - """Generic resource constructor. - - Args: - request: HTTPRequestHandler object - items: a list with variables encoded in the URL - queryargs: a dictionary with additional options from URL - - """ - self.request = request - self.items = items - self.queryargs = queryargs - - -class R_root(R_Generic): - """/ resource. - - """ - DOC_URI = "/" - - def GET(self): - """Show the list of mapped resources. - - Returns: - A dictionary with 'name' and 'uri' keys for each of them. - - """ - root_pattern = re.compile('^R_([a-zA-Z0-9]+)$') - - rootlist = [] - for handler in _CONNECTOR.values(): - m = root_pattern.match(handler.__name__) - if m: - name = m.group(1) - if name != 'root': - rootlist.append(name) - - return BuildUriList(rootlist, "/%s") - - -class R_version(R_Generic): +class R_version(baserlib.R_Generic): """/version resource. This resource should be used to determine the remote API version and to adapt @@ -217,7 +52,7 @@ class R_version(R_Generic): return constants.RAPI_VERSION -class R_tags(R_Generic): +class R_tags(baserlib.R_Generic): """/tags resource. Manages cluster tags. @@ -231,10 +66,10 @@ class R_tags(R_Generic): Example: ["tag1", "tag2", "tag3"] """ - return _Tags_GET(constants.TAG_CLUSTER) + return baserlib._Tags_GET(constants.TAG_CLUSTER) -class R_info(R_Generic): +class R_info(baserlib.R_Generic): """Cluster info. """ @@ -263,7 +98,7 @@ class R_info(R_Generic): return ganeti.cli.SubmitOpCode(op) -class R_nodes(R_Generic): +class R_nodes(baserlib.R_Generic): """/nodes resource. """ @@ -289,7 +124,7 @@ class R_nodes(R_Generic): nodes_details = [] for node in result: - mapped = MapFields(fields, node) + mapped = baserlib.MapFields(fields, node) nodes_details.append(mapped) return nodes_details @@ -330,15 +165,15 @@ class R_nodes(R_Generic): """ op = ganeti.opcodes.OpQueryNodes(output_fields=["name"], names=[]) - nodeslist = ExtractField(ganeti.cli.SubmitOpCode(op), 0) + nodeslist = baserlib.ExtractField(ganeti.cli.SubmitOpCode(op), 0) if 'bulk' in self.queryargs: return self._GetDetails(nodeslist) - return BuildUriList(nodeslist, "/nodes/%s") + return baserlib.BuildUriList(nodeslist, "/nodes/%s") -class R_nodes_name(R_Generic): +class R_nodes_name(baserlib.R_Generic): """/nodes/[node_name] resources. """ @@ -357,10 +192,10 @@ class R_nodes_name(R_Generic): names=[node_name]) result = ganeti.cli.SubmitOpCode(op) - return MapFields(fields, result[0]) + return baserlib.MapFields(fields, result[0]) -class R_nodes_name_tags(R_Generic): +class R_nodes_name_tags(baserlib.R_Generic): """/nodes/[node_name]/tags resource. Manages per-node tags. @@ -374,10 +209,10 @@ class R_nodes_name_tags(R_Generic): Example: ["tag1", "tag2", "tag3"] """ - return _Tags_GET(constants.TAG_NODE, name=self.items[0]) + return baserlib._Tags_GET(constants.TAG_NODE, name=self.items[0]) -class R_instances(R_Generic): +class R_instances(baserlib.R_Generic): """/instances resource. """ @@ -405,7 +240,7 @@ class R_instances(R_Generic): instances_details = [] for instance in result: - mapped = MapFields(fields, instance) + mapped = baserlib.MapFields(fields, instance) instances_details.append(mapped) return instances_details @@ -453,16 +288,16 @@ class R_instances(R_Generic): """ op = ganeti.opcodes.OpQueryInstances(output_fields=["name"], names=[]) - instanceslist = ExtractField(ganeti.cli.SubmitOpCode(op), 0) + instanceslist = baserlib.ExtractField(ganeti.cli.SubmitOpCode(op), 0) if 'bulk' in self.queryargs: return self._GetDetails(instanceslist) else: - return BuildUriList(instanceslist, "/instances/%s") + return baserlib.BuildUriList(instanceslist, "/instances/%s") -class R_instances_name(R_Generic): +class R_instances_name(baserlib.R_Generic): """/instances/[instance_name] resources. """ @@ -483,10 +318,10 @@ class R_instances_name(R_Generic): names=[instance_name]) result = ganeti.cli.SubmitOpCode(op) - return MapFields(fields, result[0]) + return baserlib.MapFields(fields, result[0]) -class R_instances_name_tags(R_Generic): +class R_instances_name_tags(baserlib.R_Generic): """/instances/[instance_name]/tags resource. Manages per-instance tags. @@ -500,10 +335,10 @@ class R_instances_name_tags(R_Generic): Example: ["tag1", "tag2", "tag3"] """ - return _Tags_GET(constants.TAG_INSTANCE, name=self.items[0]) + return baserlib._Tags_GET(constants.TAG_INSTANCE, name=self.items[0]) -class R_os(R_Generic): +class R_os(baserlib.R_Generic): """/os resource. """ @@ -525,72 +360,3 @@ class R_os(R_Generic): raise httperror.HTTPInternalError(message="Can't get OS list") return [row[0] for row in diagnose_data if row[1]] - - -class R_2_jobs(R_Generic): - """/2/jobs resource. - - """ - DOC_URI = "/2/jobs" - - def GET(self): - """Returns a dictionary of jobs. - - Returns: - A dictionary with jobs id and uri. - - """ - fields = ["id"] - # Convert the list of lists to the list of ids - result = [job_id for [job_id] in luxi.Client().QueryJobs(None, fields)] - return BuildUriList(result, "/2/jobs/%s", uri_fields=("id", "uri")) - - -class R_2_jobs_id(R_Generic): - """/2/jobs/[job_id] resource. - - """ - DOC_URI = "/2/jobs/[job_id]" - - def GET(self): - """Returns a job status. - - Returns: - A dictionary with job parameters. - - The result includes: - id - job ID as a number - status - current job status as a string - ops - involved OpCodes as a list of dictionaries for each opcodes in - the job - opstatus - OpCodes status as a list - opresult - OpCodes results as a list of lists - - """ - fields = ["id", "ops", "status", "opstatus", "opresult"] - job_id = self.items[0] - result = luxi.Client().QueryJobs([job_id,], fields)[0] - return MapFields(fields, result) - - -_CONNECTOR.update({ - "/": R_root, - - "/version": R_version, - - "/tags": R_tags, - "/info": R_info, - - "/nodes": R_nodes, - re.compile(r'^/nodes/([\w\._-]+)$'): R_nodes_name, - re.compile(r'^/nodes/([\w\._-]+)/tags$'): R_nodes_name_tags, - - "/instances": R_instances, - re.compile(r'^/instances/([\w\._-]+)$'): R_instances_name, - re.compile(r'^/instances/([\w\._-]+)/tags$'): R_instances_name_tags, - - "/os": R_os, - - "/2/jobs": R_2_jobs, - re.compile(r'/2/jobs/(%s)$' % constants.JOB_ID_TEMPLATE): R_2_jobs_id, - }) diff --git a/lib/rapi/rlib2.py b/lib/rapi/rlib2.py new file mode 100644 index 000000000..956d2bfb3 --- /dev/null +++ b/lib/rapi/rlib2.py @@ -0,0 +1,154 @@ +# +# + +# Copyright (C) 2006, 2007, 2008 Google Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + + +"""Remote API version 2 baserlib.library. + +""" + +import re + +import ganeti.opcodes + +from ganeti import constants +from ganeti import luxi + +from ganeti.rapi import baserlib + + +class R_2_jobs(baserlib.R_Generic): + """/2/jobs resource. + + """ + DOC_URI = "/2/jobs" + + def GET(self): + """Returns a dictionary of jobs. + + Returns: + A dictionary with jobs id and uri. + + """ + fields = ["id"] + # Convert the list of lists to the list of ids + result = [job_id for [job_id] in luxi.Client().QueryJobs(None, fields)] + return baserlib.BuildUriList(result, "/2/jobs/%s", uri_fields=("id", "uri")) + + +class R_2_jobs_id(baserlib.R_Generic): + """/2/jobs/[job_id] resource. + + """ + DOC_URI = "/2/jobs/[job_id]" + + def GET(self): + """Returns a job status. + + Returns: + A dictionary with job parameters. + + The result includes: + id - job ID as a number + status - current job status as a string + ops - involved OpCodes as a list of dictionaries for each opcodes in + the job + opstatus - OpCodes status as a list + opresult - OpCodes results as a list of lists + + """ + fields = ["id", "ops", "status", "opstatus", "opresult"] + job_id = self.items[0] + result = luxi.Client().QueryJobs([job_id,], fields)[0] + return baserlib.MapFields(fields, result) + + +class R_2_nodes(baserlib.R_Generic): + """/2/nodes resource. + + """ + DOC_URI = "/2/nodes" + + def _GetDetails(self, nodeslist): + """Returns detailed instance data for bulk output. + + Args: + instance: A list of nodes names. + + Returns: + A list of nodes properties + + """ + fields = ["name","dtotal", "dfree", + "mtotal", "mnode", "mfree", + "pinst_cnt", "sinst_cnt", "tags"] + + op = ganeti.opcodes.OpQueryNodes(output_fields=fields, + names=nodeslist) + result = ganeti.cli.SubmitOpCode(op) + + nodes_details = [] + for node in result: + mapped = baserlib.MapFields(fields, node) + nodes_details.append(mapped) + return nodes_details + + def GET(self): + """Returns a list of all nodes. + + Returns: + A dictionary with 'name' and 'uri' keys for each of them. + + Example: [ + { + "id": "node1.example.com", + "uri": "\/instances\/node1.example.com" + }, + { + "id": "node2.example.com", + "uri": "\/instances\/node2.example.com" + }] + + If the optional 'bulk' argument is provided and set to 'true' + value (i.e '?bulk=1'), the output contains detailed + information about nodes as a list. + + Example: [ + { + "pinst_cnt": 1, + "mfree": 31280, + "mtotal": 32763, + "name": "www.example.com", + "tags": [], + "mnode": 512, + "dtotal": 5246208, + "sinst_cnt": 2, + "dfree": 5171712 + }, + ... + ] + + """ + op = ganeti.opcodes.OpQueryNodes(output_fields=["name"], names=[]) + nodeslist = baserlib.ExtractField(ganeti.cli.SubmitOpCode(op), 0) + + if 'bulk' in self.queryargs: + return self._GetDetails(nodeslist) + + return baserlib.BuildUriList(nodeslist, "/nodes/%s", uri_fields=("id", "uri")) diff --git a/test/ganeti.rapi.resources_unittest.py b/test/ganeti.rapi.resources_unittest.py index 8fc40eee7..461cfe9a7 100755 --- a/test/ganeti.rapi.resources_unittest.py +++ b/test/ganeti.rapi.resources_unittest.py @@ -19,7 +19,7 @@ # 02110-1301, USA. -"""Script for unittesting the rapi.resources module""" +"""Script for unittesting the RAPI resources module""" import os @@ -28,16 +28,17 @@ import tempfile import time from ganeti import errors +from ganeti.rapi import connector from ganeti.rapi import httperror -from ganeti.rapi import resources from ganeti.rapi import RESTHTTPServer +from ganeti.rapi import rlib1 class MapperTests(unittest.TestCase): """Tests for remote API URI mapper.""" def setUp(self): - self.map = resources.Mapper() + self.map = connector.Mapper() def _TestUri(self, uri, result): self.assertEquals(self.map.getController(uri), result) @@ -46,17 +47,18 @@ class MapperTests(unittest.TestCase): self.failUnlessRaises(httperror.HTTPNotFound, self.map.getController, uri) def testMapper(self): - """Testing resources.Mapper""" + """Testing Mapper""" - self._TestUri("/tags", (resources.R_tags, [], {})) + self._TestUri("/tags", (rlib1.R_tags, [], {})) + self._TestUri("/instances", (rlib1.R_instances, [], {})) self._TestUri('/instances/www.test.com', - (resources.R_instances_name, + (rlib1.R_instances_name, ['www.test.com'], {})) self._TestUri('/instances/www.test.com/tags?f=5&f=6&alt=html', - (resources.R_instances_name_tags, + (rlib1.R_instances_name_tags, ['www.test.com'], {'alt': ['html'], 'f': ['5', '6'], @@ -70,7 +72,7 @@ class R_RootTests(unittest.TestCase): """Testing for R_root class.""" def setUp(self): - self.root = resources.R_root(None, None, None) + self.root = connector.R_root(None, None, None) def testGet(self): expected = [ -- GitLab