Commit 03d14f3f authored by Christos KK Loverdos's avatar Christos KK Loverdos
Browse files

Merge branch 'master' into testing

Conflicts:
	setup.py
parents a47a0722 30114168
# Import general commission framework
from .api.exception import (CallError,
from .exception import (CallError,
CommissionException,
CorruptedError,
InvalidDataError,
......@@ -11,18 +11,12 @@ from .api.exception import (CallError,
ExportLimitError,
ImportLimitError)
from .api.callpoint import Callpoint, get_callpoint, mkcallargs
from .api.physical import Physical
from .api.controller import Controller, ControlledCallpoint
from .callpoint import Callpoint, get_callpoint, mkcallargs
from .api.specificator import (Specificator, SpecifyException,
from .specificator import (Specificator, SpecifyException,
Canonifier, CanonifyException,
Canonical,
Null, Nothing, Integer, Serial,
Text, Bytes, Tuple, ListOf, Dict, Args)
# Import quota holder API
from .api.quotaholder import QuotaholderAPI
# Import standard implementations?
# Don't import anything here,
# parent imports everything 'public'
from .exception import CorruptedError
from .callpoint import Callpoint
from .physical import Physical
from .specificator import CanonifyException
class Controller(object):
def __init__(self, quotaholder, physical):
self.quotaholder = quotaholder
self.physical = physical
self.controller_init()
def controller_init(self):
pass
def get_commission_issue(self, commission_spec):
"""Prepare and return the arguments for the
quotaholder's issue_commission call,
containing the provisions required and
the target entity for their allocation.
"""
raise NotImplementedError
def register_commission(self, serial,
clientkey,
physical_description ):
"""Register a commission to the controller's stable storage,
along with the quotaholder serial and clientkey,
and the target physical description.
This information is needed to co-ordinate the commission
execution among the quotaholder server, the controller,
and the physical layer implementing the resource.
"""
raise NotImplementedError
def get_commission(self, serial):
"""Retrieve the commission registered with serial"""
raise NotImplementedError
def complete_commission(self, serial):
"""Mark and commit in stable storage the commission identified by
a serial as to-be-completed-successfully,
i.e that it has succeeded in producing a physical resource
and is to be removed from being tracked by the holder server,
controller, and physical layers.
"""
raise NotImplementedError
def is_commission_complete(self, serial):
"""Return true if the serial is marked as
completed by complete_commission()
"""
raise NotImplementedError
def fail_commission(self, serial):
"""Mark and commit in stable storage the commission identified by
a serial as to-be-completed-unsuccessfully,
i.e. that it has failed in producing a physical resource
and is to be removed from being tracked by the holder server,
controller, and physical layers.
"""
raise NotImplementedError
def is_commission_failing(self, serial):
"""Return true if the serial is marked as
failing by fail_commission()
"""
raise NotImplementedError
def retire_commission(self, serial):
"""Stop tracking the commission identified by a serial"""
raise NotImplementedError
def undertake_commission(self, commission_spec):
"""Initiate and start tracking and co-ordinating a commission
from a commission spec.
"""
holder = self.quotaholder
physical = self.physical
commission_issue = self.get_commission_issue(commission_spec)
entity = commission_issue['entity']
clientkey = commission_issue['clientkey']
physical_description = physical.derive_description(commission_spec)
serial = holder.issue_commission(**commission_issue)
self.register_commission( serial,
clientkey,
physical_description )
self.process_controller(serial)
return serial
def process_controller(self, serial):
"""Consider the current state of a commission in the controller layer,
and schedule next actions.
"""
holder = self.quotaholder
physical = self.physical
controller = self
r = controller.get_commission(serial)
if not r:
return
serial, clientkey, physical_description, status = r
if controller.is_commission_complete(serial):
holder.accept_commission(serial=serial, clientkey=clientkey)
physical.end_commission(serial, physical_description)
controller.retire_commission(serial)
elif controller.is_commission_failing(serial):
holder.recall_commission(serial=serial, clientkey=clientkey)
physical.end_commission(serial, physical_description)
controller.retire_commission(serial)
else:
controller.process_physical(serial)
def process_physical(self, serial):
"""Consider the current state of a commission in the physical layer,
and schedule next actions.
"""
physical = self.physical
r = self.get_commission(serial)
if not r:
m = "Unknown serial %d in process_physical!" % (serial,)
raise CorruptedError(m)
target_description = r[2]
current_state = physical.get_current_state(serial, target_description)
if not current_state:
physical.initiate_commission(serial, target_description)
elif physical.complies(current_state, target_description):
self.complete_commission(serial)
self.process_controller(serial)
elif physical.attainable(current_state, target_description):
physical.continue_commission(serial, target_description)
else:
self.fail_commission(serial)
physical.end_commission(serial, target_description)
self.process_controller(serial)
class ControlledCallpoint(Callpoint):
controllables = set()
def __init__(self, *args):
self.controllables = set()
super(ControlledCallpoint, self).__init__(*args)
def commission_from_call(self, call_name, call_data):
commission_spec = {}
commission_spec['call_name'] = call_name
commission_spec['call_data'] = call_data
commission_spec['provisions'] = ()
return commission_spec, True
def register_controllable(self, call_name):
controllables = self.controllables
if call_name in controllables:
return
canonify_output = self.api_spec.canonify_output
if (canonify_output(call_name, None) is not None or
not isinstance(canonify_output(call_name, 1L), long)):
m = ("Attempt to register controllable call '%s', "
"but the api spec does not define a "
"nullable long (serial) output!" % (call_name,))
raise CanonifyException(m)
if not isinstance(canonify_output(call_name, 2**63), long):
m = ("Attempt to register controllable call '%s', "
"but the api spec does not define a nullable long "
"(serial) output with a range up to 2**63!" % (call_name,))
raise CanonifyException(m)
controllables.add(call_name)
def do_make_call(self, call_name, call_data):
r = self.commission_from_call(call_name, call_data)
commission_spec, controllable = r
controller = self.controller
if not controllable:
return controller.forward_commission(commission_spec)
if call_name not in self.controllables:
self.register_controllable(call_name)
serial = controller.undertake_commission(commission_spec)
return serial
class Physical(object):
def derive_description(self, commission_spec):
"""Derive a target physical description from a commission specification
which is understandable and executable by the physical layer.
"""
raise NotImplementedError
def initiate_commission(self, serial, description):
"""Start creating a resource with a physical description,
tagged by the given serial.
"""
raise NotImplementedError
def get_current_state(self, serial, description):
"""Query and return the current physical state for the
target physical description initiated by the given serial.
"""
raise NotImplementedError
def complies(self, state, description):
"""Compare current physical state and target physical description
and decide if the commission has been successfully implemented.
"""
raise NotImplementedError
def attainable(self, state, description):
"""Compare current physical state and target physical description
and decide if the commission can be implemented.
"""
raise NotImplementedError
def continue_commission(self, serial, description):
"""Continue an ongoing commission towards
the given target physical description
"""
raise NotImplementedError
def end_commission(self, serial, description):
"""Cancel and stop tracking the commission identified by serial"""
raise NotImplementedError
......@@ -27,7 +27,7 @@ class Callpoint(object):
m = "No api spec given to '%s'" % (type(self).__name__,)
raise NotImplementedError(m)
for call_name, call_doc in canonifier.call_attrs():
for call_name, call_doc in canonifier.call_docs():
if hasattr(self, call_name):
# don't crash: wrap the function instead
#m = ( "Method '%s' defined both in natively "
......@@ -60,13 +60,13 @@ class Callpoint(object):
setattr(self, call_name, mk_call_func())
def init_connection(self, connection):
raise NotImplementedError
pass
def commit(self):
raise NotImplementedError
pass
def rollback(self):
raise NotImplementedError
pass
def do_make_call(self, call_name, data):
raise NotImplementedError
......@@ -106,7 +106,7 @@ class Callpoint(object):
data = None
data = self.make_call(call_name, data)
json_data = self.json_dumps(data) if data is not None else None
json_data = self.json_dumps(data)
return json_data
def make_call(self, call_name, data):
......@@ -163,20 +163,16 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
versiontag = mk_versiontag(version)
components = pointname.split('.')
category = components[0]
if not category:
raise ValueError("invalid pointname '%s'" % (pointname,))
if category not in ['clients', 'servers']:
category = 'clients'
components = [category] + components
appname = components[0]
if len(components) < 2:
raise ValueError("invalid pointname '%s'" % (pointname,))
appname = components[1]
pointname = '.'.join(components)
modname = ('commissioning.%s.callpoint.API_Callpoint%s'
category = components[1]
if not category or category not in ['clients', 'servers']:
raise ValueError("invalid pointname '%s'" % (pointname,))
modname = ('%s.callpoint.API_Callpoint%s'
% (pointname, versiontag))
try:
......@@ -196,9 +192,9 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
pointname = '.'.join(components)
if pointname == 'quotaholder':
apiname = 'commissioning.api.quotaholder.QuotaholderAPI'
apiname = 'quotaholder.api.QuotaholderAPI'
else:
apiname = 'commissioning.specs.%s.API_Spec%s' % (pointname, versiontag)
apiname = '%s.api.API_Spec%s' % (pointname, versiontag)
API_Spec = imp_module(apiname)
......
#!/usr/bin/env python
from commissioning import Callpoint
from commissioning.clients.http import main, HTTP_API_Client
from commissioning.specs.fscrud import API_Spec as FSCRUD_API
class FSCRUD_HTTP(HTTP_API_Client):
api_spec = FSCRUD_API()
class FSCRUD_Debug(Callpoint):
api_spec = FSCRUD_API()
def init_connection(self, connection):
self.connection = connection
print 'connecting to', connection
def commit(self):
pass
def abort(self):
pass
def do_make_call(self, call_name, call_data):
print call_name, str(call_data)
if __name__ == '__main__':
main()
#!/usr/bin/env python
from synnefo.lib.pool.http import get_http_connection
from urlparse import urlparse
from commissioning import Callpoint, CallError
from commissioning.utils.clijson import clijson
from commissioning.utils.debug import debug
from json import loads as json_loads, dumps as json_dumps
class HTTP_API_Client(Callpoint):
"""Synchronous http client for quota holder API"""
appname = 'http'
def init_connection(self, connection):
self.url = connection
def commit(self):
return
def rollback(self):
return
def do_make_call(self, api_call, data):
url = urlparse(self.url)
scheme = url.scheme
path = url.path.strip('/')
path = ('/' + path + '/' + api_call) if path else ('/' + api_call)
netloc = url.netloc
conn = None
try:
debug("Connecting to %s\n>>>", netloc)
conn = get_http_connection(netloc=netloc, scheme=scheme)
if (api_call.startswith('list') or
api_call.startswith('get') or
api_call.startswith('read')):
method = 'GET'
else:
method = 'POST'
json_data = self.json_dumps(data)
debug("%s %s\n%s\n<<<\n", method, path, json_data)
req = conn.request(method, path, body=json_data)
resp = conn.getresponse()
debug(">>>\nStatus: %s", resp.status)
for name, value in resp.getheaders():
debug("%s: %s", name, value)
body = ''
while 1:
s = resp.read()
if not s:
break
body += s
debug("\n%s\n<<<\n", body[:128])
status = int(resp.status)
if status == 200:
if body:
body = json_loads(body)
return body
else:
try:
error = json_loads(body)
except ValueError, e:
exc = CallError(body, call_error='ValueError')
else:
exc = CallError.from_dict(error)
raise exc
finally:
if conn is not None:
conn.close()
API_Callpoint = HTTP_API_Client
def main():
from sys import argv, stdout
from os.path import basename, expanduser
from time import time
from commissioning import get_callpoint
from commissioning.utils.debug import init_logger_stderr
progname = basename(argv[0])
h, s, t = progname.rpartition('.')
if t == 'py':
progname = h
if progname == 'http':
if len(argv) < 2:
usage = "./http <appname> <app args...>"
print(usage)
raise SystemExit
argv = argv[1:]
progname = basename(argv[0])
init_logger_stderr(progname)
pointname = 'clients.' + progname
API_Callpoint = get_callpoint(pointname, automake='http')
api = API_Callpoint.api_spec
usage = "API Calls:\n\n"
for call_name in sorted(api.call_names()):
canonical = api.input_canonical(call_name)
argstring = canonical.tostring(multiline=1, showopts=0)
usage += call_name + '.' + argstring + '\n\n'
import argparse
parser = argparse.ArgumentParser (
formatter_class = argparse.RawDescriptionHelpFormatter,
description = "%s http client" % (progname,),
epilog = usage,
)
urlhelp = 'set %s server base url' % (progname,)
parser.add_argument('--url', type=str, dest='url',
action='store', help=urlhelp)
jsonhelp = 'intepret data as json'
parser.add_argument('--json', dest='json_data', action='store_false',
default=True, help=jsonhelp)
callhelp = 'api call to perform'
parser.add_argument('api_call', type=str, action='store', nargs=1,
help=callhelp)
arghelp = 'data to provide to api call'
parser.add_argument('data', type=str, action='store', nargs='*',
help=callhelp)
urlfilepath = expanduser('~/.%s.urlrc' % progname)
def get_url():
try:
with open(urlfilepath) as f:
url = f.read().strip()
except Exception:
m = "Cannot load url from %s. Try --url." % (urlfilepath,)
raise ValueError(m)
return url
def set_url(url):
url = url.strip('/')
with open(urlfilepath, "w") as f:
f.write(url)
print "Base URL set to '%s'" % (url,)
args = parser.parse_args(argv[1:])
if args.url:
set_url(args.url)
api_call = args.api_call[0]
api.input_canonical(api_call)
url = get_url()
data = args.data
if data:
if data[0] == '-':
from sys import stdin
data = stdin.read()
else:
data = clijson(data)
if not data:
data = None
client = API_Callpoint(url)
print(client.make_call_from_json(api_call, data))
if __name__ == '__main__':
main()
#!/usr/bin/env python
from commissioning.clients.http import main, HTTP_API_Client
from commissioning import QuotaholderAPI
class QuotaholderHTTP(HTTP_API_Client):
api_spec = QuotaholderAPI()
if __name__ == '__main__':
main()
#!/usr/bin/env python
from commissioning.clients.http import main, HTTP_API_Client
from commissioning import QuotaholderAPI
class QuotaholderHTTP(HTTP_API_Client):
api_spec = QuotaholderAPI()
from .models import DjangoController
Controller = DjangoController
from commissioning import Controller
from django.db.models import Model, BigIntegerField, CharField, IntegerField
from django.db import transaction
from json import dumps as json_dumps, loads as json_loads
class ControllerCommission(Model):
serial = BigIntegerField(primary_key=True)
clientkey = CharField(null=False, max_length=72)
physical_description = CharField(null=False, max_length=4096)
status = IntegerField(null=False)