APIserver.py 19.1 KB
Newer Older
John Giannelos's avatar
John Giannelos committed
1 2
#!/usr/bin/env python

3 4 5 6 7 8 9 10 11 12
import inspect
import re
import sys
from optparse import OptionParser, OptionValueError
import string
import sqlite3
import os
import json
import uuid

John Giannelos's avatar
John Giannelos committed
13
from snfOCCI.registry import snfRegistry
14
from snfOCCI.compute import ComputeBackend, SNFBackend
15
from snfOCCI.config import SERVER_CONFIG, KAMAKI_CONFIG, VOMS_CONFIG, KEYSTONE_URL
16 17 18 19 20 21 22
from snfOCCI import snf_voms
from snfOCCI.network import NetworkBackend, IpNetworkBackend, IpNetworkInterfaceBackend, NetworkInterfaceBackend
from kamaki.clients.cyclades import CycladesNetworkClient
from snfOCCI.extensions import snf_addons

# from kamaki.clients.compute import ComputeClient
from kamaki.clients.cyclades import CycladesComputeClient as ComputeClient
23
from kamaki.clients.cyclades import CycladesClient
24 25
from kamaki.clients import astakos, utils
from kamaki.clients import ClientError
John Giannelos's avatar
John Giannelos committed
26

27
from occi.core_model import Mixin, Resource
28
from occi.backend import MixinBackend
29 30 31 32
from occi.extensions.infrastructure import COMPUTE, START, STOP, SUSPEND, RESTART, RESOURCE_TEMPLATE, OS_TEMPLATE, NETWORK, IPNETWORK, NETWORKINTERFACE,IPNETWORKINTERFACE 
from occi import wsgi
from occi.exceptions import HTTPError
from occi import core_model
John Giannelos's avatar
John Giannelos committed
33 34

from wsgiref.validate import validator
35 36
from webob import Request
from pprint import pprint
John Giannelos's avatar
John Giannelos committed
37

38 39


40
def parse_arguments(args):
41 42 43 44
    kw = dict(
        usage="%prog [options]",
        description="OCCI interface to synnefo API",
    )
45 46
    parser = OptionParser(**kw)
    parser.disable_interspersed_args()
47

48 49 50 51 52 53 54 55
    parser.add_option(
        "--enable_voms",
        action="store_true", dest="enable_voms", default=False,
        help="Enable voms authorization")
    parser.add_option(
        "--voms_db",
        action="store", type="string", dest="voms_db",
        help="Path to sqlite database file")
56

57
    (opts, args) = parser.parse_args(args)
58

59 60 61
    if opts.enable_voms and not opts.voms_db:
        print "--voms_db option required"
        parser.print_help()
62

63
    return (opts, args)
John Giannelos's avatar
John Giannelos committed
64

John Giannelos's avatar
John Giannelos committed
65

66
class MyAPP(wsgi.Application):
67
    """An OCCI WSGI application"""
John Giannelos's avatar
John Giannelos committed
68

69
    def __init__(self):
70
        """Initialization of the WSGI OCCI application for synnefo"""
71 72 73 74 75
        global ENABLE_VOMS, VOMS_DB
        ENABLE_VOMS = VOMS_CONFIG['enable_voms']
        super(MyAPP,self).__init__(registry=snfRegistry())
        self._register_backends()
        VALIDATOR_APP = validator(self)
76

77 78 79
    def _register_backends(self):
        print "Inside Register Backends"
        COMPUTE_BACKEND = ComputeBackend()
80
        NETWORK_BACKEND = NetworkBackend()
81 82 83
        NETWORKINTERFACE_BACKEND = NetworkInterfaceBackend()
        IPNETWORK_BACKEND = IpNetworkBackend()
        IPNETWORKINTERFACE_BACKEND = IpNetworkInterfaceBackend()
84

85 86 87 88 89 90 91
        self.register_backend(COMPUTE, COMPUTE_BACKEND)
        self.register_backend(START, COMPUTE_BACKEND)
        self.register_backend(STOP, COMPUTE_BACKEND)
        self.register_backend(RESTART, COMPUTE_BACKEND)
        self.register_backend(SUSPEND, COMPUTE_BACKEND)
        self.register_backend(RESOURCE_TEMPLATE, MixinBackend())
        self.register_backend(OS_TEMPLATE, MixinBackend())
92

93 94 95
        # Network related backends
        self.register_backend(NETWORK, NETWORK_BACKEND)
        self.register_backend(IPNETWORK, IPNETWORK_BACKEND)
96
        self.register_backend(NETWORKINTERFACE, NETWORKINTERFACE_BACKEND)
97
        self.register_backend(IPNETWORKINTERFACE, IPNETWORKINTERFACE_BACKEND)
98 99 100
        self.register_backend(snf_addons.SNF_USER_DATA_EXT, SNFBackend())
        self.register_backend(snf_addons.SNF_KEY_PAIR_EXT,  SNFBackend())

101
    def refresh_images(self, snf, client):
102 103 104 105 106 107 108 109
        try:
            images = snf.list_images()
            for image in images:
                IMAGE_ATTRIBUTES = {'occi.core.id': str(image['id'])}
                IMAGE = Mixin(
                    "http://schemas.ogf.org/occi/os_tpl#",
                    occify_terms(str(image['name'])),
                    [OS_TEMPLATE],
110
                    title='IMAGE', attributes=IMAGE_ATTRIBUTES)
111 112 113
                self.register_backend(IMAGE, MixinBackend())
        except:
            raise HTTPError(404, "Unauthorized access")
114

115
    def refresh_flavors(self, snf, client):
John Giannelos's avatar
John Giannelos committed
116
        flavors = snf.list_flavors()
117
        print "Retrieving details for each flavor"
John Giannelos's avatar
John Giannelos committed
118 119
        for flavor in flavors:
            details = snf.get_flavor_details(flavor['id'])
120 121 122 123 124 125 126 127 128 129 130
            FLAVOR_ATTRIBUTES = {
                'occi.core.id': flavor['id'],
                'occi.compute.cores': str(details['vcpus']),
                'occi.compute.memory': str(details['ram']),
                'occi.storage.size': str(details['disk']),
            }
            FLAVOR = Mixin(
                "http://schemas.ogf.org/occi/infrastructure#",
                str(flavor['name']),
                [RESOURCE_TEMPLATE],
                attributes=FLAVOR_ATTRIBUTES)
John Giannelos's avatar
John Giannelos committed
131
            self.register_backend(FLAVOR, MixinBackend())
132

133 134 135 136
    def refresh_flavors_norecursive(self, snf, client):
        flavors = snf.list_flavors(True)
        print "@ Retrieving details for each flavor"
        for flavor in flavors:
137 138 139 140 141 142 143 144 145 146 147
            FLAVOR_ATTRIBUTES = {
                'occi.core.id': flavor['id'],
                'occi.compute.cores': str(flavor['vcpus']),
                'occi.compute.memory': str(flavor['ram']),
                'occi.storage.size': str(flavor['disk']),
            }
            FLAVOR = Mixin(
                "http://schemas.ogf.org/occi/resource_tpl#",
                occify_terms(str(flavor['name'])),
                [RESOURCE_TEMPLATE],
                title='FLAVOR', attributes=FLAVOR_ATTRIBUTES)
148
            self.register_backend(FLAVOR, MixinBackend())
149

150
    def refresh_network_instances(self, client):
151 152 153 154
        print "@ refresh NETWORKS"
        network_details = client.list_networks(detail='True')
        resources = self.registry.resources
        occi_keys = resources.keys()
155

156 157
        for network in network_details:
            if '/network/'+str(network['id']) not in occi_keys:
158
                netID = '/network/'+str(network['id'])
159
                snf_net = core_model.Resource(netID, NETWORK, [IPNETWORK])
160 161 162
                snf_net.attributes['occi.core.id'] = str(network['id'])

                # This info comes from the network details
163 164 165
                snf_net.attributes['occi.network.state'] = str(
                    network['status'])
                snf_net.attributes['occi.network.gateway'] = ''
166
                if network['public'] is True:
167 168 169
                    snf_net.attributes['occi.network.type'] = "Public = True"
                else:
                    snf_net.attributes['occi.network.type'] = "Public = False"
170
                self.registry.add_resource(netID, snf_net, None)
171

172
    def refresh_compute_instances(self, snf, client):
173
        """Syncing registry with cyclades resources"""
174
        print "@ Refresh COMPUTE INSTANCES"
175

176 177 178 179 180 181 182
        servers = snf.list_servers()
        snf_keys = []
        for server in servers:
            snf_keys.append(str(server['id']))

        resources = self.registry.resources
        occi_keys = resources.keys()
183

184 185
        print occi_keys
        for serverID in occi_keys:
186 187
            if '/compute/' in serverID and resources[serverID].attributes[
                    'occi.compute.hostname'] == "":
188 189
                self.registry.delete_resource(serverID, None)
        occi_keys = resources.keys()
190

191 192
        # Compute instances in synnefo not available in registry
        diff = [x for x in snf_keys if '/compute/'+x not in occi_keys]
193 194
        for key in diff:
            details = snf.get_server_details(int(key))
195 196
            flavor = snf.get_flavor_details(details['flavor']['id'])
            try:
197 198
                print "Get image of flavor {flavor}, VM {vm}".format(
                    flavor=details['flavor']['id'], vm=key)
199 200
                image = snf.get_image_details(details['image']['id'])
                for i in self.registry.backends:
201
                    if i.term == occify_terms(str(image['name'])):
202
                        rel_image = i
203
                    if i.term == occify_terms(str(flavor['name'])):
204
                        rel_flavor = i
205

206 207 208 209
                resource = Resource(key, COMPUTE, [rel_flavor, rel_image])
                resource.actions = [START]
                resource.attributes['occi.core.id'] = key
                resource.attributes['occi.compute.state'] = 'inactive'
210 211 212 213
                resource.attributes['occi.compute.architecture'] = (
                    SERVER_CONFIG['compute_arch'])
                resource.attributes['occi.compute.cores'] = str(
                    flavor['vcpus'])
214 215 216
                resource.attributes['occi.compute.memory'] = str(flavor['ram'])
                resource.attributes['occi.core.title'] = str(details['name'])
                networkIDs = details['addresses'].keys()
217
                if len(networkIDs) > 0:
218 219
                    resource.attributes['occi.compute.hostname'] = str(
                        details['addresses'][networkIDs[0]][0]['addr'])
220 221
                else:
                    resource.attributes['occi.compute.hostname'] = ""
222 223 224 225 226
                self.registry.add_resource(key, resource, None)

                net_str = (
                    "http://schemas.ogf.org/occi/infrastructure#"
                    "networkinterface{0}".format(link_id))
227 228
                for netKey in networkIDs:
                    link_id = str(uuid.uuid4())
229 230 231 232 233
                    NET_LINK = core_model.Link(
                        net_str,
                        NETWORKINTERFACE, [IPNETWORKINTERFACE], resource,
                        self.registry.resources['/network/'+str(netKey)])

234
                    for version in details['addresses'][netKey]:
235 236
                        ip4address = ''
                        ip6address = ''
237
                        if version['version'] == 4:
238 239
                            ip4address = str(version['addr'])
                            allocheme = str(version['OS-EXT-IPS:type'])
240 241
                        elif version['version'] == 6:
                            ip6address = str(version['addr'])
242 243
                        allocheme = str(version['OS-EXT-IPS:type'])

244 245
                    if 'attachments' in details.keys():
                        for item in details['attachments']:
246
                            NET_LINK.attributes = {
247 248
                                'occi.core.id': link_id,
                                'occi.networkinterface.allocation': allocheme,
249
                                'occi.networking.interface': str(item['id']),
250
                                'occi.networkinterface.mac': str(
251
                                    item['mac_address']),
252 253
                                'occi.networkinterface.address': ip4address,
                                'occi.networkinterface.ip6':  ip6address}
254
                    elif len(details['addresses'][netKey]) > 0:
255 256 257
                        NET_LINK.attributes = {
                            'occi.core.id': link_id,
                            'occi.networkinterface.allocation': allocheme,
258
                            'occi.networking.interface': '',
259 260 261
                            'occi.networkinterface.mac': '',
                            'occi.networkinterface.address': ip4address,
                            'occi.networkinterface.ip6':  ip6address}
262
                    else:
263 264 265
                        NET_LINK.attributes = {
                            'occi.core.id': link_id,
                            'occi.networkinterface.allocation': '',
266
                            'occi.networking.interface': '',
267 268 269
                            'occi.networkinterface.mac': '',
                            'occi.networkinterface.address': '',
                            'occi.networkinterface.ip6': ''}
270 271 272
                    resource.links.append(NET_LINK)
                    self.registry.add_resource(link_id, NET_LINK, None)
            except ClientError as ce:
273
                print ce.status
274 275 276 277 278
                if ce.status == 404 or ce.status == 500:
                    print('Image not found, sorry!!!')
                    continue
                else:
                    raise ce
279 280

        # Compute instances in registry not available in synnefo
281 282
        diff = [x for x in occi_keys if x[9:] not in snf_keys]
        for key in diff:
283 284
            if '/network/' not in key:
                self.registry.delete_resource(key, None)
285

John Giannelos's avatar
John Giannelos committed
286
    def __call__(self, environ, response):
287
        """Enable VOMS Authorization"""
288 289
        print "SNF_OCCI application has been called!"
        req = Request(environ)
290

291 292 293
        if not req.environ.has_key('HTTP_X_AUTH_TOKEN'):
            print "An authentication token has NOT been provided!"
            status = '401 Not Authorized'
294 295 296
            headers = [
                ('Content-Type', 'text/html'),
                ('Www-Authenticate', 'Keystone uri=\'{uri}\''.format(
297
                    uri=KEYSTONE_URL))]
298
            response(status, headers)
299
            print 'Ask for redirect to URL {uri}'.format(uri=KEYSTONE_URL)
300 301 302 303 304 305 306 307
            return [str(response)]
        print 'An authentication token has been provided'
        environ['HTTP_AUTH_TOKEN'] = req.environ['HTTP_X_AUTH_TOKEN']
        try:
            print "Get project"
            snf_project = req.environ['HTTP_X_SNF_PROJECT']
        except KeyError:
            print "No project provided, go to plan B"
308 309
            astakosClient = astakos.AstakosClient(
                KAMAKI_CONFIG['astakos_url'], environ['HTTP_AUTH_TOKEN'])
310 311 312 313 314 315 316 317 318
            projects = astakosClient.get_projects()
            user_info = astakosClient.authenticate()
            user_uuid = user_info['access']['user']['id']
            snf_project = '6d9ec935-fcd4-4ae1-a3a0-10e612c4f867'
            for project in projects:
                if project['id'] != user_uuid:
                    snf_project = project['id']
                    print "Project found"
                    break
319 320 321 322 323 324 325
        if ENABLE_VOMS:
            compClient = ComputeClient(
                KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            cyclClient = CycladesClient(
                KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            netClient = CycladesNetworkClient(
                KAMAKI_CONFIG['network_url'], environ['HTTP_AUTH_TOKEN'])
326
            try:
327
                # Up-to-date flavors and images
328 329 330 331 332
                self.refresh_images(compClient, cyclClient)
                self.refresh_flavors_norecursive(compClient, cyclClient)
                self.refresh_network_instances(netClient)
                self.refresh_compute_instances(compClient, cyclClient)
                # token will be represented in self.extras
333 334 335 336 337
                return self._call_occi(
                    environ, response,
                    security=None, token=environ['HTTP_AUTH_TOKEN'],
                    snf=compClient, client=cyclClient,
                    snf_network=netClient, snf_project=snf_project)
338 339 340 341 342 343
            except HTTPError:
                print "Exception from unauthorized access!"
                status = '401 Not Authorized'
                headers = [
                    ('Content-Type', 'text/html'),
                    ('Www-Authenticate', 'Keystone uri=\'{uri}\''.format(
344
                        uri=KEYSTONE_URL))]
345
                response(status, headers)
346
                print 'Ask for redirect to {uri}'.format(uri=KEYSTONE_URL)
347 348 349
                return [str(response)]
        else:
            print 'I have a token and a project, we can proceed'
350 351 352 353 354 355
            compClient = ComputeClient(
                KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            cyclClient = CycladesClient(
                KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            netClient = CycladesNetworkClient(
                KAMAKI_CONFIG['network_url'], environ['HTTP_AUTH_TOKEN'])
356

357 358 359 360
            # Up-to-date flavors and images
            self.refresh_images(compClient, cyclClient)

            self.refresh_flavors_norecursive(compClient, cyclClient)
361
            self.refresh_network_instances(cyclClient)
362 363
            self.refresh_compute_instances(compClient, cyclClient)

364
            # token will be represented in self.extras
365 366 367 368 369 370
            return self._call_occi(
                environ, response,
                security=None, token=environ['HTTP_AUTH_TOKEN'],
                snf=compClient, client=cyclClient, snf_network=netClient,
                snf_project=snf_project)

371 372 373 374

def application(env, start_response):
    """/v2.0/tokens"""
    print "In /v2.0/tokens"
375
    t = snf_voms.VomsAuthN()
376 377
    user_dn, user_vo, user_fqans, snf_token, snf_project = t.process_request(
        env)
378 379 380 381 382 383 384

    print (user_dn, user_vo, user_fqans)
    env['HTTP_AUTH_TOKEN'] = snf_token
    env['SNF_PROJECT'] = snf_project
    # Get user authentication details
    print "@ refresh_user authentication details"
    pool = False
385
    astakosClient = astakos.AstakosClient(
386
        KAMAKI_CONFIG['astakos_url'], env['HTTP_AUTH_TOKEN'], use_pool=pool)
387
    user_details = astakosClient.authenticate()
388

389 390 391 392
    response = {
        'access': {
            'token': {
                'issued_at': '',
393 394
                'expires': user_details['access']['token']['expires'],
                'id': env['HTTP_AUTH_TOKEN']
395 396 397 398
            },
            'serviceCatalog': [],
            'user': {
                'username': user_dn,
399
                'roles_links': user_details['access']['user']['roles_links'],
400
                'id': user_details['access']['user']['id'],
401 402
                'roles': [],
                'name': user_dn
403 404 405 406 407 408
            },
            'metadata': {
                'is_admin': 0,
                'roles': user_details['access']['user']['roles']
            }
        }
409
    }
410
    status = '200 OK'
411 412
    headers = [('Content-Type', 'application/json')]
    start_response(status, headers)
413 414 415
    body = json.dumps(response)
    print body
    return [body]
John Giannelos's avatar
John Giannelos committed
416

417

418 419 420 421
def app_factory(global_config, **local_config):
    """This function wraps our simple WSGI app so it
    can be used with paste.deploy"""
    return application
422

423

424 425 426 427 428
def tenant_application(env, start_response):
    """/v2.0/tennants"""
    print "In /v2.0/tennants"
    req = Request(env)
    if req.environ.has_key('HTTP_X_AUTH_TOKEN'):
429
        env['HTTP_AUTH_TOKEN'] = req.environ['HTTP_X_AUTH_TOKEN']
430
    else:
431 432
        raise HTTPError(404, "Unauthorized access")

433 434 435
    # Get user authentication details
    print "@ refresh_user authentication details"
    pool = False
436 437
    astakosClient = astakos.AstakosClient(
        KAMAKI_CONFIG['astakos_url'], env['HTTP_AUTH_TOKEN'], use_pool=pool)
438
    user_details = astakosClient.authenticate()
439

440 441
    response = {
        'tenants_links': [],
442
        'tenants': [
443 444 445 446 447 448 449 450
            {
                'description': 'Instances of EGI Federated Clouds TF',
                'enabled': True,
                'id': user_details['access']['user']['id'],
                'name': 'ops'
            },
        ]
    }
451
    status = '200 OK'
452
    headers = [('Content-Type', 'application/json'), ]
453
    start_response(status, headers)
454 455 456
    body = json.dumps(response)
    print body
    return [body]
John Giannelos's avatar
John Giannelos committed
457

458

459 460 461 462
def tenant_app_factory(global_config, **local_config):
    """This function wraps our simple WSGI app so it
    can be used with paste.deploy"""
    return tenant_application
463

464

465 466 467 468
def occify_terms(term):
    """:return: Occified term, compliant with GFD 185"""
    return term.strip().lower().replace(' ', '_').replace('.', '-').replace(
        '(', '_').replace(')', '_').replace('@', '_').replace('+', '-_')