APIserver.py 19.8 KB
Newer Older
John Giannelos's avatar
John Giannelos committed
1 2
#!/usr/bin/env python

3 4 5 6 7 8 9 10 11 12
import inspect
import re
import sys
from optparse import OptionParser, OptionValueError
import string
import sqlite3
import os
import json
import uuid

John Giannelos's avatar
John Giannelos committed
13
from snfOCCI.registry import snfRegistry
14 15 16 17 18 19 20
from snfOCCI.compute import ComputeBackend, SNFBackend
from snfOCCI.config import SERVER_CONFIG, KAMAKI_CONFIG, VOMS_CONFIG, ASTAVOMS_URL
from snfOCCI import snf_voms
from snfOCCI.network import NetworkBackend, IpNetworkBackend, IpNetworkInterfaceBackend, NetworkInterfaceBackend
from kamaki.clients.cyclades import CycladesNetworkClient
from snfOCCI.extensions import snf_addons

User's avatar
User committed
21
ASTAVOMS_IS_KEYSTONE = True
John Giannelos's avatar
John Giannelos committed
22

23 24
# from kamaki.clients.compute import ComputeClient
from kamaki.clients.cyclades import CycladesComputeClient as ComputeClient
25
from kamaki.clients.cyclades import CycladesClient
26 27
from kamaki.clients import astakos, utils
from kamaki.clients import ClientError
John Giannelos's avatar
John Giannelos committed
28

29
from occi.core_model import Mixin, Resource
30
from occi.backend import MixinBackend
31 32 33 34
from occi.extensions.infrastructure import COMPUTE, START, STOP, SUSPEND, RESTART, RESOURCE_TEMPLATE, OS_TEMPLATE, NETWORK, IPNETWORK, NETWORKINTERFACE,IPNETWORKINTERFACE 
from occi import wsgi
from occi.exceptions import HTTPError
from occi import core_model
John Giannelos's avatar
John Giannelos committed
35 36

from wsgiref.validate import validator
37 38
from webob import Request
from pprint import pprint
John Giannelos's avatar
John Giannelos committed
39

40 41 42 43 44 45 46

if ASTAVOMS_IS_KEYSTONE:
    KEYSTONE_URI = ASTAVOMS_URL
else:
    KEYSTONE_URI = 'https://okeanos-occi2.hellasgrid.gr:5000/main'


47
def parse_arguments(args):
48 49 50 51
    kw = dict(
        usage="%prog [options]",
        description="OCCI interface to synnefo API",
    )
52 53 54
    parser = OptionParser(**kw)
    parser.disable_interspersed_args()
    
55 56 57 58 59 60 61 62
    parser.add_option(
        "--enable_voms",
        action="store_true", dest="enable_voms", default=False,
        help="Enable voms authorization")
    parser.add_option(
        "--voms_db",
        action="store", type="string", dest="voms_db",
        help="Path to sqlite database file")
63 64 65 66 67 68 69 70
    
    (opts, args) = parser.parse_args(args)
    
    if opts.enable_voms and not opts.voms_db:
        print "--voms_db option required"
        parser.print_help()
        
    return (opts, args)
John Giannelos's avatar
John Giannelos committed
71

John Giannelos's avatar
John Giannelos committed
72

73
class MyAPP(wsgi.Application):
74
    """An OCCI WSGI application"""
John Giannelos's avatar
John Giannelos committed
75

76
    def __init__(self):
77
        """Initialization of the WSGI OCCI application for synnefo"""
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
        global ENABLE_VOMS, VOMS_DB
        ENABLE_VOMS = VOMS_CONFIG['enable_voms']
        super(MyAPP,self).__init__(registry=snfRegistry())
        self._register_backends()
        VALIDATOR_APP = validator(self)
         
    def _register_backends(self):
        print "Inside Register Backends"
        COMPUTE_BACKEND = ComputeBackend()
        NETWORK_BACKEND = NetworkBackend() 
        NETWORKINTERFACE_BACKEND = NetworkInterfaceBackend()
        IPNETWORK_BACKEND = IpNetworkBackend()
        IPNETWORKINTERFACE_BACKEND = IpNetworkInterfaceBackend()
    
        self.register_backend(COMPUTE, COMPUTE_BACKEND)
        self.register_backend(START, COMPUTE_BACKEND)
        self.register_backend(STOP, COMPUTE_BACKEND)
        self.register_backend(RESTART, COMPUTE_BACKEND)
        self.register_backend(SUSPEND, COMPUTE_BACKEND)
        self.register_backend(RESOURCE_TEMPLATE, MixinBackend())
        self.register_backend(OS_TEMPLATE, MixinBackend())
       
        # Network related backends
        self.register_backend(NETWORK, NETWORK_BACKEND)
        self.register_backend(IPNETWORK, IPNETWORK_BACKEND)
        self.register_backend(NETWORKINTERFACE,NETWORKINTERFACE_BACKEND)
        self.register_backend(IPNETWORKINTERFACE, IPNETWORKINTERFACE_BACKEND)
105
        self.register_backend(snf_addons.SNF_USER_DATA_EXT, SNFBackend())  
106 107 108
        self.register_backend(snf_addons.SNF_KEY_PAIR_EXT,  SNFBackend())  
     
    def refresh_images(self, snf, client):
109 110 111 112 113 114 115 116 117 118 119 120
        try:
            images = snf.list_images()
            for image in images:
                IMAGE_ATTRIBUTES = {'occi.core.id': str(image['id'])}
                IMAGE = Mixin(
                    "http://schemas.ogf.org/occi/os_tpl#",
                    occify_terms(str(image['name'])),
                    [OS_TEMPLATE],
                    title='IMAGE', attributes = IMAGE_ATTRIBUTES)
                self.register_backend(IMAGE, MixinBackend())
        except:
            raise HTTPError(404, "Unauthorized access")
121
      
122
    def refresh_flavors(self, snf, client):
John Giannelos's avatar
John Giannelos committed
123
        flavors = snf.list_flavors()
124
        print "Retrieving details for each flavor"
John Giannelos's avatar
John Giannelos committed
125 126
        for flavor in flavors:
            details = snf.get_flavor_details(flavor['id'])
127 128 129 130 131 132 133 134 135 136 137
            FLAVOR_ATTRIBUTES = {
                'occi.core.id': flavor['id'],
                'occi.compute.cores': str(details['vcpus']),
                'occi.compute.memory': str(details['ram']),
                'occi.storage.size': str(details['disk']),
            }
            FLAVOR = Mixin(
                "http://schemas.ogf.org/occi/infrastructure#",
                str(flavor['name']),
                [RESOURCE_TEMPLATE],
                attributes=FLAVOR_ATTRIBUTES)
John Giannelos's avatar
John Giannelos committed
138
            self.register_backend(FLAVOR, MixinBackend())
139

140 141 142 143
    def refresh_flavors_norecursive(self, snf, client):
        flavors = snf.list_flavors(True)
        print "@ Retrieving details for each flavor"
        for flavor in flavors:
144 145 146 147 148 149 150 151 152 153 154
            FLAVOR_ATTRIBUTES = {
                'occi.core.id': flavor['id'],
                'occi.compute.cores': str(flavor['vcpus']),
                'occi.compute.memory': str(flavor['ram']),
                'occi.storage.size': str(flavor['disk']),
            }
            FLAVOR = Mixin(
                "http://schemas.ogf.org/occi/resource_tpl#",
                occify_terms(str(flavor['name'])),
                [RESOURCE_TEMPLATE],
                title='FLAVOR', attributes=FLAVOR_ATTRIBUTES)
155
            self.register_backend(FLAVOR, MixinBackend())
156

157 158 159 160 161 162 163 164 165
    def refresh_network_instances(self,client):
        print "@ refresh NETWORKS"
        network_details = client.list_networks(detail='True')
        resources = self.registry.resources
        occi_keys = resources.keys()
         
        for network in network_details:
            if '/network/'+str(network['id']) not in occi_keys:
                netID = '/network/'+str(network['id'])   
166
                snf_net = core_model.Resource(netID, NETWORK, [IPNETWORK])
167 168 169
                snf_net.attributes['occi.core.id'] = str(network['id']) 
               
                #This info comes from the network details
170 171 172
                snf_net.attributes['occi.network.state'] = str(
                    network['status'])
                snf_net.attributes['occi.network.gateway'] = ''
173 174 175 176 177
                if network['public'] == True:
                    snf_net.attributes['occi.network.type'] = "Public = True"
                else:
                    snf_net.attributes['occi.network.type'] = "Public = False"
                self.registry.add_resource(netID, snf_net, None)       
178

179
    def refresh_compute_instances(self, snf, client):
180 181
        """Syncing registry with cyclades resources"""
        print "@ Refresh COMPUTE INSTANCES"        
182

183 184 185 186 187 188 189 190
        servers = snf.list_servers()
        snf_keys = []
        for server in servers:
            snf_keys.append(str(server['id']))

        resources = self.registry.resources
        occi_keys = resources.keys()
        
191 192
        print occi_keys
        for serverID in occi_keys:
193 194
            if '/compute/' in serverID and resources[serverID].attributes[
                    'occi.compute.hostname'] == "":
195 196
                self.registry.delete_resource(serverID, None)
        occi_keys = resources.keys()
197

198
        #Compute instances in synnefo not available in registry
199
        diff = [x for x in snf_keys if '/compute/'+x not in occi_keys]        
200 201
        for key in diff:
            details = snf.get_server_details(int(key))
202 203
            flavor = snf.get_flavor_details(details['flavor']['id'])
            try:
204 205
                print "Get image of flavor {flavor}, VM {vm}".format(
                    flavor=details['flavor']['id'], vm=key)
206 207 208 209 210 211
                image = snf.get_image_details(details['image']['id'])
                for i in self.registry.backends:
                    if i.term ==  occify_terms(str(image['name'])):
                        rel_image = i
                    if i.term ==  occify_terms(str(flavor['name'])):
                        rel_flavor = i
212

213 214 215 216
                resource = Resource(key, COMPUTE, [rel_flavor, rel_image])
                resource.actions = [START]
                resource.attributes['occi.core.id'] = key
                resource.attributes['occi.compute.state'] = 'inactive'
217 218 219 220
                resource.attributes['occi.compute.architecture'] = (
                    SERVER_CONFIG['compute_arch'])
                resource.attributes['occi.compute.cores'] = str(
                    flavor['vcpus'])
221 222 223 224
                resource.attributes['occi.compute.memory'] = str(flavor['ram'])
                resource.attributes['occi.core.title'] = str(details['name'])
                networkIDs = details['addresses'].keys()
                if len(networkIDs)>0: 
225 226
                    resource.attributes['occi.compute.hostname'] = str(
                        details['addresses'][networkIDs[0]][0]['addr'])
227 228
                else:
                    resource.attributes['occi.compute.hostname'] = ""
229 230 231 232 233
                self.registry.add_resource(key, resource, None)

                net_str = (
                    "http://schemas.ogf.org/occi/infrastructure#"
                    "networkinterface{0}".format(link_id))
234 235
                for netKey in networkIDs:
                    link_id = str(uuid.uuid4())
236 237 238 239 240 241 242 243 244
                    NET_LINK = core_model.Link(
                        net_str,
                        NETWORKINTERFACE, [IPNETWORKINTERFACE], resource,
                        self.registry.resources['/network/'+str(netKey)])

                    for version in details['addresses'][netKey]:                        
                        ip4address = ''
                        ip6address = ''
                        if version['version']==4:
245 246 247
                            ip4address = str(version['addr'])
                            allocheme = str(version['OS-EXT-IPS:type'])
                        elif version['version']==6:
248 249 250
                            ip6address = str(version['addr'])    
                        allocheme = str(version['OS-EXT-IPS:type'])

251 252
                    if 'attachments' in details.keys():
                        for item in details['attachments']:
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
                            NET_LINK.attributes = {
                                'occi.core.id':link_id,
                                'occi.networkinterface.allocation' : allocheme,
                                'occi.networking.interface': str(item['id']),
                                'occi.networkinterface.mac' : str(
                                    item['mac_address']),
                                'occi.networkinterface.address' : ip4address,
                                'occi.networkinterface.ip6' :  ip6address}
                    elif len(details['addresses'][netKey]) > 0:
                        NET_LINK.attributes ={
                            'occi.core.id':link_id,
                            'occi.networkinterface.allocation' : allocheme,
                            'occi.networking.interface': '',
                            'occi.networkinterface.mac' : '',
                            'occi.networkinterface.address' : ip4address,
                            'occi.networkinterface.ip6' :  ip6address}
269
                    else:
270 271 272 273 274 275 276
                        NET_LINK.attributes ={
                            'occi.core.id':link_id,
                            'occi.networkinterface.allocation' : '',
                            'occi.networking.interface': '',
                            'occi.networkinterface.mac' : '',
                            'occi.networkinterface.address' :'',
                            'occi.networkinterface.ip6' : '' }
277 278 279
                    resource.links.append(NET_LINK)
                    self.registry.add_resource(link_id, NET_LINK, None)
            except ClientError as ce:
280
                print ce.status
281 282 283 284 285
                if ce.status == 404 or ce.status == 500:
                    print('Image not found, sorry!!!')
                    continue
                else:
                    raise ce
286
  
287
        #Compute instances in registry not available in synnefo
288 289
        diff = [x for x in occi_keys if x[9:] not in snf_keys]
        for key in diff:
290 291
            if '/network/' not in key:
                self.registry.delete_resource(key, None)
292

John Giannelos's avatar
John Giannelos committed
293
    def __call__(self, environ, response):
294 295 296 297 298 299 300 301
        
        # Enable VOMS Authorization
        print "SNF_OCCI application has been called!"
        req = Request(environ)
        
        if not req.environ.has_key('HTTP_X_AUTH_TOKEN'):
            print "An authentication token has NOT been provided!"
            status = '401 Not Authorized'
302 303 304 305
            headers = [
                ('Content-Type', 'text/html'),
                ('Www-Authenticate', 'Keystone uri=\'{uri}\''.format(
                    uri=KEYSTONE_URI))]
306
            response(status,headers)
307
            print 'Ask for redirect to URL {uri}'.format(uri=KEYSTONE_URI)
308 309 310 311 312 313 314 315 316 317 318 319
            return [str(response)]
        print 'An authentication token has been provided'
        environ['HTTP_AUTH_TOKEN'] = req.environ['HTTP_X_AUTH_TOKEN']
        try:
            print "Get project"
            snf_project = req.environ['HTTP_X_SNF_PROJECT']
        except KeyError:
            print "No project provided, go to plan B"
            # status = '400 Bad Request No Project Provided'
            # headers = [('Content-Type', 'text/html'),('Www-Authenticate','Keystone uri=\'https://okeanos-occi2.hellasgrid.gr:5000/main\'')]
            # response(status,headers)
            # return [str(response)]
320 321
            astakosClient = astakos.AstakosClient(
                KAMAKI_CONFIG['astakos_url'], environ['HTTP_AUTH_TOKEN'])
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
            projects = astakosClient.get_projects()
            user_info = astakosClient.authenticate()
            user_uuid = user_info['access']['user']['id']
            snf_project = '6d9ec935-fcd4-4ae1-a3a0-10e612c4f867'
            for project in projects:
                if project['id'] != user_uuid:
                    snf_project = project['id']
                    print "Project found"
                    break
        if ENABLE_VOMS:                
            compClient = ComputeClient(KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            cyclClient = CycladesClient(KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            netClient = CycladesNetworkClient(KAMAKI_CONFIG['network_url'], environ['HTTP_AUTH_TOKEN'])
            try:
                #Up-to-date flavors and images
                self.refresh_images(compClient, cyclClient)
                self.refresh_flavors_norecursive(compClient, cyclClient)
                self.refresh_network_instances(netClient)
                self.refresh_compute_instances(compClient, cyclClient)
                # token will be represented in self.extras
                return self._call_occi(environ, response, security = None, token = environ['HTTP_AUTH_TOKEN'], snf = compClient, client = cyclClient, snf_network=netClient, snf_project=snf_project)
343 344 345 346 347 348 349
            except HTTPError:
                print "Exception from unauthorized access!"
                status = '401 Not Authorized'
                headers = [
                    ('Content-Type', 'text/html'),
                    ('Www-Authenticate', 'Keystone uri=\'{uri}\''.format(
                        uri=KEYSTONE_URI))]
350
                response(status,headers)
351
                print 'Ask for redirect to {uri}'.format(uri=KEYSTONE_URI)
352 353 354
                return [str(response)]
        else:
            print 'I have a token and a project, we can proceed'
355 356 357 358 359 360
            compClient = ComputeClient(
                KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            cyclClient = CycladesClient(
                KAMAKI_CONFIG['compute_url'], environ['HTTP_AUTH_TOKEN'])
            netClient = CycladesNetworkClient(
                KAMAKI_CONFIG['network_url'], environ['HTTP_AUTH_TOKEN'])
361 362 363 364 365 366 367 368 369

            #Up-to-date flavors and images
            self.refresh_images(compClient,cyclClient)
            
            self.refresh_flavors_norecursive(compClient,cyclClient)
            self.refresh_network_instances(cyclClient)
            self.refresh_compute_instances(compClient,cyclClient)
            
            # token will be represented in self.extras
370 371 372 373 374 375
            return self._call_occi(
                environ, response,
                security=None, token=environ['HTTP_AUTH_TOKEN'],
                snf=compClient, client=cyclClient, snf_network=netClient,
                snf_project=snf_project)

376 377 378 379 380

def application(env, start_response):
    """/v2.0/tokens"""
    print "In /v2.0/tokens"
    t = snf_voms.VomsAuthN()       
381 382
    user_dn, user_vo, user_fqans, snf_token, snf_project = t.process_request(
        env)
383 384 385 386 387 388 389 390

    print (user_dn, user_vo, user_fqans)
      
    env['HTTP_AUTH_TOKEN'] = snf_token
    env['SNF_PROJECT'] = snf_project
    # Get user authentication details
    print "@ refresh_user authentication details"
    pool = False
391 392
    astakosClient = astakos.AstakosClient(
        KAMAKI_CONFIG['astakos_url'], env['HTTP_AUTH_TOKEN'] , use_pool=pool)
393 394
    user_details = astakosClient.authenticate()
    
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
    response = {
        'access': {
            'token': {
                'issued_at': '',
                'expires': user_details['access']['token']['expires'] ,
                'id':env['HTTP_AUTH_TOKEN']
            },
            'serviceCatalog': [],
            'user': {
                'username': user_dn,
                'roles_links':user_details['access']['user']['roles_links'],
                'id': user_details['access']['user']['id'],
                'roles':[],
                'name':user_dn
            },
            'metadata': {
                'is_admin': 0,
                'roles': user_details['access']['user']['roles']
            }
        }
    }        
416 417 418 419 420 421
    status = '200 OK'
    headers = [('Content-Type', 'application/json')]        
    start_response(status,headers)
    body = json.dumps(response)
    print body
    return [body]
John Giannelos's avatar
John Giannelos committed
422

423

424 425 426 427
def app_factory(global_config, **local_config):
    """This function wraps our simple WSGI app so it
    can be used with paste.deploy"""
    return application
428

429

430 431 432 433 434 435 436 437 438 439
def tenant_application(env, start_response):
    """/v2.0/tennants"""
    print "In /v2.0/tennants"
    #t =snf_voms.VomsAuthN()       
    #(user_dn, user_vo, user_fqans) = t.process_request(env)
    #print (user_dn, user_vo, user_fqans)
    req = Request(env)
    if req.environ.has_key('HTTP_X_AUTH_TOKEN'):
        env['HTTP_AUTH_TOKEN']= req.environ['HTTP_X_AUTH_TOKEN']
    else:
440 441
        raise HTTPError(404, "Unauthorized access")

442 443 444
    # Get user authentication details
    print "@ refresh_user authentication details"
    pool = False
445 446
    astakosClient = astakos.AstakosClient(
        KAMAKI_CONFIG['astakos_url'], env['HTTP_AUTH_TOKEN'], use_pool=pool)
447 448
    user_details = astakosClient.authenticate()
    
449 450 451 452 453 454 455 456 457 458 459
    response = {
        'tenants_links': [],
        'tenants':[
            {
                'description': 'Instances of EGI Federated Clouds TF',
                'enabled': True,
                'id': user_details['access']['user']['id'],
                'name': 'ops'
            },
        ]
    }
460
    status = '200 OK'
461
    headers = [('Content-Type', 'application/json'), ]
462 463 464 465
    start_response(status,headers)
    body = json.dumps(response)
    print body
    return [body]
John Giannelos's avatar
John Giannelos committed
466

467

468 469 470 471
def tenant_app_factory(global_config, **local_config):
    """This function wraps our simple WSGI app so it
    can be used with paste.deploy"""
    return tenant_application
472

473 474

def occify_terms(term_name):
475
    """Occifies a term_name so that it is compliant with GFD 185"""
476
    term = term_name.strip().replace(' ', '_').replace('.', '-').lower()
477 478
    return term.replace('(', '_').replace(')', '_').replace('@', '_').replace(
        '+', '-_')