astakos: Import new-style services and resources

docs/admin-guide.rst

@@ -1011,12 +1011,11 @@ project-list                  List projects
 project-show                  Show project details
 quota                         List and check the integrity of user quota
 reconcile-resources-astakos   Reconcile resource usage of Quotaholder with Astakos DB
-resource-add                  Add resource
 resource-export-astakos       Export astakos resources in json format
-resource-import               Register service resources
+resource-import               Register resources
 resource-list                 List resources
 resource-modify               Modify a resource's default base quota and boolean flags
-service-add                   Register a service
+service-import                Register services
 service-list                  List services
 service-show                  Show service details
 term-add                      Add approval terms

snf-astakos-app/astakos/api/quotas.py

@@ -40,7 +40,7 @@ from snf_django.lib.db.transaction import commit_on_success_strict
 from snf_django.lib import api
 from snf_django.lib.api.faults import BadRequest, ItemNotFound
 
-from astakos.im.resources import get_resources
+from astakos.im.register import get_resources
 from astakos.im.quotas import get_user_quotas, service_get_quotas
 
 import astakos.quotaholder_app.exception as qh_exception

snf-astakos-app/astakos/im/astakos_resources.py

@@ -31,11 +31,10 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
-service = "astakos"
-
 resources = [
     {"desc": "Number of pending project applications",
      "name": "astakos.pending_app",
      "allow_in_projects": False,
+     "service_type": "account",
      }
 ]

snf-astakos-app/astakos/im/management/commands/_common.py

@@ -44,7 +44,7 @@ from django.core.management import CommandError
 from synnefo.util import units
 from synnefo.lib.ordereddict import OrderedDict
 from astakos.im.models import AstakosUser
-from astakos.im.resources import get_resources
+from astakos.im.register import get_resources
 
 DEFAULT_CONTENT_TYPE = None
 

snf-astakos-app/astakos/im/management/commands/resource-add.py

-# Copyright 2012 GRNET S.A. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
-# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-#
-# The views and conclusions contained in the software and
-# documentation are those of the authors and should not be
-# interpreted as representing official policies, either expressed
-# or implied, of GRNET S.A.
-
-from django.core.management.base import BaseCommand, CommandError
-from django.db.utils import IntegrityError
-
-from astakos.im.models import Resource, Service
-
-
-class Command(BaseCommand):
-    args = "<service> <resource> <desc> <unit>"
-    help = "Add a resource"
-
-    def handle(self, *args, **options):
-        if len(args) < 2:
-            raise CommandError("Invalid number of arguments")
-
-        service_name = args[0]
-        resource_name = args[1]
-
-        try:
-            service = Service.objects.get(name=service_name)
-        except Service.DoesNotExist:
-            raise CommandError("Invalid service name")
-        else:
-            try:
-                resource = Resource(name=resource_name, service=service)
-                resource.save()
-            except IntegrityError, e:
-                raise CommandError(e)
-            # else:
-#                 resource.meta.add(args[2:])

snf-astakos-app/astakos/im/management/commands/resource-export-astakos.py

@@ -34,15 +34,12 @@
 from django.utils import simplejson as json
 from django.core.management.base import BaseCommand
 
-from astakos.im.astakos_resources import service, resources
+from astakos.im.astakos_resources import resources
 
 
 class Command(BaseCommand):
     help = "Export astakos resources in json format"
 
     def handle(self, *args, **options):
-        data = {'service': service,
-                'resources': resources,
-                }
-        output = json.dumps(data, indent=4)
+        output = json.dumps(resources, indent=4)
         self.stdout.write(output + '\n')

snf-astakos-app/astakos/im/management/commands/resource-import.py

@@ -38,12 +38,12 @@ from django.db.utils import IntegrityError
 from django.utils import simplejson as json
 
 from snf_django.lib.db.transaction import commit_on_success_strict
-from astakos.im.resources import add_resource
+from astakos.im.register import add_resource, ResourceException
 from astakos.im.models import Service
 
 
 class Command(BaseCommand):
-    help = "Register service resources"
+    help = "Register resources"
 
     option_list = BaseCommand.option_list + (
         make_option('--json',
@@ -61,40 +61,33 @@ class Command(BaseCommand):
 
         else:
             with open(json_file) as file_data:
-                m = ('Input should be a JSON dict containing "service" '
-                     'and "resource" keys.')
+                m = 'Input should be a JSON list.'
                 try:
                     data = json.load(file_data)
                 except json.JSONDecodeError:
                     raise CommandError(m)
-                if not isinstance(data, dict):
+                if not isinstance(data, list):
                     raise CommandError(m)
-                else:
-                    try:
-                        service = data['service']
-                        resources = data['resources']
-                    except KeyError:
-                        raise CommandError(m)
-
-        self.add_resources(service, resources)
+        self.add_resources(data)
 
 
     @commit_on_success_strict()
-    def add_resources(self, service, resources):
-
-        try:
-            s = Service.objects.get(name=service)
-        except Service.DoesNotExist:
-            raise CommandError("Service '%s' is not registered." % (service))
-
+    def add_resources(self, resources):
+        output = []
         for resource in resources:
             if not isinstance(resource, dict):
                 raise CommandError("Malformed resource dict.")
-            r, exists = add_resource(s, resource)
+            try:
+                r, exists = add_resource(resource)
+            except ResourceException as e:
+                raise CommandError(e.message)
             name = r.name
             if exists:
                 m = "Resource '%s' updated in database.\n" % (name)
             else:
                 m = ("Resource '%s' created in database with default "
                      "quota limit 0.\n" % (name))
-            self.stdout.write(m)
+            output.append(m)
+
+        for line in output:
+            self.stdout.write(line)

snf-astakos-app/astakos/im/management/commands/resource-list.py

@@ -51,14 +51,14 @@ class Command(ListCommand):
     FIELDS = {
         "id": ("id", "ID"),
         "name": ("name", "Resource Name"),
-        "service": ("service", "Service"),
+        "service type": ("service_type", "Service"),
         "limit": ("limit_with_unit", "Base Quota"),
         "description": ("desc", "Description"),
         "allow_in_projects": ("allow_in_projects",
                               "Make resource available in projects"),
     }
 
-    fields = ["id", "name", "service", "limit", "allow_in_projects",
+    fields = ["id", "name", "service type", "limit", "allow_in_projects",
               "description"]
 
     def show_limit(self, resource):

snf-astakos-app/astakos/im/management/commands/resource-modify.py

@@ -37,7 +37,7 @@ from django.utils import simplejson as json
 
 from synnefo.webproject.management import utils
 from astakos.im.models import Resource
-from astakos.im.resources import update_resource
+from astakos.im.register import update_resource
 from ._common import show_resource_value, style_options, check_style, units
 
 

snf-astakos-app/astakos/im/management/commands/service-add.py → snf-astakos-app/astakos/im/management/commands/service-import.py

-# Copyright 2012, 2013 GRNET S.A. All rights reserved.
+# Copyright 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -32,73 +32,70 @@
 # or implied, of GRNET S.A.
 
 from optparse import make_option
-import string
 
 from django.core.management.base import BaseCommand, CommandError
 from django.db.utils import IntegrityError
+from django.utils import simplejson as json
 
-from astakos.im.models import Service
+from snf_django.lib.db.transaction import commit_on_success_strict
+from astakos.im.register import add_service, ServiceException
+from astakos.im.models import Component
 
 
 class Command(BaseCommand):
-    args = "<name> <service URL> <API URL> "
-    help = "Register a service"
+    help = "Register services"
 
     option_list = BaseCommand.option_list + (
-        make_option('--type',
-                    dest='type',
-                    help="Service type"),
-        make_option('-f', '--no-confirm',
-                    action='store_true',
-                    default=False,
-                    dest='force',
-                    help="Do not ask for confirmation"),
+        make_option('--json',
+                    dest='json',
+                    metavar='<json.file>',
+                    help="Load service definitions from a json file"),
     )
 
+    @commit_on_success_strict()
     def handle(self, *args, **options):
-        if len(args) < 2:
-            raise CommandError("Invalid number of arguments")
 
-        name = args[0]
-        url = args[1]
-        api_url = args[2]
-        kwargs = dict(name=name, url=url, api_url=api_url)
-        s_type = options['type']
-        if s_type:
-            kwargs['type'] = s_type
-
-        try:
-            s = Service.objects.get(name=name)
-            m = "There already exists service named '%s'." % name
+        json_file = options['json']
+        if not json_file:
+            m = "Expecting option --json."
             raise CommandError(m)
-        except Service.DoesNotExist:
-            pass
 
-        services = list(Service.objects.filter(url=url))
-        if services:
-            m = "Service URL '%s' is registered for another service." % url
-            raise CommandError(m)
+        else:
+            with open(json_file) as file_data:
+                m = ('Input should be a JSON dict mapping service names '
+                     'to definitions.')
+                try:
+                    data = json.load(file_data)
+                except json.JSONDecodeError:
+                    raise CommandError(m)
+                if not isinstance(data, dict):
+                    raise CommandError(m)
+        self.add_services(data)
 
-        services = list(Service.objects.filter(api_url=api_url))
-        if services:
-            m = "API URL '%s' is registered for another service." % api_url
-            raise CommandError(m)
+    def add_services(self, data):
+        write = self.stdout.write
+        output = []
+        for name, service_dict in data.iteritems():
+            try:
+                component_name = service_dict['component']
+                service_type = service_dict['type']
+                endpoints = service_dict['endpoints']
+            except KeyError:
+                raise CommandError('Malformed service definition.')
 
-        force = options['force']
-        if not force:
-            tp = (' of type %s' % s_type) if s_type else ''
-            self.stdout.write("Add service %s%s with:\n" % (name, tp))
-            self.stdout.write("service URL: %s\n" % url)
-            self.stdout.write("API URL: %s\n" % api_url)
-            self.stdout.write("Confirm? (y/n) ")
-            response = raw_input()
-            if string.lower(response) not in ['y', 'yes']:
-                self.stdout.write("Aborted.\n")
-                return
+            try:
+                component = Component.objects.get(name=component_name)
+            except Component.DoesNotExist:
+                m = "Component '%s' is not registered." % component_name
+                raise CommandError(m)
 
-        try:
-            s = Service.objects.create(**kwargs)
-        except BaseException:
-            raise CommandError("Failed to create service.")
-        else:
-            self.stdout.write('Token: %s\n' % s.auth_token)
+            try:
+                existed = add_service(component, name, service_type, endpoints)
+            except ServiceException as e:
+                raise CommandError(e.message)
+
+            m = "%s service %s.\n" % ("Updated" if existed else "Added", name)
+            output.append(m)
+
+        for line in output:
+            write(line)

snf-astakos-app/astakos/im/management/commands/service-list.py

@@ -40,14 +40,10 @@ class Command(ListCommand):
     object_class = Service
 
     FIELDS = {
-        "id": ("id", "ID"),
+        "id": ("id", "Service ID"),
         "name": ("name", "Service Name"),
-        "url": ("url", "Service url"),
-        "api_url": ("api_url", "Service API url"),
-        "token": ("auth_token", "Authentication token"),
-        "token created": ("auth_token_created", "Token creation date"),
-        "token expires": ("auth_token_expires", "Token expiration date"),
+        "component": ("component.name", "Service url"),
         "type": ("type", "Service type"),
     }
 
-    fields = ["id", "name", "type", "token", "token created"]
+    fields = ["id", "name", "component", "type"]

snf-astakos-app/astakos/im/management/commands/service-show.py

@@ -32,7 +32,7 @@
 # or implied, of GRNET S.A.
 
 from django.core.management.base import CommandError
-from astakos.im.models import Service
+from astakos.im.models import Service, EndpointData
 from synnefo.lib.ordereddict import OrderedDict
 from synnefo.webproject.management.commands import SynnefoCommand
 from synnefo.webproject.management import utils
@@ -62,13 +62,20 @@ class Command(SynnefoCommand):
             [
                 ('id', service.id),
                 ('name', service.name),
+                ('component', service.component),
                 ('type', service.type),
-                ('service URL', service.url),
-                ('API URL', service.api_url),
-                ('token', service.auth_token),
-                ('token created', service.auth_token_created),
-                ('token expires', service.auth_token_expires),
             ])
 
         utils.pprint_table(self.stdout, [kv.values()], kv.keys(),
                            options["output_format"], vertical=True)
+
+        self.stdout.write('\n')
+        endpoint_data = EndpointData.objects.filter(endpoint__service=service)
+        data = []
+        for ed in endpoint_data:
+            data.append((ed.endpoint_id, ed.key, ed.value))
+
+        labels = ('endpoint', 'key', 'value')
+        utils.pprint_table(self.stdout, data, labels,
+                           options["output_format"],
+                           title='Endpoints')

snf-astakos-app/astakos/im/resources.py → snf-astakos-app/astakos/im/register.py

@@ -31,33 +31,40 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
-from astakos.im.models import Resource
+from astakos.im.models import Resource, Service, Endpoint, EndpointData
+
 from astakos.im.quotas import qh_add_resource_limit, qh_sync_new_resource
 import logging
 
 logger = logging.getLogger(__name__)
 
-fields = ['name', 'desc', 'unit', 'allow_in_projects']
+resource_fields = ['desc', 'unit', 'allow_in_projects']
 
 
 class ResourceException(Exception):
     pass
 
 
-def add_resource(service, resource_dict):
+def add_resource(resource_dict):
     name = resource_dict.get('name')
-    if not name:
+    service_type = resource_dict.get('service_type')
+    if not name or not service_type:
         raise ResourceException("Malformed resource dict.")
 
     try:
         r = Resource.objects.get_for_update(name=name)
         exists = True
+        if r.service_type != service_type:
+            m = ("There already exists a resource named %s with service "
+                 "type %s." % (name, r.service_type))
+            raise ResourceException(m)
     except Resource.DoesNotExist:
-        r = Resource(uplimit=0)
+        r = Resource(name=name,
+                     uplimit=0,
+                     service_type=service_type)
         exists = False
 
-    r.service = service
-    for field in fields:
+    for field in resource_fields:
         value = resource_dict.get(field)
         if value is not None:
             setattr(r, field, value)
@@ -99,3 +106,38 @@ def get_resources(resources=None, services=None):
         resource_dict[r.full_name()] = r.get_info()
 
     return resource_dict
+
+
+def add_endpoint(service, endpoint_dict):
+    endpoint = Endpoint.objects.create(service=service)
+    for key, value in endpoint_dict.iteritems():
+        EndpointData.objects.create(
+            endpoint=endpoint, key=key, value=value)
+
+
+class ServiceException(Exception):
+    pass
+
+
+def add_service(component, name, service_type, endpoints):
+    defaults = {'component': component,
+                'type': service_type,
+                }
+    service, created = Service.objects.get_or_create(
+        name=name, defaults=defaults)
+
+    if not created:
+        if service.component != component:
+            m = ("There is already a service named %s registered by %s." %
+                 (name, service.component.name))
+            raise ServiceException(m)
+        service.endpoints.all().delete()
+    else:
+        service.component = component
+        service.type = service_type
+        service.save()
+
+    for endpoint in endpoints:
+        add_endpoint(service, endpoint)
+
+    return not created

snf-astakos-app/astakos/im/tests/api.py

-# Copyright 2011 GRNET S.A. All rights reserved.
+# Copyright 2011, 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -48,32 +48,36 @@ u = lambda url: ROOT + url
 class QuotaAPITest(TestCase):
     def test_0(self):
         client = Client()
+
+        component1 = Component.objects.create(name="comp1")
+        register.add_service(component1, "service1", "type1", [])
         # custom service resources
-        service1 = Service.objects.create(
-            name="service1", api_url="http://service1.api")
         resource11 = {"name": "service1.resource11",
                       "desc": "resource11 desc",
+                      "service_type": "type1",
                       "allow_in_projects": True}
-        r, _ = resources.add_resource(service1, resource11)
-        resources.update_resource(r, 100)
+        r, _ = register.add_resource(resource11)
+        register.update_resource(r, 100)
         resource12 = {"name": "service1.resource12",
                       "desc": "resource11 desc",
+                      "service_type": "type1",
                       "unit": "bytes"}
-        r, _ = resources.add_resource(service1, resource12)
-        resources.update_resource(r, 1024)
+        r, _ = register.add_resource(resource12)
+        register.update_resource(r, 1024)
 
         # create user
         user = get_local_user('test@grnet.gr')
         quotas.qh_sync_user(user)
 
+        component2 = Component.objects.create(name="comp2")
+        register.add_service(component2, "service2", "type2", [])
         # create another service
-        service2 = Service.objects.create(
-            name="service2", api_url="http://service2.api")
         resource21 = {"name": "service2.resource21",
                       "desc": "resource11 desc",
+                      "service_type": "type2",
                       "allow_in_projects": False}
-        r, _ = resources.add_resource(service2, resource21)
-        resources.update_resource(r, 3)
+        r, _ = register.add_resource(resource21)
+        register.update_resource(r, 3)
 
         resource_names = [r['name'] for r in
                           [resource11, resource12, resource21]]
@@ -101,7 +105,7 @@ class QuotaAPITest(TestCase):
         r = client.get(u('service_quotas'))
         self.assertEqual(r.status_code, 401)
 
-        s1_headers = {'HTTP_X_AUTH_TOKEN': service1.auth_token}
+        s1_headers = {'HTTP_X_AUTH_TOKEN': component1.auth_token}
         r = client.get(u('service_quotas'), **s1_headers)
         self.assertEqual(r.status_code, 200)
         body = json.loads(r.content)

snf-astakos-app/astakos/im/tests/common.py

@@ -60,7 +60,7 @@ from datetime import timedelta
 from astakos.im import messages
 from astakos.im import auth_providers
 from astakos.im import quotas
-from astakos.im import resources
+from astakos.im import register
 
 from django.conf import settings
 

snf-astakos-app/astakos/im/tests/projects.py

-# Copyright 2011 GRNET S.A. All rights reserved.
+# Copyright 2011, 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -40,19 +40,15 @@ class TestProjects(TestCase):
     """
     def setUp(self):
         # astakos resources
-        self.astakos_service = Service.objects.create(name="astakos",
-                                                      api_url="/astakos/api/")
         self.resource = Resource.objects.create(name="astakos.pending_app",
                                                 uplimit=0,
                                                 allow_in_projects=False,
-                                                service=self.astakos_service)
+                                                service_type="astakos")
 
         # custom service resources
-        self.service = Service.objects.create(name="service1",
-                                              api_url="http://service.api")