cyclades: Use astakosclient for quotas and commissions

snf-cyclades-app/synnefo/api/management/commands/cyclades-astakos-migrate-013.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -41,7 +41,6 @@ from django.core.management.base import NoArgsCommand, CommandError, BaseCommand
 from django.db import transaction
 from django.conf import settings
 
-from synnefo.quotas import get_quota_holder
 from synnefo.api.util import get_existing_users
 from synnefo.lib.utils import case_unique
 from synnefo.db.models import Network, VirtualMachine

snf-cyclades-app/synnefo/quotas/__init__.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -31,93 +31,31 @@ from functools import wraps
 from contextlib import contextmanager
 
 from snf_django.lib.api import faults
-from synnefo.db.models import QuotaHolderSerial, VirtualMachine, Network
+from synnefo.db.models import QuotaHolderSerial
 from synnefo.settings import CYCLADES_USE_QUOTAHOLDER
 
-if CYCLADES_USE_QUOTAHOLDER:
-    from synnefo.settings import (CYCLADES_QUOTAHOLDER_URL,
-                                  CYCLADES_QUOTAHOLDER_TOKEN,
-                                  CYCLADES_QUOTAHOLDER_POOLSIZE)
-    from synnefo.lib.quotaholder import QuotaholderClient
-else:
-    from synnefo.settings import (VMS_USER_QUOTA, MAX_VMS_PER_USER,
-                                  NETWORKS_USER_QUOTA, MAX_NETWORKS_PER_USER)
-
-from synnefo.lib.quotaholder.api import (NoCapacityError, NoQuantityError,
-                                         NoEntityError, CallError)
+from synnefo.settings import (CYCLADES_ASTAKOS_SERVICE_TOKEN as ASTAKOS_TOKEN,
+                              ASTAKOS_URL)
+from astakosclient import AstakosClient
+from astakosclient.errors import AstakosClientException, QuotaLimit
 
 import logging
 log = logging.getLogger(__name__)
 
+DEFAULT_SOURCE = 'system'
 
-class DummySerial(QuotaHolderSerial):
-    accepted = True
-    rejected = True
-    pending = True
-    id = None
-
-    def save(*args, **kwargs):
-        pass
-
-
-class DummyQuotaholderClient(object):
-    def issue_commission(self, **commission_info):
-        provisions = commission_info["provisions"]
-        userid = commission_info["target"]
-        for provision in provisions:
-            entity, resource, size = provision
-            if resource == "cyclades.vm" and size > 0:
-                user_vms = VirtualMachine.objects.filter(userid=userid,
-                                                         deleted=False).count()
-                user_vm_limit = VMS_USER_QUOTA.get(userid, MAX_VMS_PER_USER)
-                log.debug("Users VMs %s User Limits %s", user_vms,
-                          user_vm_limit)
-                if user_vms + size > user_vm_limit:
-                    raise NoQuantityError(source='cyclades',
-                                          target=userid,
-                                          resource=resource,
-                                          requested=size,
-                                          current=user_vms,
-                                          limit=user_vm_limit)
-            if resource == "cyclades.network.private" and size > 0:
-                user_networks = Network.objects.filter(userid=userid,
-                                                       deleted=False).count()
-                user_network_limit =\
-                    NETWORKS_USER_QUOTA.get(userid, MAX_NETWORKS_PER_USER)
-                if user_networks + size > user_network_limit:
-                    raise NoQuantityError(source='cyclades',
-                                          target=userid,
-                                          resource=resource,
-                                          requested=size,
-                                          current=user_networks,
-                                          limit=user_network_limit)
-
-        return None
-
-    def accept_commission(self, *args, **kwargs):
-        pass
-
-    def reject_commission(self, *args, **kwargs):
-        pass
-
-    def get_pending_commissions(self, *args, **kwargs):
-        return []
-
-
-@contextmanager
-def get_quota_holder():
-    """Context manager for using a QuotaHolder."""
-    if CYCLADES_USE_QUOTAHOLDER:
-        quotaholder = QuotaholderClient(CYCLADES_QUOTAHOLDER_URL,
-                                        token=CYCLADES_QUOTAHOLDER_TOKEN,
-                                        poolsize=CYCLADES_QUOTAHOLDER_POOLSIZE)
-    else:
-        quotaholder = DummyQuotaholderClient()
 
-    try:
-        yield quotaholder
-    finally:
-        pass
+class Quotaholder(object):
+    _object = None
+
+    @classmethod
+    def get(cls):
+        if cls._object is None:
+            cls._object = AstakosClient(
+                ASTAKOS_URL,
+                use_pool=True,
+                logger=log)
+        return cls._object
 
 
 def uses_commission(func):
@@ -171,10 +109,8 @@ def accept_commission(serials, update_db=True):
                 s.accepted = True
                 s.save()
 
-    with get_quota_holder() as qh:
-        qh.accept_commission(context={},
-                             clientkey='cyclades',
-                             serials=[s.serial for s in serials])
+    accept_serials = [s.serial for s in serials]
+    qh_resolve_commissions(accept=accept_serials)
 
 
 def reject_commission(serials, update_db=True):
@@ -189,13 +125,12 @@ def reject_commission(serials, update_db=True):
                 s.rejected = True
                 s.save()
 
-    with get_quota_holder() as qh:
-        qh.reject_commission(context={},
-                             clientkey='cyclades',
-                             serials=[s.serial for s in serials])
+    reject_serials = [s.serial for s in serials]
+    qh_resolve_commissions(reject=reject_serials)
 
 
-def issue_commission(**commission_info):
+def issue_commission(user, source, provisions,
+                     force=False, auto_accept=False):
     """Issue a new commission to the quotaholder.
 
     Issue a new commission to the quotaholder, and create the
@@ -203,20 +138,20 @@ def issue_commission(**commission_info):
 
     """
 
-    with get_quota_holder() as qh:
-        try:
-            serial = qh.issue_commission(**commission_info)
-        except (NoCapacityError, NoQuantityError) as e:
-            msg, details = render_quotaholder_exception(e)
-            raise faults.OverLimit(msg, details=details)
-        except CallError as e:
-            log.exception("Unexpected error")
-            raise
+    qh = Quotaholder.get()
+    try:
+        serial = qh.issue_one_commission(ASTAKOS_TOKEN,
+                                         user, source, provisions,
+                                         force, auto_accept)
+    except QuotaLimit as e:
+        msg, details = render_overlimit_exception(e)
+        raise faults.OverLimit(msg, details=details)
+    except AstakosClientException as e:
+        log.exception("Unexpected error")
+        raise
 
     if serial:
         return QuotaHolderSerial.objects.create(serial=serial)
-    elif not CYCLADES_USE_QUOTAHOLDER:
-        return DummySerial()
     else:
         raise Exception("No serial")
 
@@ -228,10 +163,8 @@ def issue_commission(**commission_info):
 
 
 def issue_vm_commission(user, flavor, delete=False):
-    resources = get_server_resources(flavor)
-    commission_info = create_commission(user, resources, delete)
-
-    return issue_commission(**commission_info)
+    resources = prepare(get_server_resources(flavor), delete)
+    return issue_commission(user, DEFAULT_SOURCE, resources)
 
 
 def get_server_resources(flavor):
@@ -244,33 +177,19 @@ def get_server_resources(flavor):
 
 
 def issue_network_commission(user, delete=False):
-    resources = get_network_resources()
-    commission_info = create_commission(user, resources, delete)
-
-    return issue_commission(**commission_info)
+    resources = prepare(get_network_resources(), delete)
+    return issue_commission(user, DEFAULT_SOURCE, resources)
 
 
 def get_network_resources():
     return {"network.private": 1}
 
 
-def invert_resources(resources_dict):
-    return dict((r, -s) for r, s in resources_dict.items())
-
-
-def create_commission(user, resources, delete=False):
+def prepare(resources_dict, delete):
     if delete:
-        resources = invert_resources(resources)
-    provisions = [('cyclades', 'cyclades.' + r, s)
-                  for r, s in resources.items()]
-    return {"context": {},
-            "target": user,
-            "key": "1",
-            "clientkey": "cyclades",
-            #"owner":      "",
-            #"ownerkey":   "1",
-            "name": "",
-            "provisions": provisions}
+        return dict((r, -s) for r, s in resources_dict.items())
+    return resources_dict
+
 
 ##
 ## Reconcile pending commissions
@@ -278,31 +197,26 @@ def create_commission(user, resources, delete=False):
 
 
 def accept_commissions(accepted):
-    with get_quota_holder() as qh:
-        qh.accept_commission(context={},
-                             clientkey='cyclades',
-                             serials=accepted)
+    qh_resolve_commissions(accept=accepted)
 
 
 def reject_commissions(rejected):
-    with get_quota_holder() as qh:
-            qh.reject_commission(context={},
-                                 clientkey='cyclades',
-                                 serials=rejected)
+    qh_resolve_commissions(reject=rejected)
 
 
 def fix_pending_commissions():
     (accepted, rejected) = resolve_pending_commissions()
+    qh_resolve_commissions(accepted, rejected)
+
+
+def qh_resolve_commissions(accept=None, reject=None):
+    if accept is None:
+        accept = []
+    if reject is None:
+        reject = []
 
-    with get_quota_holder() as qh:
-        if accepted:
-            qh.accept_commission(context={},
-                                 clientkey='cyclades',
-                                 serials=accepted)
-        if rejected:
-            qh.reject_commission(context={},
-                                 clientkey='cyclades',
-                                 serials=rejected)
+    qh = Quotaholder.get()
+    qh.resolve_commissions(ASTAKOS_TOKEN, accept, reject)
 
 
 def resolve_pending_commissions():
@@ -333,28 +247,30 @@ def resolve_pending_commissions():
 
 
 def get_quotaholder_pending():
-    with get_quota_holder() as qh:
-        pending_serials = qh.get_pending_commissions(context={},
-                                                     clientkey='cyclades')
+    qh = Quotaholder.get()
+    pending_serials = qh.get_pending_commissions(ASTAKOS_TOKEN)
     return pending_serials
 
 
-def render_quotaholder_exception(e):
+def render_overlimit_exception(e):
     resource_name = {"vm": "Virtual Machine",
                      "cpu": "CPU",
                      "ram": "RAM",
                      "network.private": "Private Network"}
-    res = e.resource.replace("cyclades.", "", 1)
+    details = e.details
+    data = details['overLimit']['data']
+    available = data['available']
+    provision = data['provision']
+    requested = provision['quantity']
+    resource = provision['resource']
+    res = resource.replace("cyclades.", "", 1)
     try:
         resource = resource_name[res]
     except KeyError:
         resource = res
 
-    requested = e.requested
-    current = e.current
-    limit = e.limit
     msg = "Resource Limit Exceeded for your account."
     details = "Limit for resource '%s' exceeded for your account."\
-              " Current value: %s, Limit: %s, Requested: %s"\
-              % (resource, current, limit, requested)
+              " Available: %s, Requested: %s"\
+              % (resource, available, requested)
     return msg, details

snf-cyclades-app/synnefo/quotas/management/commands/cyclades-reset-usage.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -34,7 +34,7 @@
 from django.core.management.base import BaseCommand
 from optparse import make_option
 
-from synnefo.quotas import get_quota_holder
+from synnefo.quotas import Quotaholder
 from synnefo.quotas.util import get_db_holdings
 
 
@@ -58,19 +58,19 @@ class Command(BaseCommand):
         db_holdings = get_db_holdings(users)
 
         # Create commissions
-        with get_quota_holder() as qh:
-            for user, resources in db_holdings.items():
-                if not user:
-                    continue
-                reset_holding = []
-                for res, val in resources.items():
-                    reset_holding.append((user, "cyclades." + res, "1", val, 0,
-                                          0, 0))
-                if not options['dry_run']:
-                    try:
-                        qh.reset_holding(context={},
-                                         reset_holding=reset_holding)
-                    except Exception as e:
-                        self.stderr.write("Can not set up holding:%s" % e)
-                else:
-                    self.stdout.write("Reseting holding: %s\n" % reset_holding)
+        qh = Quotaholder.get()
+        for user, resources in db_holdings.items():
+            if not user:
+                continue
+            reset_holding = []
+            for res, val in resources.items():
+                reset_holding.append((user, "cyclades." + res, "1", val, 0,
+                                      0, 0))
+            if not options['dry_run']:
+                try:
+                    qh.reset_holding(context={},
+                                     reset_holding=reset_holding)
+                except Exception as e:
+                    self.stderr.write("Can not set up holding:%s" % e)
+            else:
+                self.stdout.write("Reseting holding: %s\n" % reset_holding)

snf-cyclades-app/synnefo/quotas/management/commands/cyclades-usage-verify.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -34,7 +34,9 @@
 from django.core.management.base import BaseCommand
 from optparse import make_option
 
-from synnefo.quotas.util import get_db_holdings, get_quotaholder_holdings
+from synnefo.quotas import DEFAULT_SOURCE
+from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings,
+                                 transform_quotas)
 from synnefo.webproject.management.utils import pprint_table
 
 
@@ -61,7 +63,7 @@ class Command(BaseCommand):
         # Get info from DB
         db_holdings = get_db_holdings(users)
         users = db_holdings.keys()
-        qh_holdings = get_quotaholder_holdings(users)
+        qh_holdings = get_quotaholder_holdings(userid)
         qh_users = qh_holdings.keys()
 
         if len(qh_users) < len(users):
@@ -73,31 +75,27 @@ class Command(BaseCommand):
         unsynced = []
         for user in users:
             db = db_holdings[user]
-            qh = qh_holdings[user]
-            if not self.verify_resources(user, db.keys(), qh.keys()):
-                continue
+            qh_all = qh_holdings[user]
+            # Assuming only one source
+            qh = qh_all[DEFAULT_SOURCE]
+            qh = transform_quotas(qh)
 
-            for res in db.keys():
-                if db[res] != qh[res]:
-                    unsynced.append((user, res, str(db[res]), str(qh[res])))
+            for resource, (value, value1) in qh.iteritems:
+                db_value = db.pop(resource, None)
+                if value != value1:
+                    write("Commission pending for %s"
+                          % str((user, resource)))
+                    continue
+                if db_value is None:
+                    write("Resource %s exists in QH for %s but not in DB\n"
+                          % (resource, user))
+                elif db_value != value:
+                    data = (user, resource, str(db_value), str(value))
+                    unsynced.append(data)
+
+            for resource, db_value in db.iteritems():
+                write("Resource %s exists in DB for %s but not in QH\n"
+                      % (resource, user))
 
         if unsynced:
             pprint_table(self.stderr, unsynced, headers)
-
-    def verify_resources(self, user, db_resources, qh_resources):
-        write = self.stderr.write
-        db_res = set(db_resources)
-        qh_res = set(qh_resources)
-        if qh_res == db_res:
-            return True
-        db_extra = db_res - qh_res
-        if db_extra:
-            for res in db_extra:
-                write("Resource %s exists in DB for %s but not in QH\n"
-                      % (res, user))
-        qh_extra = qh_res - db_res
-        if qh_extra:
-            for res in qh_extra:
-                write("Resource %s exists in QH for %s but not in DB\n"
-                      % (res, user))
-        return False

snf-cyclades-app/synnefo/quotas/util.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -34,7 +34,7 @@
 from django.db.models import Sum, Count
 
 from synnefo.db.models import VirtualMachine, Network
-from synnefo.quotas import get_quota_holder, NoEntityError
+from synnefo.quotas import Quotaholder, ASTAKOS_TOKEN
 
 
 def get_db_holdings(users=None):
@@ -74,31 +74,22 @@ def get_db_holdings(users=None):
     return holdings
 
 
-def get_quotaholder_holdings(users=[]):
-    """Get holdings from Quotaholder.
+def get_quotaholder_holdings(user=None):
+    """Get quotas from Quotaholder for all Cyclades resources.
 
-    If the entity for the user does not exist in quotaholder, no holding
-    is returned.
+    Returns quotas for all users, unless a single user is specified.
     """
-    users = filter(lambda u: not u is None, users)
-    holdings = {}
-    with get_quota_holder() as qh:
-        list_holdings = [(user, "1") for user in users]
-        (qh_holdings, rejected) = qh.list_holdings(context={},
-                                                   list_holdings=list_holdings)
-        found_users = filter(lambda u: not u in rejected, users)
-    for user, user_holdings in zip(found_users, qh_holdings):
-        if not user_holdings:
-            continue
-        for h in user_holdings:
+    qh = Quotaholder.get()
+    return qh.get_service_quotas(ASTAKOS_TOKEN, user)