Commit 8db8ccfb authored by Giorgos Korfiatis's avatar Giorgos Korfiatis Committed by Christos Stavrakakis
Browse files

cyclades: Use astakosclient for quotas and commissions

parent d8a59a0f
# Copyright 2012 GRNET S.A. All rights reserved. # Copyright 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -41,7 +41,6 @@ from django.core.management.base import NoArgsCommand, CommandError, BaseCommand ...@@ -41,7 +41,6 @@ from django.core.management.base import NoArgsCommand, CommandError, BaseCommand
from django.db import transaction from django.db import transaction
from django.conf import settings from django.conf import settings
from synnefo.quotas import get_quota_holder
from synnefo.api.util import get_existing_users from synnefo.api.util import get_existing_users
from synnefo.lib.utils import case_unique from synnefo.lib.utils import case_unique
from synnefo.db.models import Network, VirtualMachine from synnefo.db.models import Network, VirtualMachine
......
# Copyright 2012 GRNET S.A. All rights reserved. # Copyright 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions # modification, are permitted provided that the following conditions
...@@ -31,93 +31,31 @@ from functools import wraps ...@@ -31,93 +31,31 @@ from functools import wraps
from contextlib import contextmanager from contextlib import contextmanager
from snf_django.lib.api import faults from snf_django.lib.api import faults
from synnefo.db.models import QuotaHolderSerial, VirtualMachine, Network from synnefo.db.models import QuotaHolderSerial
from synnefo.settings import CYCLADES_USE_QUOTAHOLDER from synnefo.settings import CYCLADES_USE_QUOTAHOLDER
if CYCLADES_USE_QUOTAHOLDER: from synnefo.settings import (CYCLADES_ASTAKOS_SERVICE_TOKEN as ASTAKOS_TOKEN,
from synnefo.settings import (CYCLADES_QUOTAHOLDER_URL, ASTAKOS_URL)
CYCLADES_QUOTAHOLDER_TOKEN, from astakosclient import AstakosClient
CYCLADES_QUOTAHOLDER_POOLSIZE) from astakosclient.errors import AstakosClientException, QuotaLimit
from synnefo.lib.quotaholder import QuotaholderClient
else:
from synnefo.settings import (VMS_USER_QUOTA, MAX_VMS_PER_USER,
NETWORKS_USER_QUOTA, MAX_NETWORKS_PER_USER)
from synnefo.lib.quotaholder.api import (NoCapacityError, NoQuantityError,
NoEntityError, CallError)
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
DEFAULT_SOURCE = 'system'
class DummySerial(QuotaHolderSerial):
accepted = True
rejected = True
pending = True
id = None
def save(*args, **kwargs):
pass
class DummyQuotaholderClient(object):
def issue_commission(self, **commission_info):
provisions = commission_info["provisions"]
userid = commission_info["target"]
for provision in provisions:
entity, resource, size = provision
if resource == "cyclades.vm" and size > 0:
user_vms = VirtualMachine.objects.filter(userid=userid,
deleted=False).count()
user_vm_limit = VMS_USER_QUOTA.get(userid, MAX_VMS_PER_USER)
log.debug("Users VMs %s User Limits %s", user_vms,
user_vm_limit)
if user_vms + size > user_vm_limit:
raise NoQuantityError(source='cyclades',
target=userid,
resource=resource,
requested=size,
current=user_vms,
limit=user_vm_limit)
if resource == "cyclades.network.private" and size > 0:
user_networks = Network.objects.filter(userid=userid,
deleted=False).count()
user_network_limit =\
NETWORKS_USER_QUOTA.get(userid, MAX_NETWORKS_PER_USER)
if user_networks + size > user_network_limit:
raise NoQuantityError(source='cyclades',
target=userid,
resource=resource,
requested=size,
current=user_networks,
limit=user_network_limit)
return None
def accept_commission(self, *args, **kwargs):
pass
def reject_commission(self, *args, **kwargs):
pass
def get_pending_commissions(self, *args, **kwargs):
return []
@contextmanager
def get_quota_holder():
"""Context manager for using a QuotaHolder."""
if CYCLADES_USE_QUOTAHOLDER:
quotaholder = QuotaholderClient(CYCLADES_QUOTAHOLDER_URL,
token=CYCLADES_QUOTAHOLDER_TOKEN,
poolsize=CYCLADES_QUOTAHOLDER_POOLSIZE)
else:
quotaholder = DummyQuotaholderClient()
try: class Quotaholder(object):
yield quotaholder _object = None
finally:
pass @classmethod
def get(cls):
if cls._object is None:
cls._object = AstakosClient(
ASTAKOS_URL,
use_pool=True,
logger=log)
return cls._object
def uses_commission(func): def uses_commission(func):
...@@ -171,10 +109,8 @@ def accept_commission(serials, update_db=True): ...@@ -171,10 +109,8 @@ def accept_commission(serials, update_db=True):
s.accepted = True s.accepted = True
s.save() s.save()
with get_quota_holder() as qh: accept_serials = [s.serial for s in serials]
qh.accept_commission(context={}, qh_resolve_commissions(accept=accept_serials)
clientkey='cyclades',
serials=[s.serial for s in serials])
def reject_commission(serials, update_db=True): def reject_commission(serials, update_db=True):
...@@ -189,13 +125,12 @@ def reject_commission(serials, update_db=True): ...@@ -189,13 +125,12 @@ def reject_commission(serials, update_db=True):
s.rejected = True s.rejected = True
s.save() s.save()
with get_quota_holder() as qh: reject_serials = [s.serial for s in serials]
qh.reject_commission(context={}, qh_resolve_commissions(reject=reject_serials)
clientkey='cyclades',
serials=[s.serial for s in serials])
def issue_commission(**commission_info): def issue_commission(user, source, provisions,
force=False, auto_accept=False):
"""Issue a new commission to the quotaholder. """Issue a new commission to the quotaholder.
Issue a new commission to the quotaholder, and create the Issue a new commission to the quotaholder, and create the
...@@ -203,20 +138,20 @@ def issue_commission(**commission_info): ...@@ -203,20 +138,20 @@ def issue_commission(**commission_info):
""" """
with get_quota_holder() as qh: qh = Quotaholder.get()
try: try:
serial = qh.issue_commission(**commission_info) serial = qh.issue_one_commission(ASTAKOS_TOKEN,
except (NoCapacityError, NoQuantityError) as e: user, source, provisions,
msg, details = render_quotaholder_exception(e) force, auto_accept)
raise faults.OverLimit(msg, details=details) except QuotaLimit as e:
except CallError as e: msg, details = render_overlimit_exception(e)
log.exception("Unexpected error") raise faults.OverLimit(msg, details=details)
raise except AstakosClientException as e:
log.exception("Unexpected error")
raise
if serial: if serial:
return QuotaHolderSerial.objects.create(serial=serial) return QuotaHolderSerial.objects.create(serial=serial)
elif not CYCLADES_USE_QUOTAHOLDER:
return DummySerial()
else: else:
raise Exception("No serial") raise Exception("No serial")
...@@ -228,10 +163,8 @@ def issue_commission(**commission_info): ...@@ -228,10 +163,8 @@ def issue_commission(**commission_info):
def issue_vm_commission(user, flavor, delete=False): def issue_vm_commission(user, flavor, delete=False):
resources = get_server_resources(flavor) resources = prepare(get_server_resources(flavor), delete)
commission_info = create_commission(user, resources, delete) return issue_commission(user, DEFAULT_SOURCE, resources)
return issue_commission(**commission_info)
def get_server_resources(flavor): def get_server_resources(flavor):
...@@ -244,33 +177,19 @@ def get_server_resources(flavor): ...@@ -244,33 +177,19 @@ def get_server_resources(flavor):
def issue_network_commission(user, delete=False): def issue_network_commission(user, delete=False):
resources = get_network_resources() resources = prepare(get_network_resources(), delete)
commission_info = create_commission(user, resources, delete) return issue_commission(user, DEFAULT_SOURCE, resources)
return issue_commission(**commission_info)
def get_network_resources(): def get_network_resources():
return {"network.private": 1} return {"network.private": 1}
def invert_resources(resources_dict): def prepare(resources_dict, delete):
return dict((r, -s) for r, s in resources_dict.items())
def create_commission(user, resources, delete=False):
if delete: if delete:
resources = invert_resources(resources) return dict((r, -s) for r, s in resources_dict.items())
provisions = [('cyclades', 'cyclades.' + r, s) return resources_dict
for r, s in resources.items()]
return {"context": {},
"target": user,
"key": "1",
"clientkey": "cyclades",
#"owner": "",
#"ownerkey": "1",
"name": "",
"provisions": provisions}
## ##
## Reconcile pending commissions ## Reconcile pending commissions
...@@ -278,31 +197,26 @@ def create_commission(user, resources, delete=False): ...@@ -278,31 +197,26 @@ def create_commission(user, resources, delete=False):
def accept_commissions(accepted): def accept_commissions(accepted):
with get_quota_holder() as qh: qh_resolve_commissions(accept=accepted)
qh.accept_commission(context={},
clientkey='cyclades',
serials=accepted)
def reject_commissions(rejected): def reject_commissions(rejected):
with get_quota_holder() as qh: qh_resolve_commissions(reject=rejected)
qh.reject_commission(context={},
clientkey='cyclades',
serials=rejected)
def fix_pending_commissions(): def fix_pending_commissions():
(accepted, rejected) = resolve_pending_commissions() (accepted, rejected) = resolve_pending_commissions()
qh_resolve_commissions(accepted, rejected)
def qh_resolve_commissions(accept=None, reject=None):
if accept is None:
accept = []
if reject is None:
reject = []
with get_quota_holder() as qh: qh = Quotaholder.get()
if accepted: qh.resolve_commissions(ASTAKOS_TOKEN, accept, reject)
qh.accept_commission(context={},
clientkey='cyclades',
serials=accepted)
if rejected:
qh.reject_commission(context={},
clientkey='cyclades',
serials=rejected)
def resolve_pending_commissions(): def resolve_pending_commissions():
...@@ -333,28 +247,30 @@ def resolve_pending_commissions(): ...@@ -333,28 +247,30 @@ def resolve_pending_commissions():
def get_quotaholder_pending(): def get_quotaholder_pending():
with get_quota_holder() as qh: qh = Quotaholder.get()
pending_serials = qh.get_pending_commissions(context={}, pending_serials = qh.get_pending_commissions(ASTAKOS_TOKEN)
clientkey='cyclades')
return pending_serials return pending_serials
def render_quotaholder_exception(e): def render_overlimit_exception(e):
resource_name = {"vm": "Virtual Machine", resource_name = {"vm": "Virtual Machine",
"cpu": "CPU", "cpu": "CPU",
"ram": "RAM", "ram": "RAM",
"network.private": "Private Network"} "network.private": "Private Network"}
res = e.resource.replace("cyclades.", "", 1) details = e.details
data = details['overLimit']['data']
available = data['available']
provision = data['provision']
requested = provision['quantity']
resource = provision['resource']
res = resource.replace("cyclades.", "", 1)
try: try:
resource = resource_name[res] resource = resource_name[res]
except KeyError: except KeyError:
resource = res resource = res
requested = e.requested
current = e.current
limit = e.limit
msg = "Resource Limit Exceeded for your account." msg = "Resource Limit Exceeded for your account."
details = "Limit for resource '%s' exceeded for your account."\ details = "Limit for resource '%s' exceeded for your account."\
" Current value: %s, Limit: %s, Requested: %s"\ " Available: %s, Requested: %s"\
% (resource, current, limit, requested) % (resource, available, requested)
return msg, details return msg, details
# Copyright 2012 GRNET S.A. All rights reserved. # Copyright 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from optparse import make_option from optparse import make_option
from synnefo.quotas import get_quota_holder from synnefo.quotas import Quotaholder
from synnefo.quotas.util import get_db_holdings from synnefo.quotas.util import get_db_holdings
...@@ -58,19 +58,19 @@ class Command(BaseCommand): ...@@ -58,19 +58,19 @@ class Command(BaseCommand):
db_holdings = get_db_holdings(users) db_holdings = get_db_holdings(users)
# Create commissions # Create commissions
with get_quota_holder() as qh: qh = Quotaholder.get()
for user, resources in db_holdings.items(): for user, resources in db_holdings.items():
if not user: if not user:
continue continue
reset_holding = [] reset_holding = []
for res, val in resources.items(): for res, val in resources.items():
reset_holding.append((user, "cyclades." + res, "1", val, 0, reset_holding.append((user, "cyclades." + res, "1", val, 0,
0, 0)) 0, 0))
if not options['dry_run']: if not options['dry_run']:
try: try:
qh.reset_holding(context={}, qh.reset_holding(context={},
reset_holding=reset_holding) reset_holding=reset_holding)
except Exception as e: except Exception as e:
self.stderr.write("Can not set up holding:%s" % e) self.stderr.write("Can not set up holding:%s" % e)
else: else:
self.stdout.write("Reseting holding: %s\n" % reset_holding) self.stdout.write("Reseting holding: %s\n" % reset_holding)
# Copyright 2012 GRNET S.A. All rights reserved. # Copyright 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -34,7 +34,9 @@ ...@@ -34,7 +34,9 @@
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from optparse import make_option from optparse import make_option
from synnefo.quotas.util import get_db_holdings, get_quotaholder_holdings from synnefo.quotas import DEFAULT_SOURCE
from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings,
transform_quotas)
from synnefo.webproject.management.utils import pprint_table from synnefo.webproject.management.utils import pprint_table
...@@ -61,7 +63,7 @@ class Command(BaseCommand): ...@@ -61,7 +63,7 @@ class Command(BaseCommand):
# Get info from DB # Get info from DB
db_holdings = get_db_holdings(users) db_holdings = get_db_holdings(users)
users = db_holdings.keys() users = db_holdings.keys()
qh_holdings = get_quotaholder_holdings(users) qh_holdings = get_quotaholder_holdings(userid)
qh_users = qh_holdings.keys() qh_users = qh_holdings.keys()
if len(qh_users) < len(users): if len(qh_users) < len(users):
...@@ -73,31 +75,27 @@ class Command(BaseCommand): ...@@ -73,31 +75,27 @@ class Command(BaseCommand):
unsynced = [] unsynced = []
for user in users: for user in users:
db = db_holdings[user] db = db_holdings[user]
qh = qh_holdings[user] qh_all = qh_holdings[user]
if not self.verify_resources(user, db.keys(), qh.keys()): # Assuming only one source
continue qh = qh_all[DEFAULT_SOURCE]
qh = transform_quotas(qh)
for res in db.keys(): for resource, (value, value1) in qh.iteritems:
if db[res] != qh[res]: db_value = db.pop(resource, None)
unsynced.append((user, res, str(db[res]), str(qh[res]))) if value != value1:
write("Commission pending for %s"
% str((user, resource)))
continue
if db_value is None:
write("Resource %s exists in QH for %s but not in DB\n"
% (resource, user))
elif db_value != value:
data = (user, resource, str(db_value), str(value))
unsynced.append(data)
for resource, db_value in db.iteritems():
write("Resource %s exists in DB for %s but not in QH\n"
% (resource, user))
if unsynced: if unsynced:
pprint_table(self.stderr, unsynced, headers) pprint_table(self.stderr, unsynced, headers)
def verify_resources(self, user, db_resources, qh_resources):
write = self.stderr.write
db_res = set(db_resources)
qh_res = set(qh_resources)
if qh_res == db_res:
return True
db_extra = db_res - qh_res
if db_extra:
for res in db_extra:
write("Resource %s exists in DB for %s but not in QH\n"
% (res, user))
qh_extra = qh_res - db_res
if qh_extra:
for res in qh_extra:
write("Resource %s exists in QH for %s but not in DB\n"
% (res, user))
return False
# Copyright 2012 GRNET S.A. All rights reserved. # Copyright 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
from django.db.models import Sum, Count from django.db.models import Sum, Count
from synnefo.db.models import VirtualMachine, Network from synnefo.db.models import VirtualMachine, Network
from synnefo.quotas import get_quota_holder, NoEntityError from synnefo.quotas import Quotaholder, ASTAKOS_TOKEN
def get_db_holdings(users=None): def get_db_holdings(users=None):
...@@ -74,31 +74,22 @@ def get_db_holdings(users=None): ...@@ -74,31 +74,22 @@ def get_db_holdings(users=None):
return holdings return holdings
def get_quotaholder_holdings(users=[]): def get_quotaholder_holdings(user=None):
"""Get holdings from Quotaholder. """Get quotas from Quotaholder for all Cyclades resources.
If the entity for the user does not exist in quotaholder, no holding Returns quotas for all users, unless a single user is specified.
is returned.
""" """
users = filter(lambda u: not u is None, users) qh = Quotaholder.get()
holdings = {} return qh.get_service_quotas(ASTAKOS_TOKEN, user)
with get_quota_holder() as qh:
list_holdings = [(user, "1") for user in users]
(qh_holdings, rejected) = qh.list_holdings(context={},
list_holdings=list_holdings)
found_users = filter(lambda u: not u in rejected, users)
for user, user_holdings in zip(found_users, qh_holdings):
if not user_holdings:
continue
for h in user_holdings: