Skip to content

GitLab

Commit 8db8ccfb authored by Giorgos Korfiatis's avatar Giorgos Korfiatis Committed by Christos Stavrakakis
Browse files

cyclades: Use astakosclient for quotas and commissions

parent d8a59a0f
Show whitespace changes
Inline Side-by-side
snf-cyclades-app/synnefo/api/management/commands/cyclades-astakos-migrate-013.py
View file @ 8db8ccfb
# Copyright 2012 GRNET S.A. All rights reserved.
# Copyright 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -41,7 +41,6 @@ from django.core.management.base import NoArgsCommand, CommandError, BaseCommand
from django.db import transaction
from django.conf import settings
from synnefo.quotas import get_quota_holder
from synnefo.api.util import get_existing_users
from synnefo.lib.utils import case_unique
from synnefo.db.models import Network, VirtualMachine
......
snf-cyclades-app/synnefo/quotas/__init__.py
View file @ 8db8ccfb
# Copyright 2012 GRNET S.A. All rights reserved.
# Copyright 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
......@@ -31,93 +31,31 @@ from functools import wraps
from contextlib import contextmanager
from snf_django.lib.api import faults
from synnefo.db.models import QuotaHolderSerial, VirtualMachine, Network
from synnefo.db.models import QuotaHolderSerial
from synnefo.settings import CYCLADES_USE_QUOTAHOLDER
if CYCLADES_USE_QUOTAHOLDER:
from synnefo.settings import (CYCLADES_QUOTAHOLDER_URL,
CYCLADES_QUOTAHOLDER_TOKEN,
CYCLADES_QUOTAHOLDER_POOLSIZE)
from synnefo.lib.quotaholder import QuotaholderClient
else:
from synnefo.settings import (VMS_USER_QUOTA, MAX_VMS_PER_USER,
NETWORKS_USER_QUOTA, MAX_NETWORKS_PER_USER)
from synnefo.lib.quotaholder.api import (NoCapacityError, NoQuantityError,
NoEntityError, CallError)
from synnefo.settings import (CYCLADES_ASTAKOS_SERVICE_TOKEN as ASTAKOS_TOKEN,
ASTAKOS_URL)
from astakosclient import AstakosClient
from astakosclient.errors import AstakosClientException, QuotaLimit
import logging
log = logging.getLogger(__name__)
DEFAULT_SOURCE = 'system'
class DummySerial(QuotaHolderSerial):
accepted = True
rejected = True
pending = True
id = None
def save(*args, **kwargs):
pass
class DummyQuotaholderClient(object):
def issue_commission(self, **commission_info):
provisions = commission_info["provisions"]
userid = commission_info["target"]
for provision in provisions:
entity, resource, size = provision
if resource == "cyclades.vm" and size > 0:
user_vms = VirtualMachine.objects.filter(userid=userid,
deleted=False).count()
user_vm_limit = VMS_USER_QUOTA.get(userid, MAX_VMS_PER_USER)
log.debug("Users VMs %s User Limits %s", user_vms,
user_vm_limit)
if user_vms + size > user_vm_limit:
raise NoQuantityError(source='cyclades',
target=userid,
resource=resource,
requested=size,
current=user_vms,
limit=user_vm_limit)
if resource == "cyclades.network.private" and size > 0:
user_networks = Network.objects.filter(userid=userid,
deleted=False).count()
user_network_limit =\
NETWORKS_USER_QUOTA.get(userid, MAX_NETWORKS_PER_USER)
if user_networks + size > user_network_limit:
raise NoQuantityError(source='cyclades',
target=userid,
resource=resource,
requested=size,
current=user_networks,
limit=user_network_limit)
return None
def accept_commission(self, *args, **kwargs):
pass
def reject_commission(self, *args, **kwargs):
pass
def get_pending_commissions(self, *args, **kwargs):
return []
@contextmanager
def get_quota_holder():
"""Context manager for using a QuotaHolder."""
if CYCLADES_USE_QUOTAHOLDER:
quotaholder = QuotaholderClient(CYCLADES_QUOTAHOLDER_URL,
token=CYCLADES_QUOTAHOLDER_TOKEN,
poolsize=CYCLADES_QUOTAHOLDER_POOLSIZE)
else:
quotaholder = DummyQuotaholderClient()
try:
yield quotaholder
finally:
pass
class Quotaholder(object):
_object = None
@classmethod
def get(cls):
if cls._object is None:
cls._object = AstakosClient(
ASTAKOS_URL,
use_pool=True,
logger=log)
return cls._object
def uses_commission(func):
......@@ -171,10 +109,8 @@ def accept_commission(serials, update_db=True):
s.accepted = True
s.save()
with get_quota_holder() as qh:
qh.accept_commission(context={},
clientkey='cyclades',
serials=[s.serial for s in serials])
accept_serials = [s.serial for s in serials]
qh_resolve_commissions(accept=accept_serials)
def reject_commission(serials, update_db=True):
......@@ -189,13 +125,12 @@ def reject_commission(serials, update_db=True):
s.rejected = True
s.save()
with get_quota_holder() as qh:
qh.reject_commission(context={},
clientkey='cyclades',
serials=[s.serial for s in serials])
reject_serials = [s.serial for s in serials]
qh_resolve_commissions(reject=reject_serials)
def issue_commission(**commission_info):
def issue_commission(user, source, provisions,
force=False, auto_accept=False):
"""Issue a new commission to the quotaholder.
Issue a new commission to the quotaholder, and create the
......@@ -203,20 +138,20 @@ def issue_commission(**commission_info):
"""
with get_quota_holder() as qh:
qh = Quotaholder.get()
try:
serial = qh.issue_commission(**commission_info)
except (NoCapacityError, NoQuantityError) as e:
msg, details = render_quotaholder_exception(e)
serial = qh.issue_one_commission(ASTAKOS_TOKEN,
user, source, provisions,
force, auto_accept)
except QuotaLimit as e:
msg, details = render_overlimit_exception(e)
raise faults.OverLimit(msg, details=details)
except CallError as e:
except AstakosClientException as e:
log.exception("Unexpected error")
raise
if serial:
return QuotaHolderSerial.objects.create(serial=serial)
elif not CYCLADES_USE_QUOTAHOLDER:
return DummySerial()
else:
raise Exception("No serial")
......@@ -228,10 +163,8 @@ def issue_commission(**commission_info):
def issue_vm_commission(user, flavor, delete=False):
resources = get_server_resources(flavor)
commission_info = create_commission(user, resources, delete)
return issue_commission(**commission_info)
resources = prepare(get_server_resources(flavor), delete)
return issue_commission(user, DEFAULT_SOURCE, resources)
def get_server_resources(flavor):
......@@ -244,65 +177,46 @@ def get_server_resources(flavor):
def issue_network_commission(user, delete=False):
resources = get_network_resources()
commission_info = create_commission(user, resources, delete)
return issue_commission(**commission_info)
resources = prepare(get_network_resources(), delete)
return issue_commission(user, DEFAULT_SOURCE, resources)
def get_network_resources():
return {"network.private": 1}
def invert_resources(resources_dict):
def prepare(resources_dict, delete):
if delete:
return dict((r, -s) for r, s in resources_dict.items())
return resources_dict
def create_commission(user, resources, delete=False):
if delete:
resources = invert_resources(resources)
provisions = [('cyclades', 'cyclades.' + r, s)
for r, s in resources.items()]
return {"context": {},
"target": user,
"key": "1",
"clientkey": "cyclades",
#"owner": "",
#"ownerkey": "1",
"name": "",
"provisions": provisions}
##
## Reconcile pending commissions
##
def accept_commissions(accepted):
with get_quota_holder() as qh:
qh.accept_commission(context={},
clientkey='cyclades',
serials=accepted)
qh_resolve_commissions(accept=accepted)
def reject_commissions(rejected):
with get_quota_holder() as qh:
qh.reject_commission(context={},
clientkey='cyclades',
serials=rejected)
qh_resolve_commissions(reject=rejected)
def fix_pending_commissions():
(accepted, rejected) = resolve_pending_commissions()
qh_resolve_commissions(accepted, rejected)
def qh_resolve_commissions(accept=None, reject=None):
if accept is None:
accept = []
if reject is None:
reject = []
with get_quota_holder() as qh:
if accepted:
qh.accept_commission(context={},
clientkey='cyclades',
serials=accepted)
if rejected:
qh.reject_commission(context={},
clientkey='cyclades',
serials=rejected)
qh = Quotaholder.get()
qh.resolve_commissions(ASTAKOS_TOKEN, accept, reject)
def resolve_pending_commissions():
......@@ -333,28 +247,30 @@ def resolve_pending_commissions():
def get_quotaholder_pending():
with get_quota_holder() as qh:
pending_serials = qh.get_pending_commissions(context={},
clientkey='cyclades')
qh = Quotaholder.get()
pending_serials = qh.get_pending_commissions(ASTAKOS_TOKEN)
return pending_serials
def render_quotaholder_exception(e):
def render_overlimit_exception(e):
resource_name = {"vm": "Virtual Machine",
"cpu": "CPU",
"ram": "RAM",
"network.private": "Private Network"}
res = e.resource.replace("cyclades.", "", 1)
details = e.details
data = details['overLimit']['data']
available = data['available']
provision = data['provision']
requested = provision['quantity']
resource = provision['resource']
res = resource.replace("cyclades.", "", 1)
try:
resource = resource_name[res]
except KeyError:
resource = res
requested = e.requested
current = e.current
limit = e.limit
msg = "Resource Limit Exceeded for your account."
details = "Limit for resource '%s' exceeded for your account."\
" Current value: %s, Limit: %s, Requested: %s"\
% (resource, current, limit, requested)
" Available: %s, Requested: %s"\
% (resource, available, requested)
return msg, details
snf-cyclades-app/synnefo/quotas/management/commands/cyclades-reset-usage.py
View file @ 8db8ccfb
# Copyright 2012 GRNET S.A. All rights reserved.
# Copyright 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -34,7 +34,7 @@
from django.core.management.base import BaseCommand
from optparse import make_option
from synnefo.quotas import get_quota_holder
from synnefo.quotas import Quotaholder
from synnefo.quotas.util import get_db_holdings
......@@ -58,7 +58,7 @@ class Command(BaseCommand):
db_holdings = get_db_holdings(users)
# Create commissions
with get_quota_holder() as qh:
qh = Quotaholder.get()
for user, resources in db_holdings.items():
if not user:
continue
......
snf-cyclades-app/synnefo/quotas/management/commands/cyclades-usage-verify.py
View file @ 8db8ccfb
# Copyright 2012 GRNET S.A. All rights reserved.
# Copyright 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -34,7 +34,9 @@
from django.core.management.base import BaseCommand
from optparse import make_option
from synnefo.quotas.util import get_db_holdings, get_quotaholder_holdings
from synnefo.quotas import DEFAULT_SOURCE
from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings,
transform_quotas)
from synnefo.webproject.management.utils import pprint_table
......@@ -61,7 +63,7 @@ class Command(BaseCommand):
# Get info from DB
db_holdings = get_db_holdings(users)
users = db_holdings.keys()
qh_holdings = get_quotaholder_holdings(users)
qh_holdings = get_quotaholder_holdings(userid)
qh_users = qh_holdings.keys()
if len(qh_users) < len(users):
......@@ -73,31 +75,27 @@ class Command(BaseCommand):
unsynced = []
for user in users:
db = db_holdings[user]
qh = qh_holdings[user]
if not self.verify_resources(user, db.keys(), qh.keys()):
qh_all = qh_holdings[user]
# Assuming only one source
qh = qh_all[DEFAULT_SOURCE]
qh = transform_quotas(qh)
for resource, (value, value1) in qh.iteritems:
db_value = db.pop(resource, None)
if value != value1:
write("Commission pending for %s"
% str((user, resource)))
continue
if db_value is None:
write("Resource %s exists in QH for %s but not in DB\n"
% (resource, user))
elif db_value != value:
data = (user, resource, str(db_value), str(value))
unsynced.append(data)
for res in db.keys():
if db[res] != qh[res]:
unsynced.append((user, res, str(db[res]), str(qh[res])))
for resource, db_value in db.iteritems():
write("Resource %s exists in DB for %s but not in QH\n"
% (resource, user))
if unsynced:
pprint_table(self.stderr, unsynced, headers)
def verify_resources(self, user, db_resources, qh_resources):
write = self.stderr.write
db_res = set(db_resources)
qh_res = set(qh_resources)
if qh_res == db_res:
return True
db_extra = db_res - qh_res
if db_extra:
for res in db_extra:
write("Resource %s exists in DB for %s but not in QH\n"
% (res, user))
qh_extra = qh_res - db_res
if qh_extra:
for res in qh_extra:
write("Resource %s exists in QH for %s but not in DB\n"
% (res, user))
return False
snf-cyclades-app/synnefo/quotas/util.py
View file @ 8db8ccfb
# Copyright 2012 GRNET S.A. All rights reserved.
# Copyright 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -34,7 +34,7 @@
from django.db.models import Sum, Count
from synnefo.db.models import VirtualMachine, Network
from synnefo.quotas import get_quota_holder, NoEntityError
from synnefo.quotas import Quotaholder, ASTAKOS_TOKEN
def get_db_holdings(users=None):
......@@ -74,31 +74,22 @@ def get_db_holdings(users=None):
return holdings
def get_quotaholder_holdings(users=[]):
"""Get holdings from Quotaholder.
def get_quotaholder_holdings(user=None):
"""Get quotas from Quotaholder for all Cyclades resources.
If the entity for the user does not exist in quotaholder, no holding
is returned.
Returns quotas for all users, unless a single user is specified.
"""
users = filter(lambda u: not u is None, users)
holdings = {}
with get_quota_holder() as qh:
list_holdings = [(user, "1") for user in users]
(qh_holdings, rejected) = qh.list_holdings(context={},
list_holdings=list_holdings)
found_users = filter(lambda u: not u in rejected, users)
for user, user_holdings in zip(found_users, qh_holdings):
if not user_holdings:
continue
for h in user_holdings:
assert(h[0] == user)
user_holdings = filter(lambda x: x[1].startswith("cyclades."),
user_holdings)
holdings[user] = dict(map(decode_holding, user_holdings))
return holdings
qh = Quotaholder.get()
return qh.get_service_quotas(ASTAKOS_TOKEN, user)
def decode_holding(holding):
entity, resource, imported, exported, returned, released = holding
def transform_quotas(quotas):
d = {}
for resource, counters in quotas.iteritems():
res = resource.replace("cyclades.", "")
return (res, imported - exported + returned - released)
available = counters['available']
limit = counters['limit']
used = counters['used']
used_max = limit - available
d[res] = (used, used_max)
return d
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment