Commit 42304537 authored by Giorgos Korfiatis's avatar Giorgos Korfiatis

astakos: Sync project-based quota

Update quota code to take into account the differentiated sources.
There is no more need to lock the user when updating quota; locking
the project is now adequate.

Update quota listing in management commands; introduce option --quota in
parent 4d04cadb
......@@ -448,7 +448,7 @@ def accept_membership(memb_id, request_user=None, reason=None):
accept_membership_checks(membership, request_user)
user = membership.person
membership.perform_action("accept", actor=request_user, reason=reason)
quotas.qh_sync_membership(membership)"User %s has been accepted in %s." %
(user.log_display, project))
......@@ -519,7 +519,7 @@ def remove_membership(memb_id, request_user=None, reason=None):
remove_membership_checks(membership, request_user)
user = membership.person
membership.perform_action("remove", actor=request_user, reason=reason)
quotas.qh_sync_membership(membership)"User %s has been removed from %s." %
(user.log_display, project))
......@@ -552,7 +552,7 @@ def enroll_member(project_id, user, request_user=None, reason=None):
membership = new_membership(project, user, actor=request_user,
quotas.qh_sync_membership(membership)"User %s has been enrolled in %s." %
(membership.person.log_display, project))
......@@ -595,7 +595,7 @@ def leave_project(memb_id, request_user, reason=None):
leave_policy = project.member_leave_policy
if leave_policy == AUTO_ACCEPT_POLICY:
membership.perform_action("remove", actor=request_user, reason=reason)
quotas.qh_sync_membership(membership)"User %s has left %s." %
(request_user.log_display, project))
auto_accepted = True
......@@ -661,7 +661,7 @@ def join_project(project_id, request_user, reason=None):
if (join_policy == AUTO_ACCEPT_POLICY and (
not project.violates_members_limit(adding=1))):
membership.perform_action("accept", actor=request_user, reason=reason)
quotas.qh_sync_membership(membership)"User %s joined %s." %
(request_user.log_display, project))
......@@ -945,15 +945,7 @@ def approve_application(application_id, project_id=None, request_user=None,
# Pre-lock members and owner together in order to impose an ordering
# on locking users
members = quotas.members_to_sync(project)
uids_to_sync = [ for member in members]
applicant = application.applicant
qh_release_pending_app(applicant, locked=True)
application.approve(actor=request_user, reason=reason)
if project.state == Project.UNINITIALIZED:
......@@ -962,7 +954,7 @@ def approve_application(application_id, project_id=None, request_user=None,
_apply_modifications(project, application)
project.activate(actor=request_user, reason=reason)
quotas.qh_sync_project(project)"%s has been approved." % (application.log_display))
project_notif.application_notify(application, "approve")
return project
......@@ -1122,10 +1114,12 @@ def get_pending_app_diff(project):
return diff
def qh_add_pending_app(user, project=None, force=False):
user = AstakosUser.objects.select_for_update().get(
def qh_add_pending_app(user, project=None, force=False, assign_project=None):
if assign_project is None:
assign_project = user.base_project
diff = get_pending_app_diff(project)
return quotas.register_pending_apps(user, diff, force)
return quotas.register_pending_apps(user, assign_project,
diff, force=force)
def check_pending_app_quota(user, project=None):
......@@ -1138,7 +1132,7 @@ def check_pending_app_quota(user, project=None):
return True, None
def qh_release_pending_app(user, locked=False):
if not locked:
user = AstakosUser.objects.select_for_update().get(
quotas.register_pending_apps(user, -1)
def qh_release_pending_app(user, assign_project=None):
if assign_project is None:
assign_project = user.base_project
quotas.register_pending_apps(user, assign_project, -1)
......@@ -197,48 +197,34 @@ def show_resource_value(number, resource, style):
return, unit, style)
def collect_holder_quotas(holder_quotas, h_initial, style=None):
def collect_holder_quotas(holder_quotas, style=None):
print_data = []
for source, source_quotas in holder_quotas.iteritems():
s_initial = h_initial[source]
except KeyError:
for resource, values in source_quotas.iteritems():
initial = s_initial[resource]
except KeyError:
initial = show_resource_value(initial, resource, style)
limit = show_resource_value(values['limit'], resource, style)
usage = show_resource_value(values['usage'], resource, style)
fields = (source, resource, initial, limit, usage)
fields = (source, resource, limit, usage)
return print_data
def show_user_quotas(holder_quotas, h_initial, style=None):
labels = ('source', 'resource', 'base_quota', 'total_quota', 'usage')
print_data = collect_holder_quotas(holder_quotas, h_initial, style=style)
def show_user_quotas(holder_quotas, style=None):
labels = ('source', 'resource', 'limit', 'usage')
print_data = collect_holder_quotas(holder_quotas, style=style)
return print_data, labels
def show_quotas(qh_quotas, astakos_initial, info=None, style=None):
labels = ('user', 'source', 'resource', 'base_quota', 'total_quota',
def show_quotas(qh_quotas, info=None, style=None):
labels = ('holder', 'source', 'resource', 'limit', 'usage')
if info is not None:
labels = ('displayname',) + labels
print_data = []
for holder, holder_quotas in qh_quotas.iteritems():
h_initial = astakos_initial.get(holder)
if h_initial is None:
if info is not None:
email = info.get(holder, "")
h_data = collect_holder_quotas(holder_quotas, h_initial, style=style)
h_data = collect_holder_quotas(holder_quotas, style=style)
if info is not None:
h_data = [(email, holder) + fields for fields in h_data]
......@@ -38,6 +38,7 @@ from synnefo.lib.ordereddict import OrderedDict
from import SynnefoCommand
from import utils
from import ProjectApplication, Project
from import quotas
from ._common import show_resource_value, style_options, check_style
from synnefo.util import units
......@@ -60,6 +61,11 @@ class Command(SynnefoCommand):
help=("Show a list of project memberships")
help="List project quota"),
help=("Specify display unit for resource values "
......@@ -81,7 +87,7 @@ class Command(SynnefoCommand):
id_ = args[0]
if True:
project = get_chain_state(id_)
self.print_project(project, show_quota)
if show_members and project is not None:
fields, labels = members_fields(project)
......@@ -105,13 +111,15 @@ class Command(SynnefoCommand):
def print_project(self, project):
def print_project(self, project, show_quota=False):
quota = (quotas.get_project_quota(project)
if show_quota else None)
self.print_resources(project, quota=quota)
def print_resources(self, project):
def print_resources(self, project, quota=None):
policies = project.projectresourcequota_set.all()
fields, labels = resource_fields(policies, self.unit_style)
fields, labels = resource_fields(policies, quota, self.unit_style)
if fields:
self.pprint_table(fields, labels, title="Resource limits")
......@@ -131,15 +139,23 @@ def get_chain_state(project_id):
raise CommandError("Project with id %s not found." % project_id)
def resource_fields(policies, style):
labels = ('name', 'description', 'max per member')
def resource_fields(policies, quota, style):
labels = ('name', 'max per member', 'max per project')
if quota:
labels += ('usage',)
collect = []
for policy in policies:
name =
desc = policy.resource.desc
capacity = policy.member_capacity
collect.append((name, desc,
show_resource_value(capacity, name, style)))
p_capacity = policy.project_capacity
row = (name,
show_resource_value(capacity, name, style),
show_resource_value(p_capacity, name, style))
if quota:
r_quota = quota.get(name)
usage = r_quota.get('project_usage')
row += (show_resource_value(usage, name, style),)
return collect, labels
......@@ -58,10 +58,6 @@ class Command(SynnefoCommand):
help="Show quota that is over limit"),
help=("Filter quota different from the default or "
"equal to it")),
help="Filter by field; "
"e.g. \"user=uuid,usage>=10M,base_quota<inf\""),
......@@ -71,17 +67,13 @@ class Command(SynnefoCommand):
"total_quota": ("limit", filtering.parse_with_unit),
"limit": ("limit", filtering.parse_with_unit),
"usage": ("usage_max", filtering.parse_with_unit),
"user": ("holder", lambda x: x),
"resource": ("resource", lambda x: x),
"source": ("source", lambda x: x),
"base_quota": ("capacity", filtering.parse_with_unit),
def handle(self, *args, **options):
output_format = options["output_format"]
......@@ -95,30 +87,19 @@ class Command(SynnefoCommand):
filters = []
QHQ, INITQ = Q(), Q()
QHQ = Q()
for flt in filters:
q = filtering.make_query(flt, self.QHFLT)
if q is not None:
QHQ &= q
q = filtering.make_query(flt, self.INITFLT)
if q is not None:
INITQ &= q
overlimit = bool(options["overlimit"])
if overlimit:
QHQ &= Q(usage_max__gt=F("limit"))
with_custom = options["with_custom"]
if with_custom is not None:
qeq = Q(capacity=F("resource__uplimit"))
INITQ &= ~qeq if utils.parse_bool(with_custom) else qeq
except ValueError as e:
raise CommandError(e)
users = AstakosUser.objects.accepted()
qh_quotas, astakos_i = list_user_quotas(
users, qhflt=QHQ, initflt=INITQ)
qh_quotas = list_user_quotas(
users, qhflt=QHQ)
if displayname:
info = {}
......@@ -128,5 +109,5 @@ class Command(SynnefoCommand):
info = None
print_data, labels = common.show_quotas(
qh_quotas, astakos_i, info, style=unit_style)
qh_quotas, info, style=unit_style)
utils.pprint_table(self.stdout, print_data, labels, output_format)
......@@ -36,7 +36,7 @@ from optparse import make_option
from django.db.models import Q
from import AstakosUser, get_latest_terms, Project
from import list_user_quotas
from import get_user_quotas
from synnefo.lib.ordereddict import OrderedDict
from import SynnefoCommand
......@@ -127,12 +127,10 @@ class Command(SynnefoCommand):
unit_style = options["unit_style"]
quotas, initial = list_user_quotas([user])
h_quotas = quotas[user.uuid]
h_initial = initial[user.uuid]
quotas = get_user_quotas(user)
if quotas:
print_data, labels = show_user_quotas(h_quotas, h_initial,
print_data, labels = show_user_quotas(quotas,
utils.pprint_table(self.stdout, print_data, labels,
......@@ -1850,8 +1850,10 @@ class ProjectMembershipManager(models.Manager):
q = self.model.Q_ACCEPTED_STATES
return self.filter(q)
def actually_accepted(self):
def actually_accepted(self, projects=None):
q = self.model.Q_ACTUALLY_ACCEPTED
if projects is not None:
q &= Q(project__in=projects)
return self.filter(q)
def initialized(self, projects=None):
......@@ -31,64 +31,99 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from synnefo.util import units
from import (
Resource, AstakosUserQuota, AstakosUser, Service,
Project, ProjectMembership, ProjectResourceQuota, ProjectApplication)
Project, ProjectMembership, ProjectResourceQuota)
import astakos.quotaholder_app.callpoint as qh
from astakos.quotaholder_app.exception import NoCapacityError
from django.db.models import Q
from synnefo.util.keypath import set_path
def from_holding(holding):
limit, usage_min, usage_max = holding
body = {'limit': limit,
'usage': usage_max,
'pending': usage_max-usage_min,
return body
PROJECT_TAG = "project:"
USER_TAG = "user:"
def project_ref(value):
return PROJECT_TAG + value
def limits_only(holding):
limit, usage_min, usage_max = holding
return limit
def get_project_ref(project):
return project_ref(project.uuid)
def transform_data(holdings, func=None):
if func is None:
func = from_holding
quota = {}
for (holder, source, resource), value in holdings.iteritems():
holder_quota = quota.get(holder, {})
source_quota = holder_quota.get(source, {})
body = func(value)
source_quota[resource] = body
holder_quota[source] = source_quota
quota[holder] = holder_quota
return quota
def user_ref(value):
return USER_TAG + value
def get_user_ref(user):
return user_ref(user.uuid)
def get_counters(users, resources=None, sources=None, flt=None):
uuids = [user.uuid for user in users]
def from_holding(holding, is_project=False):
limit, usage_min, usage_max = holding
prefix = 'project_' if is_project else ''
body = {prefix+'limit': limit,
prefix+'usage': usage_max,
prefix+'pending': usage_max-usage_min,
return body
counters = qh.get_quota(holders=uuids,
def get_user_counters(users, resources=None, sources=None, flt=None):
holders = [get_user_ref(user) for user in users]
return qh.get_quota(holders=holders,
return counters
def get_users_quotas(users, resources=None, sources=None, flt=None):
counters = get_counters(users, resources, sources, flt=flt)
quotas = transform_data(counters)
return quotas
def get_project_counters(projects, resources=None, sources=None):
holders = [get_project_ref(project) for project in projects]
return qh.get_quota(holders=holders,
def strip_names(counters):
stripped = {}
for ((holder, source, resource), value) in counters.iteritems():
prefix, sep, holder = holder.partition(":")
assert prefix in ["user", "project"]
if source is not None:
prefix, sep, source = source.partition(":")
assert prefix == "project"
stripped[(holder, source, resource)] = value
return stripped
def get_related_sources(counters):
projects = set()
for (holder, source, resource) in counters.iterkeys():
return list(projects)
def mk_quota_dict(users_counters, project_counters):
quota = {}
for (holder, source, resource), u_value in users_counters.iteritems():
p_value = project_counters[(source, None, resource)]
values_dict = from_holding(u_value)
values_dict.update(from_holding(p_value, is_project=True))
set_path(quota, [holder, source, resource], values_dict,
return quota
def get_users_quotas_counters(users, resources=None, sources=None, flt=None):
user_counters = get_user_counters(users, resources, sources, flt=flt)
projects = get_related_sources(user_counters)
project_counters = qh.get_quota(holders=projects, resources=resources)
return strip_names(user_counters), strip_names(project_counters)
def get_users_quota_limits(users, resources=None, sources=None):
counters = get_counters(users, resources, sources)
limits = transform_data(counters, limits_only)
return limits
def get_users_quotas(users, resources=None, sources=None, flt=None):
u_c, p_c = get_users_quotas_counters(users, resources, sources, flt=flt)
return mk_quota_dict(u_c, p_c)
def get_user_quotas(user, resources=None, sources=None):
......@@ -102,8 +137,43 @@ def service_get_quotas(component, users=None):
service_names = [t for (t,) in name_values]
resources = Resource.objects.filter(service_origin__in=service_names)
resource_names = [ for r in resources]
counters = qh.get_quota(holders=users, resources=resource_names)
return transform_data(counters)
astakosusers = AstakosUser.objects.verified()
if users is not None:
astakosusers = astakosusers.filter(uuid__in=users)
return get_users_quotas(astakosusers, resources=resource_names)
def mk_limits_dict(counters):
quota = {}
for key, (limit, _, _) in counters.iteritems():
path = list(key)
set_path(quota, path, limit, createpath=True)
return quota
def mk_project_quota_dict(project_counters):
quota = {}
for (holder, _, resource), p_value in project_counters.iteritems():
values_dict = from_holding(p_value, is_project=True)
set_path(quota, [holder, resource], values_dict,
return quota
def get_projects_quota(projects, resources=None, sources=None):
project_counters = get_project_counters(projects, resources, sources)
return mk_project_quota_dict(strip_names(project_counters))
def get_project_quota(project, resources=None, sources=None):
quotas = get_projects_quota([project], resources, sources)
return quotas.get(project.uuid, {})
def get_projects_quota_limits():
project_counters = qh.get_quota(flt=Q(holder__startswith=PROJECT_TAG))
user_counters = qh.get_quota(flt=Q(holder__startswith=USER_TAG))
return mk_limits_dict(project_counters), mk_limits_dict(user_counters)
def _level_quota_dict(quotas):
......@@ -116,21 +186,37 @@ def _level_quota_dict(quotas):
return lst
def _set_user_quota(quotas, resource=None):
def set_quota(quotas, resource=None):
q = _level_quota_dict(quotas)
qh.set_quota(q, resource=resource)
SYSTEM = 'system'
PENDING_APP_RESOURCE = 'astakos.pending_app'
def register_pending_apps(user, quantity, force=False):
provision = (user.uuid, SYSTEM, PENDING_APP_RESOURCE), quantity
def mk_user_provision(user, source, resource, quantity):
holder = user_ref(user)
source = project_ref(source)
return (holder, source, resource), quantity
def mk_project_provision(project, resource, quantity):
holder = project_ref(project)
return (holder, None, resource), quantity
def _mk_provisions(holder, source, resource, quantity):
return [((holder, source, resource), quantity),
((source, None, resource), quantity)]
def register_pending_apps(user, project, quantity, force=False):
provisions = _mk_provisions(get_user_ref(user), get_project_ref(project),
s = qh.issue_commission(clientkey='astakos',
except NoCapacityError as e:
limit =['limit']
return False, limit
......@@ -140,7 +226,8 @@ def register_pending_apps(user, quantity, force=False):
def get_pending_app_quota(user):
quota = get_user_quotas(user)
source = user.base_project.uuid
return quota[source][PENDING_APP_RESOURCE]
def update_base_quota(users, resource, value):
......@@ -161,154 +248,101 @@ def _partition_by(f, l):
return d
def initial_quotas(users, flt=None):
if flt is None:
flt = Q()
userids = [ for user in users]
objs = AstakosUserQuota.objects.select_related('resource')
orig_quotas = objs.filter(user__pk__in=userids).filter(flt)
orig_quotas = _partition_by(lambda q: q.user_id, orig_quotas)
initial = {}
for user in users:
qs = {}
for q in orig_quotas.get(, []):
qs[] = q.capacity
initial[user.uuid] = {SYSTEM: qs}