Commit 42304537 authored by Giorgos Korfiatis's avatar Giorgos Korfiatis

astakos: Sync project-based quota

Update quota code to take into account the differentiated sources.
There is no more need to lock the user when updating quota; locking
the project is now adequate.

Update quota listing in management commands; introduce option --quota in
project-show.
parent 4d04cadb
...@@ -448,7 +448,7 @@ def accept_membership(memb_id, request_user=None, reason=None): ...@@ -448,7 +448,7 @@ def accept_membership(memb_id, request_user=None, reason=None):
accept_membership_checks(membership, request_user) accept_membership_checks(membership, request_user)
user = membership.person user = membership.person
membership.perform_action("accept", actor=request_user, reason=reason) membership.perform_action("accept", actor=request_user, reason=reason)
quotas.qh_sync_user(user) quotas.qh_sync_membership(membership)
logger.info("User %s has been accepted in %s." % logger.info("User %s has been accepted in %s." %
(user.log_display, project)) (user.log_display, project))
...@@ -519,7 +519,7 @@ def remove_membership(memb_id, request_user=None, reason=None): ...@@ -519,7 +519,7 @@ def remove_membership(memb_id, request_user=None, reason=None):
remove_membership_checks(membership, request_user) remove_membership_checks(membership, request_user)
user = membership.person user = membership.person
membership.perform_action("remove", actor=request_user, reason=reason) membership.perform_action("remove", actor=request_user, reason=reason)
quotas.qh_sync_user(user) quotas.qh_sync_membership(membership)
logger.info("User %s has been removed from %s." % logger.info("User %s has been removed from %s." %
(user.log_display, project)) (user.log_display, project))
...@@ -552,7 +552,7 @@ def enroll_member(project_id, user, request_user=None, reason=None): ...@@ -552,7 +552,7 @@ def enroll_member(project_id, user, request_user=None, reason=None):
membership = new_membership(project, user, actor=request_user, membership = new_membership(project, user, actor=request_user,
enroll=True) enroll=True)
quotas.qh_sync_user(user) quotas.qh_sync_membership(membership)
logger.info("User %s has been enrolled in %s." % logger.info("User %s has been enrolled in %s." %
(membership.person.log_display, project)) (membership.person.log_display, project))
...@@ -595,7 +595,7 @@ def leave_project(memb_id, request_user, reason=None): ...@@ -595,7 +595,7 @@ def leave_project(memb_id, request_user, reason=None):
leave_policy = project.member_leave_policy leave_policy = project.member_leave_policy
if leave_policy == AUTO_ACCEPT_POLICY: if leave_policy == AUTO_ACCEPT_POLICY:
membership.perform_action("remove", actor=request_user, reason=reason) membership.perform_action("remove", actor=request_user, reason=reason)
quotas.qh_sync_user(request_user) quotas.qh_sync_membership(membership)
logger.info("User %s has left %s." % logger.info("User %s has left %s." %
(request_user.log_display, project)) (request_user.log_display, project))
auto_accepted = True auto_accepted = True
...@@ -661,7 +661,7 @@ def join_project(project_id, request_user, reason=None): ...@@ -661,7 +661,7 @@ def join_project(project_id, request_user, reason=None):
if (join_policy == AUTO_ACCEPT_POLICY and ( if (join_policy == AUTO_ACCEPT_POLICY and (
not project.violates_members_limit(adding=1))): not project.violates_members_limit(adding=1))):
membership.perform_action("accept", actor=request_user, reason=reason) membership.perform_action("accept", actor=request_user, reason=reason)
quotas.qh_sync_user(request_user) quotas.qh_sync_membership(membership)
logger.info("User %s joined %s." % logger.info("User %s joined %s." %
(request_user.log_display, project)) (request_user.log_display, project))
else: else:
...@@ -945,15 +945,7 @@ def approve_application(application_id, project_id=None, request_user=None, ...@@ -945,15 +945,7 @@ def approve_application(application_id, project_id=None, request_user=None,
if application.name: if application.name:
check_conflicting_projects(project, application.name) check_conflicting_projects(project, application.name)
# Pre-lock members and owner together in order to impose an ordering qh_release_pending_app(application.applicant)
# on locking users
members = quotas.members_to_sync(project)
uids_to_sync = [member.id for member in members]
applicant = application.applicant
uids_to_sync.append(applicant.id)
quotas.get_users_for_update(uids_to_sync)
qh_release_pending_app(applicant, locked=True)
application.approve(actor=request_user, reason=reason) application.approve(actor=request_user, reason=reason)
if project.state == Project.UNINITIALIZED: if project.state == Project.UNINITIALIZED:
...@@ -962,7 +954,7 @@ def approve_application(application_id, project_id=None, request_user=None, ...@@ -962,7 +954,7 @@ def approve_application(application_id, project_id=None, request_user=None,
_apply_modifications(project, application) _apply_modifications(project, application)
project.activate(actor=request_user, reason=reason) project.activate(actor=request_user, reason=reason)
quotas.qh_sync_locked_users(members) quotas.qh_sync_project(project)
logger.info("%s has been approved." % (application.log_display)) logger.info("%s has been approved." % (application.log_display))
project_notif.application_notify(application, "approve") project_notif.application_notify(application, "approve")
return project return project
...@@ -1122,10 +1114,12 @@ def get_pending_app_diff(project): ...@@ -1122,10 +1114,12 @@ def get_pending_app_diff(project):
return diff return diff
def qh_add_pending_app(user, project=None, force=False): def qh_add_pending_app(user, project=None, force=False, assign_project=None):
user = AstakosUser.objects.select_for_update().get(id=user.id) if assign_project is None:
assign_project = user.base_project
diff = get_pending_app_diff(project) diff = get_pending_app_diff(project)
return quotas.register_pending_apps(user, diff, force) return quotas.register_pending_apps(user, assign_project,
diff, force=force)
def check_pending_app_quota(user, project=None): def check_pending_app_quota(user, project=None):
...@@ -1138,7 +1132,7 @@ def check_pending_app_quota(user, project=None): ...@@ -1138,7 +1132,7 @@ def check_pending_app_quota(user, project=None):
return True, None return True, None
def qh_release_pending_app(user, locked=False): def qh_release_pending_app(user, assign_project=None):
if not locked: if assign_project is None:
user = AstakosUser.objects.select_for_update().get(id=user.id) assign_project = user.base_project
quotas.register_pending_apps(user, -1) quotas.register_pending_apps(user, assign_project, -1)
...@@ -197,48 +197,34 @@ def show_resource_value(number, resource, style): ...@@ -197,48 +197,34 @@ def show_resource_value(number, resource, style):
return units.show(number, unit, style) return units.show(number, unit, style)
def collect_holder_quotas(holder_quotas, h_initial, style=None): def collect_holder_quotas(holder_quotas, style=None):
print_data = [] print_data = []
for source, source_quotas in holder_quotas.iteritems(): for source, source_quotas in holder_quotas.iteritems():
try:
s_initial = h_initial[source]
except KeyError:
continue
for resource, values in source_quotas.iteritems(): for resource, values in source_quotas.iteritems():
try:
initial = s_initial[resource]
except KeyError:
continue
initial = show_resource_value(initial, resource, style)
limit = show_resource_value(values['limit'], resource, style) limit = show_resource_value(values['limit'], resource, style)
usage = show_resource_value(values['usage'], resource, style) usage = show_resource_value(values['usage'], resource, style)
fields = (source, resource, initial, limit, usage) fields = (source, resource, limit, usage)
print_data.append(fields) print_data.append(fields)
return print_data return print_data
def show_user_quotas(holder_quotas, h_initial, style=None): def show_user_quotas(holder_quotas, style=None):
labels = ('source', 'resource', 'base_quota', 'total_quota', 'usage') labels = ('source', 'resource', 'limit', 'usage')
print_data = collect_holder_quotas(holder_quotas, h_initial, style=style) print_data = collect_holder_quotas(holder_quotas, style=style)
return print_data, labels return print_data, labels
def show_quotas(qh_quotas, astakos_initial, info=None, style=None): def show_quotas(qh_quotas, info=None, style=None):
labels = ('user', 'source', 'resource', 'base_quota', 'total_quota', labels = ('holder', 'source', 'resource', 'limit', 'usage')
'usage')
if info is not None: if info is not None:
labels = ('displayname',) + labels labels = ('displayname',) + labels
print_data = [] print_data = []
for holder, holder_quotas in qh_quotas.iteritems(): for holder, holder_quotas in qh_quotas.iteritems():
h_initial = astakos_initial.get(holder)
if h_initial is None:
continue
if info is not None: if info is not None:
email = info.get(holder, "") email = info.get(holder, "")
h_data = collect_holder_quotas(holder_quotas, h_initial, style=style) h_data = collect_holder_quotas(holder_quotas, style=style)
if info is not None: if info is not None:
h_data = [(email, holder) + fields for fields in h_data] h_data = [(email, holder) + fields for fields in h_data]
else: else:
......
...@@ -38,6 +38,7 @@ from synnefo.lib.ordereddict import OrderedDict ...@@ -38,6 +38,7 @@ from synnefo.lib.ordereddict import OrderedDict
from snf_django.management.commands import SynnefoCommand from snf_django.management.commands import SynnefoCommand
from snf_django.management import utils from snf_django.management import utils
from astakos.im.models import ProjectApplication, Project from astakos.im.models import ProjectApplication, Project
from astakos.im import quotas
from ._common import show_resource_value, style_options, check_style from ._common import show_resource_value, style_options, check_style
from synnefo.util import units from synnefo.util import units
...@@ -60,6 +61,11 @@ class Command(SynnefoCommand): ...@@ -60,6 +61,11 @@ class Command(SynnefoCommand):
default=False, default=False,
help=("Show a list of project memberships") help=("Show a list of project memberships")
), ),
make_option('--quota',
action='store_true',
dest='list_quotas',
default=False,
help="List project quota"),
make_option('--unit-style', make_option('--unit-style',
default='mb', default='mb',
help=("Specify display unit for resource values " help=("Specify display unit for resource values "
...@@ -81,7 +87,7 @@ class Command(SynnefoCommand): ...@@ -81,7 +87,7 @@ class Command(SynnefoCommand):
id_ = args[0] id_ = args[0]
if True: if True:
project = get_chain_state(id_) project = get_chain_state(id_)
self.print_project(project) self.print_project(project, show_quota)
if show_members and project is not None: if show_members and project is not None:
self.stdout.write("\n") self.stdout.write("\n")
fields, labels = members_fields(project) fields, labels = members_fields(project)
...@@ -105,13 +111,15 @@ class Command(SynnefoCommand): ...@@ -105,13 +111,15 @@ class Command(SynnefoCommand):
self.pprint_dict(app_info) self.pprint_dict(app_info)
self.print_app_resources(app) self.print_app_resources(app)
def print_project(self, project): def print_project(self, project, show_quota=False):
self.pprint_dict(project_fields(project)) self.pprint_dict(project_fields(project))
self.print_resources(project) quota = (quotas.get_project_quota(project)
if show_quota else None)
self.print_resources(project, quota=quota)
def print_resources(self, project): def print_resources(self, project, quota=None):
policies = project.projectresourcequota_set.all() policies = project.projectresourcequota_set.all()
fields, labels = resource_fields(policies, self.unit_style) fields, labels = resource_fields(policies, quota, self.unit_style)
if fields: if fields:
self.stdout.write("\n") self.stdout.write("\n")
self.pprint_table(fields, labels, title="Resource limits") self.pprint_table(fields, labels, title="Resource limits")
...@@ -131,15 +139,23 @@ def get_chain_state(project_id): ...@@ -131,15 +139,23 @@ def get_chain_state(project_id):
raise CommandError("Project with id %s not found." % project_id) raise CommandError("Project with id %s not found." % project_id)
def resource_fields(policies, style): def resource_fields(policies, quota, style):
labels = ('name', 'description', 'max per member') labels = ('name', 'max per member', 'max per project')
if quota:
labels += ('usage',)
collect = [] collect = []
for policy in policies: for policy in policies:
name = policy.resource.name name = policy.resource.name
desc = policy.resource.desc
capacity = policy.member_capacity capacity = policy.member_capacity
collect.append((name, desc, p_capacity = policy.project_capacity
show_resource_value(capacity, name, style))) row = (name,
show_resource_value(capacity, name, style),
show_resource_value(p_capacity, name, style))
if quota:
r_quota = quota.get(name)
usage = r_quota.get('project_usage')
row += (show_resource_value(usage, name, style),)
collect.append(row)
return collect, labels return collect, labels
......
...@@ -58,10 +58,6 @@ class Command(SynnefoCommand): ...@@ -58,10 +58,6 @@ class Command(SynnefoCommand):
make_option('--overlimit', make_option('--overlimit',
action='store_true', action='store_true',
help="Show quota that is over limit"), help="Show quota that is over limit"),
make_option('--with-custom',
metavar='True|False',
help=("Filter quota different from the default or "
"equal to it")),
make_option('--filter-by', make_option('--filter-by',
help="Filter by field; " help="Filter by field; "
"e.g. \"user=uuid,usage>=10M,base_quota<inf\""), "e.g. \"user=uuid,usage>=10M,base_quota<inf\""),
...@@ -71,17 +67,13 @@ class Command(SynnefoCommand): ...@@ -71,17 +67,13 @@ class Command(SynnefoCommand):
) )
QHFLT = { QHFLT = {
"total_quota": ("limit", filtering.parse_with_unit), "limit": ("limit", filtering.parse_with_unit),
"usage": ("usage_max", filtering.parse_with_unit), "usage": ("usage_max", filtering.parse_with_unit),
"user": ("holder", lambda x: x), "user": ("holder", lambda x: x),
"resource": ("resource", lambda x: x), "resource": ("resource", lambda x: x),
"source": ("source", lambda x: x), "source": ("source", lambda x: x),
} }
INITFLT = {
"base_quota": ("capacity", filtering.parse_with_unit),
}
@transaction.commit_on_success @transaction.commit_on_success
def handle(self, *args, **options): def handle(self, *args, **options):
output_format = options["output_format"] output_format = options["output_format"]
...@@ -95,30 +87,19 @@ class Command(SynnefoCommand): ...@@ -95,30 +87,19 @@ class Command(SynnefoCommand):
else: else:
filters = [] filters = []
QHQ, INITQ = Q(), Q() QHQ = Q()
for flt in filters: for flt in filters:
q = filtering.make_query(flt, self.QHFLT) q = filtering.make_query(flt, self.QHFLT)
if q is not None: if q is not None:
QHQ &= q QHQ &= q
q = filtering.make_query(flt, self.INITFLT)
if q is not None:
INITQ &= q
overlimit = bool(options["overlimit"]) overlimit = bool(options["overlimit"])
if overlimit: if overlimit:
QHQ &= Q(usage_max__gt=F("limit")) QHQ &= Q(usage_max__gt=F("limit"))
with_custom = options["with_custom"]
if with_custom is not None:
qeq = Q(capacity=F("resource__uplimit"))
try:
INITQ &= ~qeq if utils.parse_bool(with_custom) else qeq
except ValueError as e:
raise CommandError(e)
users = AstakosUser.objects.accepted() users = AstakosUser.objects.accepted()
qh_quotas, astakos_i = list_user_quotas( qh_quotas = list_user_quotas(
users, qhflt=QHQ, initflt=INITQ) users, qhflt=QHQ)
if displayname: if displayname:
info = {} info = {}
...@@ -128,5 +109,5 @@ class Command(SynnefoCommand): ...@@ -128,5 +109,5 @@ class Command(SynnefoCommand):
info = None info = None
print_data, labels = common.show_quotas( print_data, labels = common.show_quotas(
qh_quotas, astakos_i, info, style=unit_style) qh_quotas, info, style=unit_style)
utils.pprint_table(self.stdout, print_data, labels, output_format) utils.pprint_table(self.stdout, print_data, labels, output_format)
...@@ -36,7 +36,7 @@ from optparse import make_option ...@@ -36,7 +36,7 @@ from optparse import make_option
from django.db.models import Q from django.db.models import Q
from astakos.im.models import AstakosUser, get_latest_terms, Project from astakos.im.models import AstakosUser, get_latest_terms, Project
from astakos.im.quotas import list_user_quotas from astakos.im.quotas import get_user_quotas
from synnefo.lib.ordereddict import OrderedDict from synnefo.lib.ordereddict import OrderedDict
from snf_django.management.commands import SynnefoCommand from snf_django.management.commands import SynnefoCommand
...@@ -127,12 +127,10 @@ class Command(SynnefoCommand): ...@@ -127,12 +127,10 @@ class Command(SynnefoCommand):
unit_style = options["unit_style"] unit_style = options["unit_style"]
check_style(unit_style) check_style(unit_style)
quotas, initial = list_user_quotas([user]) quotas = get_user_quotas(user)
h_quotas = quotas[user.uuid]
h_initial = initial[user.uuid]
if quotas: if quotas:
self.stdout.write("\n") self.stdout.write("\n")
print_data, labels = show_user_quotas(h_quotas, h_initial, print_data, labels = show_user_quotas(quotas,
style=unit_style) style=unit_style)
utils.pprint_table(self.stdout, print_data, labels, utils.pprint_table(self.stdout, print_data, labels,
options["output_format"], options["output_format"],
......
...@@ -1850,8 +1850,10 @@ class ProjectMembershipManager(models.Manager): ...@@ -1850,8 +1850,10 @@ class ProjectMembershipManager(models.Manager):
q = self.model.Q_ACCEPTED_STATES q = self.model.Q_ACCEPTED_STATES
return self.filter(q) return self.filter(q)
def actually_accepted(self): def actually_accepted(self, projects=None):
q = self.model.Q_ACTUALLY_ACCEPTED q = self.model.Q_ACTUALLY_ACCEPTED
if projects is not None:
q &= Q(project__in=projects)
return self.filter(q) return self.filter(q)
def initialized(self, projects=None): def initialized(self, projects=None):
......
...@@ -31,64 +31,99 @@ ...@@ -31,64 +31,99 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from synnefo.util import units
from astakos.im.models import ( from astakos.im.models import (
Resource, AstakosUserQuota, AstakosUser, Service, Resource, AstakosUserQuota, AstakosUser, Service,
Project, ProjectMembership, ProjectResourceQuota, ProjectApplication) Project, ProjectMembership, ProjectResourceQuota)
import astakos.quotaholder_app.callpoint as qh import astakos.quotaholder_app.callpoint as qh
from astakos.quotaholder_app.exception import NoCapacityError from astakos.quotaholder_app.exception import NoCapacityError
from django.db.models import Q from django.db.models import Q
from synnefo.util.keypath import set_path
def from_holding(holding): PROJECT_TAG = "project:"
limit, usage_min, usage_max = holding USER_TAG = "user:"
body = {'limit': limit,
'usage': usage_max,
'pending': usage_max-usage_min,
}
return body
def project_ref(value):
return PROJECT_TAG + value
def limits_only(holding):
limit, usage_min, usage_max = holding
return limit
def get_project_ref(project):
return project_ref(project.uuid)
def transform_data(holdings, func=None):
if func is None:
func = from_holding
quota = {} def user_ref(value):
for (holder, source, resource), value in holdings.iteritems(): return USER_TAG + value
holder_quota = quota.get(holder, {})
source_quota = holder_quota.get(source, {})
body = func(value) def get_user_ref(user):
source_quota[resource] = body return user_ref(user.uuid)
holder_quota[source] = source_quota
quota[holder] = holder_quota
return quota
def get_counters(users, resources=None, sources=None, flt=None): def from_holding(holding, is_project=False):
uuids = [user.uuid for user in users] limit, usage_min, usage_max = holding
prefix = 'project_' if is_project else ''
body = {prefix+'limit': limit,
prefix+'usage': usage_max,
prefix+'pending': usage_max-usage_min,
}
return body
counters = qh.get_quota(holders=uuids, def get_user_counters(users, resources=None, sources=None, flt=None):
holders = [get_user_ref(user) for user in users]
return qh.get_quota(holders=holders,
resources=resources, resources=resources,
sources=sources, sources=sources,
flt=flt) flt=flt)
return counters
def get_users_quotas(users, resources=None, sources=None, flt=None): def get_project_counters(projects, resources=None, sources=None):
counters = get_counters(users, resources, sources, flt=flt) holders = [get_project_ref(project) for project in projects]
quotas = transform_data(counters) return qh.get_quota(holders=holders,
return quotas resources=resources,
sources=sources)
def strip_names(counters):
stripped = {}
for ((holder, source, resource), value) in counters.iteritems():
prefix, sep, holder = holder.partition(":")
assert prefix in ["user", "project"]
if source is not None:
prefix, sep, source = source.partition(":")
assert prefix == "project"
stripped[(holder, source, resource)] = value