From 9d5f4be2d378bbba876a6276a05124db80b98fec Mon Sep 17 00:00:00 2001 From: Giorgos Korfiatis Date: Fri, 11 Oct 2013 11:58:27 +0300 Subject: [PATCH] astakos: Adapt resource reconciliation Assume base project when reconciling astakos resources. --- snf-astakos-app/astakos/im/functions.py | 5 +- .../commands/reconcile-resources-astakos.py | 98 ++++++++++--------- 2 files changed, 58 insertions(+), 45 deletions(-) diff --git a/snf-astakos-app/astakos/im/functions.py b/snf-astakos-app/astakos/im/functions.py index 48d59ce4d..513c94604 100644 --- a/snf-astakos-app/astakos/im/functions.py +++ b/snf-astakos-app/astakos/im/functions.py @@ -42,6 +42,7 @@ from django.contrib.auth import login as auth_login, logout as auth_logout from django.db.models import Q from synnefo_branding.utils import render_to_string +from synnefo.util.keypath import set_path from synnefo.lib import join_urls from astakos.im.models import AstakosUser, Invitation, ProjectMembership, \ @@ -1178,7 +1179,9 @@ def count_pending_app(users): usage = {} for user in users: uuid = user.uuid - usage[uuid] = len(apps_d.get(uuid, [])) + set_path(usage, + [uuid, user.base_project.uuid, quotas.PENDING_APP_RESOURCE], + len(apps_d.get(uuid, [])), createpath=True) return usage diff --git a/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py b/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py index 52db8359b..d3c24b5db 100644 --- a/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py +++ b/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py @@ -32,13 +32,15 @@ # or implied, of GRNET S.A. from optparse import make_option +from datetime import datetime from django.core.management.base import BaseCommand, CommandError from django.db import transaction +from snf_django.utils import reconcile from snf_django.management.utils import pprint_table from astakos.im.models import Component, AstakosUser -from astakos.im.quotas import service_get_quotas, SYSTEM +from astakos.im import quotas from astakos.im.functions import count_pending_app import astakos.quotaholder_app.callpoint as qh import astakos.quotaholder_app.exception as qh_exception @@ -56,6 +58,8 @@ class Command(BaseCommand): make_option("--userid", dest="userid", default=None, help="Reconcile resources only for this user"), + make_option("--project", + help="Reconcile resources only for this project"), make_option("--fix", dest="fix", default=False, action="store_true", @@ -72,6 +76,9 @@ class Command(BaseCommand): write = self.stderr.write force = options['force'] userid = options['userid'] + project = options['project'] + + RESOURCES = [quotas.PENDING_APP_RESOURCE] try: astakos = Component.objects.get(name="astakos") @@ -79,10 +86,13 @@ class Command(BaseCommand): raise CommandError("Component 'astakos' not found.") query = [userid] if userid is not None else None - qh_holdings = service_get_quotas(astakos, query) + qh_holdings = quotas.service_get_quotas(astakos, query) + query = [project] if project is not None else None + qh_project_holdings = quotas.service_get_project_quotas(astakos, query) if userid is None: - users = AstakosUser.objects.accepted() + users = AstakosUser.objects.accepted().select_related( + 'base_project') else: try: user = AstakosUser.objects.get(uuid=userid) @@ -94,48 +104,35 @@ class Command(BaseCommand): db_holdings = count_pending_app(users) - pending_exists = False - unknown_user_exists = False - unsynced = [] - for user in users: - uuid = user.uuid - db_value = db_holdings.get(uuid, 0) - try: - qh_all = qh_holdings[uuid] - except KeyError: - write("User '%s' does not exist in Quotaholder!\n" % uuid) - unknown_user_exists = True - continue - - # Assuming only one source - system_qh = qh_all.get(SYSTEM, {}) - # Assuming only one resource - resource = 'astakos.pending_app' - try: - qh_values = system_qh[resource] - qh_value = qh_values['usage'] - qh_pending = qh_values['pending'] - except KeyError: - write("Resource '%s' does not exist in Quotaholder" - " for user '%s'!\n" % (resource, uuid)) - continue - if qh_pending: - write("Pending commission. User '%s', resource '%s'.\n" % - (uuid, resource)) - pending_exists = True - continue - if db_value != qh_value: - data = (uuid, resource, db_value, qh_value) - unsynced.append(data) - - headers = ("User", "Resource", "Astakos", "Quotaholder") + db_project_holdings = {} + for user, user_holdings in db_holdings.iteritems(): + db_project_holdings.update(user_holdings) + + unsynced_users, users_pending, users_unknown =\ + reconcile.check_users(self.stderr, RESOURCES, + db_holdings, qh_holdings) + + unsynced_projects, projects_pending, projects_unknown =\ + reconcile.check_projects(self.stderr, RESOURCES, + db_project_holdings, qh_project_holdings) + pending_exists = users_pending or projects_pending + unknown_exists = users_unknown or projects_unknown + + headers = ("Type", "Holder", "Source", "Resource", + "Astakos", "Quotaholder") + unsynced = unsynced_users + unsynced_projects if unsynced: pprint_table(self.stdout, unsynced, headers) if options["fix"]: - provisions = map(create_provision, unsynced) + user_provisions = create_user_provisions(unsynced_users) + project_provisions = create_project_provisions( + unsynced_projects) + provisions = user_provisions + project_provisions + name = ("client: reconcile-resources-astakos, time: %s" + % datetime.now()) try: s = qh.issue_commission('astakos', provisions, - name='RECONCILE', force=force) + name=name, force=force) except qh_exception.NoCapacityError: write("Reconciling failed because a limit has been " "reached. Use --force to ignore the check.\n") @@ -147,10 +144,23 @@ class Command(BaseCommand): if pending_exists: write("Found pending commissions. " "This is probably a bug. Please report.\n") - elif not (unsynced or unknown_user_exists): + elif not (unsynced or unknown_exists): write("Everything in sync.\n") -def create_provision(provision_info): - user, resource, db_value, qh_value = provision_info - return (user, SYSTEM, resource), (db_value - qh_value) +def create_user_provisions(provision_list): + provisions = [] + for _, holder, source, resource, db_value, qh_value in provision_list: + value = db_value - qh_value + provisions.append( + quotas.mk_user_provision(holder, source, resource, value)) + return provisions + + +def create_project_provisions(provision_list): + provisions = [] + for _, holder, _, resource, db_value, qh_value in provision_list: + value = db_value - qh_value + provisions.append( + quotas.mk_project_provision(holder, resource, value)) + return provisions -- GitLab