From 9d5f4be2d378bbba876a6276a05124db80b98fec Mon Sep 17 00:00:00 2001
From: Giorgos Korfiatis <gkorf@grnet.gr>
Date: Fri, 11 Oct 2013 11:58:27 +0300
Subject: [PATCH] astakos: Adapt resource reconciliation
Assume base project when reconciling astakos resources.
---
snf-astakos-app/astakos/im/functions.py | 5 +-
.../commands/reconcile-resources-astakos.py | 98 ++++++++++---------
2 files changed, 58 insertions(+), 45 deletions(-)
diff --git a/snf-astakos-app/astakos/im/functions.py b/snf-astakos-app/astakos/im/functions.py
index 48d59ce4d..513c94604 100644
--- a/snf-astakos-app/astakos/im/functions.py
+++ b/snf-astakos-app/astakos/im/functions.py
@@ -42,6 +42,7 @@ from django.contrib.auth import login as auth_login, logout as auth_logout
from django.db.models import Q
from synnefo_branding.utils import render_to_string
+from synnefo.util.keypath import set_path
from synnefo.lib import join_urls
from astakos.im.models import AstakosUser, Invitation, ProjectMembership, \
@@ -1178,7 +1179,9 @@ def count_pending_app(users):
usage = {}
for user in users:
uuid = user.uuid
- usage[uuid] = len(apps_d.get(uuid, []))
+ set_path(usage,
+ [uuid, user.base_project.uuid, quotas.PENDING_APP_RESOURCE],
+ len(apps_d.get(uuid, [])), createpath=True)
return usage
diff --git a/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py b/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py
index 52db8359b..d3c24b5db 100644
--- a/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py
+++ b/snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py
@@ -32,13 +32,15 @@
# or implied, of GRNET S.A.
from optparse import make_option
+from datetime import datetime
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
+from snf_django.utils import reconcile
from snf_django.management.utils import pprint_table
from astakos.im.models import Component, AstakosUser
-from astakos.im.quotas import service_get_quotas, SYSTEM
+from astakos.im import quotas
from astakos.im.functions import count_pending_app
import astakos.quotaholder_app.callpoint as qh
import astakos.quotaholder_app.exception as qh_exception
@@ -56,6 +58,8 @@ class Command(BaseCommand):
make_option("--userid", dest="userid",
default=None,
help="Reconcile resources only for this user"),
+ make_option("--project",
+ help="Reconcile resources only for this project"),
make_option("--fix", dest="fix",
default=False,
action="store_true",
@@ -72,6 +76,9 @@ class Command(BaseCommand):
write = self.stderr.write
force = options['force']
userid = options['userid']
+ project = options['project']
+
+ RESOURCES = [quotas.PENDING_APP_RESOURCE]
try:
astakos = Component.objects.get(name="astakos")
@@ -79,10 +86,13 @@ class Command(BaseCommand):
raise CommandError("Component 'astakos' not found.")
query = [userid] if userid is not None else None
- qh_holdings = service_get_quotas(astakos, query)
+ qh_holdings = quotas.service_get_quotas(astakos, query)
+ query = [project] if project is not None else None
+ qh_project_holdings = quotas.service_get_project_quotas(astakos, query)
if userid is None:
- users = AstakosUser.objects.accepted()
+ users = AstakosUser.objects.accepted().select_related(
+ 'base_project')
else:
try:
user = AstakosUser.objects.get(uuid=userid)
@@ -94,48 +104,35 @@ class Command(BaseCommand):
db_holdings = count_pending_app(users)
- pending_exists = False
- unknown_user_exists = False
- unsynced = []
- for user in users:
- uuid = user.uuid
- db_value = db_holdings.get(uuid, 0)
- try:
- qh_all = qh_holdings[uuid]
- except KeyError:
- write("User '%s' does not exist in Quotaholder!\n" % uuid)
- unknown_user_exists = True
- continue
-
- # Assuming only one source
- system_qh = qh_all.get(SYSTEM, {})
- # Assuming only one resource
- resource = 'astakos.pending_app'
- try:
- qh_values = system_qh[resource]
- qh_value = qh_values['usage']
- qh_pending = qh_values['pending']
- except KeyError:
- write("Resource '%s' does not exist in Quotaholder"
- " for user '%s'!\n" % (resource, uuid))
- continue
- if qh_pending:
- write("Pending commission. User '%s', resource '%s'.\n" %
- (uuid, resource))
- pending_exists = True
- continue
- if db_value != qh_value:
- data = (uuid, resource, db_value, qh_value)
- unsynced.append(data)
-
- headers = ("User", "Resource", "Astakos", "Quotaholder")
+ db_project_holdings = {}
+ for user, user_holdings in db_holdings.iteritems():
+ db_project_holdings.update(user_holdings)
+
+ unsynced_users, users_pending, users_unknown =\
+ reconcile.check_users(self.stderr, RESOURCES,
+ db_holdings, qh_holdings)
+
+ unsynced_projects, projects_pending, projects_unknown =\
+ reconcile.check_projects(self.stderr, RESOURCES,
+ db_project_holdings, qh_project_holdings)
+ pending_exists = users_pending or projects_pending
+ unknown_exists = users_unknown or projects_unknown
+
+ headers = ("Type", "Holder", "Source", "Resource",
+ "Astakos", "Quotaholder")
+ unsynced = unsynced_users + unsynced_projects
if unsynced:
pprint_table(self.stdout, unsynced, headers)
if options["fix"]:
- provisions = map(create_provision, unsynced)
+ user_provisions = create_user_provisions(unsynced_users)
+ project_provisions = create_project_provisions(
+ unsynced_projects)
+ provisions = user_provisions + project_provisions
+ name = ("client: reconcile-resources-astakos, time: %s"
+ % datetime.now())
try:
s = qh.issue_commission('astakos', provisions,
- name='RECONCILE', force=force)
+ name=name, force=force)
except qh_exception.NoCapacityError:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
@@ -147,10 +144,23 @@ class Command(BaseCommand):
if pending_exists:
write("Found pending commissions. "
"This is probably a bug. Please report.\n")
- elif not (unsynced or unknown_user_exists):
+ elif not (unsynced or unknown_exists):
write("Everything in sync.\n")
-def create_provision(provision_info):
- user, resource, db_value, qh_value = provision_info
- return (user, SYSTEM, resource), (db_value - qh_value)
+def create_user_provisions(provision_list):
+ provisions = []
+ for _, holder, source, resource, db_value, qh_value in provision_list:
+ value = db_value - qh_value
+ provisions.append(
+ quotas.mk_user_provision(holder, source, resource, value))
+ return provisions
+
+
+def create_project_provisions(provision_list):
+ provisions = []
+ for _, holder, _, resource, db_value, qh_value in provision_list:
+ value = db_value - qh_value
+ provisions.append(
+ quotas.mk_project_provision(holder, resource, value))
+ return provisions
--
GitLab