astakos: Adapt resource reconciliation

Assume base project when reconciling astakos resources.

snf-astakos-app/astakos/im/functions.py

@@ -42,6 +42,7 @@ from django.contrib.auth import login as auth_login, logout as auth_logout
 from django.db.models import Q
 
 from synnefo_branding.utils import render_to_string
+from synnefo.util.keypath import set_path
 
 from synnefo.lib import join_urls
 from astakos.im.models import AstakosUser, Invitation, ProjectMembership, \
@@ -1178,7 +1179,9 @@ def count_pending_app(users):
     usage = {}
     for user in users:
         uuid = user.uuid
-        usage[uuid] = len(apps_d.get(uuid, []))
+        set_path(usage,
+                 [uuid, user.base_project.uuid, quotas.PENDING_APP_RESOURCE],
+                 len(apps_d.get(uuid, [])), createpath=True)
     return usage
 
 

snf-astakos-app/astakos/im/management/commands/reconcile-resources-astakos.py

@@ -32,13 +32,15 @@
 # or implied, of GRNET S.A.
 
 from optparse import make_option
+from datetime import datetime
 
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
+from snf_django.utils import reconcile
 from snf_django.management.utils import pprint_table
 from astakos.im.models import Component, AstakosUser
-from astakos.im.quotas import service_get_quotas, SYSTEM
+from astakos.im import quotas
 from astakos.im.functions import count_pending_app
 import astakos.quotaholder_app.callpoint as qh
 import astakos.quotaholder_app.exception as qh_exception
@@ -56,6 +58,8 @@ class Command(BaseCommand):
         make_option("--userid", dest="userid",
                     default=None,
                     help="Reconcile resources only for this user"),
+        make_option("--project",
+                    help="Reconcile resources only for this project"),
         make_option("--fix", dest="fix",
                     default=False,
                     action="store_true",
@@ -72,6 +76,9 @@ class Command(BaseCommand):
         write = self.stderr.write
         force = options['force']
         userid = options['userid']
+        project = options['project']
+
+        RESOURCES = [quotas.PENDING_APP_RESOURCE]
 
         try:
             astakos = Component.objects.get(name="astakos")
@@ -79,10 +86,13 @@ class Command(BaseCommand):
             raise CommandError("Component 'astakos' not found.")
 
         query = [userid] if userid is not None else None
-        qh_holdings = service_get_quotas(astakos, query)
+        qh_holdings = quotas.service_get_quotas(astakos, query)
+        query = [project] if project is not None else None
+        qh_project_holdings = quotas.service_get_project_quotas(astakos, query)
 
         if userid is None:
-            users = AstakosUser.objects.accepted()
+            users = AstakosUser.objects.accepted().select_related(
+                'base_project')
         else:
             try:
                 user = AstakosUser.objects.get(uuid=userid)
@@ -94,48 +104,35 @@ class Command(BaseCommand):
 
         db_holdings = count_pending_app(users)
 
-        pending_exists = False
-        unknown_user_exists = False
-        unsynced = []
-        for user in users:
-            uuid = user.uuid
-            db_value = db_holdings.get(uuid, 0)
-            try:
-                qh_all = qh_holdings[uuid]
-            except KeyError:
-                write("User '%s' does not exist in Quotaholder!\n" % uuid)
-                unknown_user_exists = True
-                continue
-
-            # Assuming only one source
-            system_qh = qh_all.get(SYSTEM, {})
-            # Assuming only one resource
-            resource = 'astakos.pending_app'
-            try:
-                qh_values = system_qh[resource]
-                qh_value = qh_values['usage']
-                qh_pending = qh_values['pending']
-            except KeyError:
-                write("Resource '%s' does not exist in Quotaholder"
-                      " for user '%s'!\n" % (resource, uuid))
-                continue
-            if qh_pending:
-                write("Pending commission. User '%s', resource '%s'.\n" %
-                      (uuid, resource))
-                pending_exists = True
-                continue
-            if db_value != qh_value:
-                data = (uuid, resource, db_value, qh_value)
-                unsynced.append(data)
-
-        headers = ("User", "Resource", "Astakos", "Quotaholder")
+        db_project_holdings = {}
+        for user, user_holdings in db_holdings.iteritems():
+            db_project_holdings.update(user_holdings)
+
+        unsynced_users, users_pending, users_unknown =\
+            reconcile.check_users(self.stderr, RESOURCES,
+                                  db_holdings, qh_holdings)
+
+        unsynced_projects, projects_pending, projects_unknown =\
+            reconcile.check_projects(self.stderr, RESOURCES,
+                                     db_project_holdings, qh_project_holdings)
+        pending_exists = users_pending or projects_pending
+        unknown_exists = users_unknown or projects_unknown
+
+        headers = ("Type", "Holder", "Source", "Resource",
+                   "Astakos", "Quotaholder")
+        unsynced = unsynced_users + unsynced_projects
         if unsynced:
             pprint_table(self.stdout, unsynced, headers)
             if options["fix"]:
-                provisions = map(create_provision, unsynced)
+                user_provisions = create_user_provisions(unsynced_users)
+                project_provisions = create_project_provisions(
+                    unsynced_projects)
+                provisions = user_provisions + project_provisions
+                name = ("client: reconcile-resources-astakos, time: %s"
+                        % datetime.now())
                 try:
                     s = qh.issue_commission('astakos', provisions,
-                                            name='RECONCILE', force=force)
+                                            name=name, force=force)
                 except qh_exception.NoCapacityError:
                     write("Reconciling failed because a limit has been "
                           "reached. Use --force to ignore the check.\n")
@@ -147,10 +144,23 @@ class Command(BaseCommand):
         if pending_exists:
             write("Found pending commissions. "
                   "This is probably a bug. Please report.\n")
-        elif not (unsynced or unknown_user_exists):
+        elif not (unsynced or unknown_exists):
             write("Everything in sync.\n")
 
 
-def create_provision(provision_info):
-    user, resource, db_value, qh_value = provision_info
-    return (user, SYSTEM, resource), (db_value - qh_value)
+def create_user_provisions(provision_list):
+    provisions = []
+    for _, holder, source, resource, db_value, qh_value in provision_list:
+        value = db_value - qh_value
+        provisions.append(
+            quotas.mk_user_provision(holder, source, resource, value))
+    return provisions
+
+
+def create_project_provisions(provision_list):
+    provisions = []
+    for _, holder, _, resource, db_value, qh_value in provision_list:
+        value = db_value - qh_value
+        provisions.append(
+            quotas.mk_project_provision(holder, resource, value))
+    return provisions