cyclades: Adapt resource reconciliation

Take projects into account when computing resource usage and comparing
with the quotaholder values.

snf-cyclades-app/synnefo/quotas/management/commands/reconcile-resources-cyclades.py

@@ -35,11 +35,10 @@ from datetime import datetime
 from django.core.management.base import BaseCommand
 from optparse import make_option
 
-
 from synnefo import quotas
-from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings,
-                                 transform_quotas)
+from synnefo.quotas import util
 from snf_django.management.utils import pprint_table
+from snf_django.utils import reconcile
 
 
 class Command(BaseCommand):
@@ -53,6 +52,8 @@ class Command(BaseCommand):
         make_option("--userid", dest="userid",
                     default=None,
                     help="Reconcile resources only for this user"),
+        make_option("--project",
+                    help="Reconcile resources only for this project"),
         make_option("--fix", dest="fix",
                     default=False,
                     action="store_true",
@@ -67,68 +68,47 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         write = self.stderr.write
         userid = options['userid']
+        project = options["project"]
 
         # Get holdings from Cyclades DB
-        db_holdings = get_db_holdings(userid)
-        # Get holdings from QuotaHolder
-        qh_holdings = get_quotaholder_holdings(userid)
-
-        users = set(db_holdings.keys())
-        users.update(qh_holdings.keys())
-        # Remove 'None' user
-        users.discard(None)
-
-        if userid and userid not in users:
-            write("User '%s' does not exist in Quotaholder!", userid)
-            return
-
-        pending_exists = False
-        unknown_user_exists = False
-        unsynced = []
-        for user in users:
-            db = db_holdings.get(user, {})
-            try:
-                qh_all = qh_holdings[user]
-            except KeyError:
-                write("User '%s' does not exist in Quotaholder!\n" %
-                      user)
-                unknown_user_exists = True
-                continue
+        db_holdings = util.get_db_holdings(userid, project)
+        db_project_holdings = util.get_db_project_holdings(project)
 
-            # Assuming only one source
-            qh = qh_all.get(quotas.DEFAULT_SOURCE, {})
-            qh = transform_quotas(qh)
-            for resource in quotas.RESOURCES:
-                db_value = db.pop(resource, 0)
-                try:
-                    qh_value, _, qh_pending = qh[resource]
-                except KeyError:
-                    write("Resource '%s' does not exist in Quotaholder"
-                          " for user '%s'!\n" % (resource, user))
-                    continue
-                if qh_pending:
-                    write("Pending commission. User '%s', resource '%s'.\n" %
-                          (user, resource))
-                    pending_exists = True
-                    continue
-                if db_value != qh_value:
-                    data = (user, resource, db_value, qh_value)
-                    unsynced.append(data)
-
-        headers = ("User", "Resource", "Database", "Quotaholder")
+        # Get holdings from QuotaHolder
+        qh_holdings = util.get_qh_users_holdings(
+            [userid] if userid is not None else None)
+        qh_project_holdings = util.get_qh_project_holdings(
+            [project] if project is not None else None)
+
+        unsynced_users, users_pending, users_unknown =\
+            reconcile.check_users(self.stderr, quotas.RESOURCES,
+                                  db_holdings, qh_holdings)
+
+        unsynced_projects, projects_pending, projects_unknown =\
+            reconcile.check_projects(self.stderr, quotas.RESOURCES,
+                                     db_project_holdings, qh_project_holdings)
+        pending_exists = users_pending or projects_pending
+        unknown_exists = users_unknown or projects_unknown
+
+        headers = ("Type", "Holder", "Source", "Resource",
+                   "Database", "Quotaholder")
+        unsynced = unsynced_users + unsynced_projects
         if unsynced:
             pprint_table(self.stdout, unsynced, headers)
             if options["fix"]:
                 qh = quotas.Quotaholder.get()
-                request = {}
-                request["force"] = options["force"]
-                request["auto_accept"] = True
-                request["name"] = \
-                    ("client: reconcile-resources-cyclades, time: %s"
-                     % datetime.now())
-                request["provisions"] = map(create_provision, unsynced)
+                force = options["force"]
+                name = ("client: reconcile-resources-cyclades, time: %s"
+                        % datetime.now())
+                user_provisions = reconcile.create_user_provisions(
+                    unsynced_users)
+                project_provisions = reconcile.create_project_provisions(
+                    unsynced_projects)
                 try:
-                    qh.issue_commission(request)
+                    qh.issue_commission_generic(
+                        user_provisions, project_provisions,
+                        name=name, force=force,
+                        auto_accept=True)
                 except quotas.errors.QuotaLimit:
                     write("Reconciling failed because a limit has been "
                           "reached. Use --force to ignore the check.\n")
@@ -138,13 +118,5 @@ class Command(BaseCommand):
         if pending_exists:
             write("Found pending commissions. Run 'snf-manage"
                   " reconcile-commissions-cyclades'\n")
-        elif not (unsynced or unknown_user_exists):
+        elif not (unsynced or unknown_exists):
             write("Everything in sync.\n")
-
-
-def create_provision(provision_info):
-    user, resource, db_value, qh_value = provision_info
-    return {"holder": user,
-            "source": quotas.DEFAULT_SOURCE,
-            "resource": resource,
-            "quantity": db_value - qh_value}

snf-cyclades-app/synnefo/quotas/util.py

@@ -35,9 +35,14 @@ from django.db.models import Sum, Count, Q
 
 from synnefo.db.models import VirtualMachine, Network, IPAddress
 from synnefo.quotas import Quotaholder
+from synnefo.util.keypath import set_path
 
 
-def get_db_holdings(user=None):
+MiB = 2 ** 20
+GiB = 2 ** 30
+
+
+def get_db_holdings(user=None, project=None):
     """Get holdings from Cyclades DB."""
     holdings = {}
 
@@ -50,46 +55,123 @@ def get_db_holdings(user=None):
         networks = networks.filter(userid=user)
         floating_ips = floating_ips.filter(userid=user)
 
+    if project is not None:
+        vms = vms.filter(project=project)
+        networks = networks.filter(project=project)
+        floating_ips = floating_ips.filter(project=project)
+
     # Get resources related with VMs
-    vm_resources = vms.values("userid")\
-                      .annotate(num=Count("id"),
-                                total_ram=Sum("flavor__ram"),
-                                total_cpu=Sum("flavor__cpu"),
-                                disk=Sum("flavor__disk"))
-    vm_active_resources = \
-        vms.values("userid")\
-           .filter(Q(operstate="STARTED") | Q(operstate="BUILD") |
-                   Q(operstate="ERROR"))\
-           .annotate(ram=Sum("flavor__ram"),
-                     cpu=Sum("flavor__cpu"))
+    vm_resources = vms.values("userid", "project")\
+        .annotate(num=Count("id"),
+                  total_ram=Sum("flavor__ram"),
+                  total_cpu=Sum("flavor__cpu"),
+                  disk=Sum("flavor__disk"))
 
     for vm_res in vm_resources.iterator():
         user = vm_res['userid']
+        project = vm_res['project']
         res = {"cyclades.vm": vm_res["num"],
                "cyclades.total_cpu": vm_res["total_cpu"],
-               "cyclades.disk": 1073741824 * vm_res["disk"],
-               "cyclades.total_ram": 1048576 * vm_res["total_ram"]}
-        holdings[user] = res
+               "cyclades.disk": vm_res["disk"] * GiB,
+               "cyclades.total_ram": vm_res["total_ram"] * MiB}
+        set_path(holdings, [user, project], res, createpath=True)
+
+    vm_active_resources = vms.values("userid", "project")\
+        .filter(Q(operstate="STARTED") | Q(operstate="BUILD") |
+                Q(operstate="ERROR"))\
+        .annotate(ram=Sum("flavor__ram"),
+                  cpu=Sum("flavor__cpu"))
 
     for vm_res in vm_active_resources.iterator():
         user = vm_res['userid']
-        holdings[user]["cyclades.cpu"] = vm_res["cpu"]
-        holdings[user]["cyclades.ram"] = 1048576 * vm_res["ram"]
+        project = vm_res['project']
+        set_path(holdings, [user, project, "cyclades.cpu"], vm_res["cpu"],
+                 createpath=True)
+        set_path(holdings, [user, project, "cyclades.ram"],
+                 vm_res["ram"] * MiB, createpath=True)
 
     # Get resources related with networks
-    net_resources = networks.values("userid")\
+    net_resources = networks.values("userid", "project")\
                             .annotate(num=Count("id"))
+
     for net_res in net_resources.iterator():
         user = net_res['userid']
-        holdings.setdefault(user, {})
-        holdings[user]["cyclades.network.private"] = net_res["num"]
+        if user is None:
+            continue
+        project = net_res['project']
+        set_path(holdings, [user, project, "cyclades.network.private"],
+                 net_res["num"], createpath=True)
 
-    floating_ips_resources = floating_ips.values("userid")\
+    floating_ips_resources = floating_ips.values("userid", "project")\
                                          .annotate(num=Count("id"))
+
     for floating_ip_res in floating_ips_resources.iterator():
         user = floating_ip_res["userid"]
-        holdings.setdefault(user, {})
-        holdings[user]["cyclades.floating_ip"] = floating_ip_res["num"]
+        project = floating_ip_res["project"]
+        set_path(holdings, [user, project, "cyclades.floating_ip"],
+                 floating_ip_res["num"], createpath=True)
+
+    return holdings
+
+
+def get_db_project_holdings(project=None):
+    """Get holdings from Cyclades DB."""
+    holdings = {}
+
+    vms = VirtualMachine.objects.filter(deleted=False)
+    networks = Network.objects.filter(deleted=False)
+    floating_ips = IPAddress.objects.filter(deleted=False, floating_ip=True)
+
+    if project is not None:
+        vms = vms.filter(project=project)
+        networks = networks.filter(project=project)
+        floating_ips = floating_ips.filter(project=project)
+
+    # Get resources related with VMs
+    vm_resources = vms.values("project")\
+        .annotate(num=Count("id"),
+                  total_ram=Sum("flavor__ram"),
+                  total_cpu=Sum("flavor__cpu"),
+                  disk=Sum("flavor__disk"))
+
+    for vm_res in vm_resources.iterator():
+        project = vm_res['project']
+        res = {"cyclades.vm": vm_res["num"],
+               "cyclades.total_cpu": vm_res["total_cpu"],
+               "cyclades.disk": vm_res["disk"] * GiB,
+               "cyclades.total_ram": vm_res["total_ram"] * MiB}
+        set_path(holdings, [project], res, createpath=True)
+
+    vm_active_resources = vms.values("project")\
+        .filter(Q(operstate="STARTED") | Q(operstate="BUILD") |
+                Q(operstate="ERROR"))\
+        .annotate(ram=Sum("flavor__ram"),
+                  cpu=Sum("flavor__cpu"))
+
+    for vm_res in vm_active_resources.iterator():
+        project = vm_res['project']
+        set_path(holdings, [project, "cyclades.cpu"], vm_res["cpu"],
+                 createpath=True)
+        set_path(holdings, [project, "cyclades.ram"],
+                 vm_res["ram"] * MiB, createpath=True)
+
+    # Get resources related with networks
+    net_resources = networks.values("project").annotate(num=Count("id"))
+
+    for net_res in net_resources.iterator():
+        project = net_res['project']
+        if project is None:
+            continue
+        set_path(holdings, [project, "cyclades.network.private"],
+                 net_res["num"], createpath=True)
+
+    floating_ips_resources = floating_ips.values("project")\
+        .annotate(num=Count("id"))
+
+    for floating_ip_res in floating_ips_resources.iterator():
+        project = floating_ip_res["project"]
+        set_path(holdings, [project, "cyclades.floating_ip"],
+                 floating_ip_res["num"], createpath=True)
 
     return holdings