diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-resources-cyclades.py b/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-resources-cyclades.py index 38892ba3653833dd8687ec29dbb750018562d878..d471c5483c82df7bb16f6ce67b66a94e7fdc9961 100644 --- a/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-resources-cyclades.py +++ b/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-resources-cyclades.py @@ -35,11 +35,10 @@ from datetime import datetime from django.core.management.base import BaseCommand from optparse import make_option - from synnefo import quotas -from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings, - transform_quotas) +from synnefo.quotas import util from snf_django.management.utils import pprint_table +from snf_django.utils import reconcile class Command(BaseCommand): @@ -53,6 +52,8 @@ class Command(BaseCommand): make_option("--userid", dest="userid", default=None, help="Reconcile resources only for this user"), + make_option("--project", + help="Reconcile resources only for this project"), make_option("--fix", dest="fix", default=False, action="store_true", @@ -67,68 +68,47 @@ class Command(BaseCommand): def handle(self, *args, **options): write = self.stderr.write userid = options['userid'] + project = options["project"] # Get holdings from Cyclades DB - db_holdings = get_db_holdings(userid) - # Get holdings from QuotaHolder - qh_holdings = get_quotaholder_holdings(userid) - - users = set(db_holdings.keys()) - users.update(qh_holdings.keys()) - # Remove 'None' user - users.discard(None) - - if userid and userid not in users: - write("User '%s' does not exist in Quotaholder!", userid) - return - - pending_exists = False - unknown_user_exists = False - unsynced = [] - for user in users: - db = db_holdings.get(user, {}) - try: - qh_all = qh_holdings[user] - except KeyError: - write("User '%s' does not exist in Quotaholder!\n" % - user) - unknown_user_exists = True - continue + db_holdings = util.get_db_holdings(userid, project) + db_project_holdings = util.get_db_project_holdings(project) - # Assuming only one source - qh = qh_all.get(quotas.DEFAULT_SOURCE, {}) - qh = transform_quotas(qh) - for resource in quotas.RESOURCES: - db_value = db.pop(resource, 0) - try: - qh_value, _, qh_pending = qh[resource] - except KeyError: - write("Resource '%s' does not exist in Quotaholder" - " for user '%s'!\n" % (resource, user)) - continue - if qh_pending: - write("Pending commission. User '%s', resource '%s'.\n" % - (user, resource)) - pending_exists = True - continue - if db_value != qh_value: - data = (user, resource, db_value, qh_value) - unsynced.append(data) - - headers = ("User", "Resource", "Database", "Quotaholder") + # Get holdings from QuotaHolder + qh_holdings = util.get_qh_users_holdings( + [userid] if userid is not None else None) + qh_project_holdings = util.get_qh_project_holdings( + [project] if project is not None else None) + + unsynced_users, users_pending, users_unknown =\ + reconcile.check_users(self.stderr, quotas.RESOURCES, + db_holdings, qh_holdings) + + unsynced_projects, projects_pending, projects_unknown =\ + reconcile.check_projects(self.stderr, quotas.RESOURCES, + db_project_holdings, qh_project_holdings) + pending_exists = users_pending or projects_pending + unknown_exists = users_unknown or projects_unknown + + headers = ("Type", "Holder", "Source", "Resource", + "Database", "Quotaholder") + unsynced = unsynced_users + unsynced_projects if unsynced: pprint_table(self.stdout, unsynced, headers) if options["fix"]: qh = quotas.Quotaholder.get() - request = {} - request["force"] = options["force"] - request["auto_accept"] = True - request["name"] = \ - ("client: reconcile-resources-cyclades, time: %s" - % datetime.now()) - request["provisions"] = map(create_provision, unsynced) + force = options["force"] + name = ("client: reconcile-resources-cyclades, time: %s" + % datetime.now()) + user_provisions = reconcile.create_user_provisions( + unsynced_users) + project_provisions = reconcile.create_project_provisions( + unsynced_projects) try: - qh.issue_commission(request) + qh.issue_commission_generic( + user_provisions, project_provisions, + name=name, force=force, + auto_accept=True) except quotas.errors.QuotaLimit: write("Reconciling failed because a limit has been " "reached. Use --force to ignore the check.\n") @@ -138,13 +118,5 @@ class Command(BaseCommand): if pending_exists: write("Found pending commissions. Run 'snf-manage" " reconcile-commissions-cyclades'\n") - elif not (unsynced or unknown_user_exists): + elif not (unsynced or unknown_exists): write("Everything in sync.\n") - - -def create_provision(provision_info): - user, resource, db_value, qh_value = provision_info - return {"holder": user, - "source": quotas.DEFAULT_SOURCE, - "resource": resource, - "quantity": db_value - qh_value} diff --git a/snf-cyclades-app/synnefo/quotas/util.py b/snf-cyclades-app/synnefo/quotas/util.py index 38674f6998bbccb2f404237b1339b832391f6f9f..415d0d1c2f91de3b109d5c518c736145fa7e633b 100644 --- a/snf-cyclades-app/synnefo/quotas/util.py +++ b/snf-cyclades-app/synnefo/quotas/util.py @@ -35,9 +35,14 @@ from django.db.models import Sum, Count, Q from synnefo.db.models import VirtualMachine, Network, IPAddress from synnefo.quotas import Quotaholder +from synnefo.util.keypath import set_path -def get_db_holdings(user=None): +MiB = 2 ** 20 +GiB = 2 ** 30 + + +def get_db_holdings(user=None, project=None): """Get holdings from Cyclades DB.""" holdings = {} @@ -50,46 +55,123 @@ def get_db_holdings(user=None): networks = networks.filter(userid=user) floating_ips = floating_ips.filter(userid=user) + if project is not None: + vms = vms.filter(project=project) + networks = networks.filter(project=project) + floating_ips = floating_ips.filter(project=project) + # Get resources related with VMs - vm_resources = vms.values("userid")\ - .annotate(num=Count("id"), - total_ram=Sum("flavor__ram"), - total_cpu=Sum("flavor__cpu"), - disk=Sum("flavor__disk")) - vm_active_resources = \ - vms.values("userid")\ - .filter(Q(operstate="STARTED") | Q(operstate="BUILD") | - Q(operstate="ERROR"))\ - .annotate(ram=Sum("flavor__ram"), - cpu=Sum("flavor__cpu")) + vm_resources = vms.values("userid", "project")\ + .annotate(num=Count("id"), + total_ram=Sum("flavor__ram"), + total_cpu=Sum("flavor__cpu"), + disk=Sum("flavor__disk")) for vm_res in vm_resources.iterator(): user = vm_res['userid'] + project = vm_res['project'] res = {"cyclades.vm": vm_res["num"], "cyclades.total_cpu": vm_res["total_cpu"], - "cyclades.disk": 1073741824 * vm_res["disk"], - "cyclades.total_ram": 1048576 * vm_res["total_ram"]} - holdings[user] = res + "cyclades.disk": vm_res["disk"] * GiB, + "cyclades.total_ram": vm_res["total_ram"] * MiB} + set_path(holdings, [user, project], res, createpath=True) + + vm_active_resources = vms.values("userid", "project")\ + .filter(Q(operstate="STARTED") | Q(operstate="BUILD") | + Q(operstate="ERROR"))\ + .annotate(ram=Sum("flavor__ram"), + cpu=Sum("flavor__cpu")) for vm_res in vm_active_resources.iterator(): user = vm_res['userid'] - holdings[user]["cyclades.cpu"] = vm_res["cpu"] - holdings[user]["cyclades.ram"] = 1048576 * vm_res["ram"] + project = vm_res['project'] + set_path(holdings, [user, project, "cyclades.cpu"], vm_res["cpu"], + createpath=True) + set_path(holdings, [user, project, "cyclades.ram"], + vm_res["ram"] * MiB, createpath=True) # Get resources related with networks - net_resources = networks.values("userid")\ + net_resources = networks.values("userid", "project")\ .annotate(num=Count("id")) + for net_res in net_resources.iterator(): user = net_res['userid'] - holdings.setdefault(user, {}) - holdings[user]["cyclades.network.private"] = net_res["num"] + if user is None: + continue + project = net_res['project'] + set_path(holdings, [user, project, "cyclades.network.private"], + net_res["num"], createpath=True) - floating_ips_resources = floating_ips.values("userid")\ + floating_ips_resources = floating_ips.values("userid", "project")\ .annotate(num=Count("id")) + for floating_ip_res in floating_ips_resources.iterator(): user = floating_ip_res["userid"] - holdings.setdefault(user, {}) - holdings[user]["cyclades.floating_ip"] = floating_ip_res["num"] + project = floating_ip_res["project"] + set_path(holdings, [user, project, "cyclades.floating_ip"], + floating_ip_res["num"], createpath=True) + + return holdings + + +def get_db_project_holdings(project=None): + """Get holdings from Cyclades DB.""" + holdings = {} + + vms = VirtualMachine.objects.filter(deleted=False) + networks = Network.objects.filter(deleted=False) + floating_ips = IPAddress.objects.filter(deleted=False, floating_ip=True) + + if project is not None: + vms = vms.filter(project=project) + networks = networks.filter(project=project) + floating_ips = floating_ips.filter(project=project) + + # Get resources related with VMs + vm_resources = vms.values("project")\ + .annotate(num=Count("id"), + total_ram=Sum("flavor__ram"), + total_cpu=Sum("flavor__cpu"), + disk=Sum("flavor__disk")) + + for vm_res in vm_resources.iterator(): + project = vm_res['project'] + res = {"cyclades.vm": vm_res["num"], + "cyclades.total_cpu": vm_res["total_cpu"], + "cyclades.disk": vm_res["disk"] * GiB, + "cyclades.total_ram": vm_res["total_ram"] * MiB} + set_path(holdings, [project], res, createpath=True) + + vm_active_resources = vms.values("project")\ + .filter(Q(operstate="STARTED") | Q(operstate="BUILD") | + Q(operstate="ERROR"))\ + .annotate(ram=Sum("flavor__ram"), + cpu=Sum("flavor__cpu")) + + for vm_res in vm_active_resources.iterator(): + project = vm_res['project'] + set_path(holdings, [project, "cyclades.cpu"], vm_res["cpu"], + createpath=True) + set_path(holdings, [project, "cyclades.ram"], + vm_res["ram"] * MiB, createpath=True) + + # Get resources related with networks + net_resources = networks.values("project").annotate(num=Count("id")) + + for net_res in net_resources.iterator(): + project = net_res['project'] + if project is None: + continue + set_path(holdings, [project, "cyclades.network.private"], + net_res["num"], createpath=True) + + floating_ips_resources = floating_ips.values("project")\ + .annotate(num=Count("id")) + + for floating_ip_res in floating_ips_resources.iterator(): + project = floating_ip_res["project"] + set_path(holdings, [project, "cyclades.floating_ip"], + floating_ip_res["num"], createpath=True) return holdings