From 2855792ed187cfe368f8ba61a2336336dbcae2e3 Mon Sep 17 00:00:00 2001 From: Christos Stavrakakis Date: Mon, 3 Dec 2012 13:56:50 +0200 Subject: [PATCH] Implement quotas-init and quotas-verify Implement quotas-init and quotas-verify management commands. - quotas-init sets quotas of Quotaholder to the state of quotas in Cyclades DB - quotas-verify detects incosistencies between quotas in Quotaholder and Cycldes DB. --- .../synnefo/app_settings/__init__.py | 1 + .../synnefo/quotas/management/__init__.py | 0 .../quotas/management/commands/__init__.py | 0 .../quotas/management/commands/models.py | 0 .../quotas/management/commands/quotas-init.py | 79 +++++++++++++ .../management/commands/quotas-verify.py | 96 ++++++++++++++++ .../management/commands/reconcile-quotas.py | 0 snf-cyclades-app/synnefo/quotas/util.py | 105 ++++++++++++++++++ 8 files changed, 281 insertions(+) create mode 100644 snf-cyclades-app/synnefo/quotas/management/__init__.py create mode 100644 snf-cyclades-app/synnefo/quotas/management/commands/__init__.py create mode 100644 snf-cyclades-app/synnefo/quotas/management/commands/models.py create mode 100644 snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py create mode 100644 snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py rename snf-cyclades-app/synnefo/{logic => quotas}/management/commands/reconcile-quotas.py (100%) create mode 100644 snf-cyclades-app/synnefo/quotas/util.py diff --git a/snf-cyclades-app/synnefo/app_settings/__init__.py b/snf-cyclades-app/synnefo/app_settings/__init__.py index 4c725761b..bc55fc717 100644 --- a/snf-cyclades-app/synnefo/app_settings/__init__.py +++ b/snf-cyclades-app/synnefo/app_settings/__init__.py @@ -9,6 +9,7 @@ synnefo_web_apps = [ 'synnefo.helpdesk', 'synnefo.ui.userdata', 'synnefo.helpdesk', + 'synnefo.quotas', ] synnefo_web_middleware = [] diff --git a/snf-cyclades-app/synnefo/quotas/management/__init__.py b/snf-cyclades-app/synnefo/quotas/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/__init__.py b/snf-cyclades-app/synnefo/quotas/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/models.py b/snf-cyclades-app/synnefo/quotas/management/commands/models.py new file mode 100644 index 000000000..e69de29bb diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py new file mode 100644 index 000000000..d800c14a7 --- /dev/null +++ b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py @@ -0,0 +1,79 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from django.core.management.base import BaseCommand +from optparse import make_option + +from synnefo.quotas import get_quota_holder +from synnefo.quotas.util import get_db_holdings + + +class Command(BaseCommand): + help = "Reconcile quotas with Quotaholder" + output_transaction = True + option_list = BaseCommand.option_list + ( + make_option("--userid", dest="userid", + default=None, + help="Verify quotas only for this user"), + make_option("--fix", dest="fix", + action='store_true', + default=False, + help="Fix pending commissions" + ), + make_option("--dry-run", dest="dry_run", + action='store_true', + default=False), + ) + + def handle(self, *args, **options): + userid = options['userid'] + + users = [userid] if userid else None + # Get info from DB + db_holdings = get_db_holdings(users) + + # Create commissions + with get_quota_holder() as qh: + for user, resources in db_holdings.items(): + if not user: + continue + reset_holding = [] + for res, val in resources.items(): + reset_holding.append((user, "cyclades." + res, "1", val, 0, 0, 0)) + if not options['dry_run']: + try: + qh.reset_holding(context={}, reset_holding=reset_holding) + except Exception as e: + self.stderr.write("Can not set up holding:%s" % e) + else: + self.stdout.write("Reseting holding: %s\n" % reset_holding) diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py new file mode 100644 index 000000000..1dea204eb --- /dev/null +++ b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py @@ -0,0 +1,96 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from django.core.management.base import BaseCommand +from optparse import make_option + +from synnefo.quotas.util import get_db_holdings, get_quotaholder_holdings +from synnefo.management.common import pprint_table + + +class Command(BaseCommand): + output_transaction = True + option_list = BaseCommand.option_list + ( + make_option("--userid", dest="userid", + default=None, + help="Verify quotas only for this user"), + ) + + def handle(self, *args, **options): + write = self.stdout.write + userid = options['userid'] + + users = [userid] if userid else None + # Get info from DB + db_holdings = get_db_holdings(users) + users = db_holdings.keys() + qh_holdings = get_quotaholder_holdings(users) + qh_users = qh_holdings.keys() + + if len(qh_users) < len(users): + for u in set(users) - set(qh_users): + write("Unknown entity: %s\n" % u) + users = qh_users + + headers = ("User", "Resource", "Database", "Quotaholder") + unsynced = [] + for user in users: + db = db_holdings[user] + qh = qh_holdings[user] + if not self.verify_resources(user, db.keys(), qh.keys()): + continue + + for res in db.keys(): + if db[res] != qh[res]: + unsynced.append((user, res, str(db[res]), str(qh[res]))) + + if unsynced: + pprint_table(self.stderr, unsynced, headers) + + def verify_resources(self, user, db_resources, qh_resources): + write = self.stderr.write + db_res = set(db_resources) + qh_res = set(qh_resources) + if qh_res == db_res: + return True + db_extra = db_res - qh_res + if db_extra: + for res in db_extra: + write("Resource %s exists in DB for %s but not in QH\n"\ + % (res, user)) + qh_extra = qh_res - db_res + if qh_extra: + for res in qh_extra: + write("Resource %s exists in QH for %s but not in DB\n"\ + % (res, user)) + return False diff --git a/snf-cyclades-app/synnefo/logic/management/commands/reconcile-quotas.py b/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-quotas.py similarity index 100% rename from snf-cyclades-app/synnefo/logic/management/commands/reconcile-quotas.py rename to snf-cyclades-app/synnefo/quotas/management/commands/reconcile-quotas.py diff --git a/snf-cyclades-app/synnefo/quotas/util.py b/snf-cyclades-app/synnefo/quotas/util.py new file mode 100644 index 000000000..5651c5aed --- /dev/null +++ b/snf-cyclades-app/synnefo/quotas/util.py @@ -0,0 +1,105 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from django.db.models import Sum, Count + +from synnefo.db.models import VirtualMachine, Network +from synnefo.quotas import get_quota_holder +from synnefo.lib.quotaholder.api.exception import NoEntityError + + +def get_db_holdings(users=None): + """Get holdings from Cyclades DB.""" + holdings = {} + + vms = VirtualMachine.objects + networks = Network.objects + + if users: + assert(type(users) is list) + vms = vms.filter(userid__in=users) + networks = networks.filter(userid__in=users) + + # Get resources related with VMs + vm_resources = vms.values("userid").annotate(num=Count("id"), + ram=Sum("flavor__ram"), + cpu=Sum("flavor__cpu"), + disk=Sum("flavor__disk")) + for vm_res in vm_resources: + user = vm_res['userid'] + res = {"vm": vm_res["num"], + "cpu": vm_res["cpu"], + "disk": 1073741824 * vm_res["disk"], + "ram": 1048576 * vm_res["ram"]} + holdings[user] = res + + # Get resources related with networks + net_resources = networks.values("userid")\ + .annotate(num=Count("id")) + for net_res in net_resources: + user = net_res['userid'] + if user not in holdings: + holdings[user] = {} + holdings[user]["network.private"] = net_res["num"] + + return holdings + + +def get_quotaholder_holdings(users=[]): + """Get holdings from Quotaholder. + + If the entity for the user does not exist in quotaholder, no holding + is returned. + """ + holdings = {} + with get_quota_holder() as qh: + for user in users: + try: + (qh_holdings, _) = \ + qh.list_holdings(context={}, list_holdings=[(user, "1")]) + if not qh_holdings: + continue + qh_holdings = qh_holdings[0] + qh_holdings = filter(lambda x: x[1].startswith("cyclades."), + qh_holdings) + holdings[user] = dict(map(decode_holding, qh_holdings)) + except NoEntityError: + pass + return holdings + + +def decode_holding(holding): + entity, resource, imported, exported, returned, released = \ + holding + res = resource.replace("cyclades.", "") + return (res, imported - exported + returned - released) -- GitLab