diff --git a/snf-cyclades-app/synnefo/app_settings/__init__.py b/snf-cyclades-app/synnefo/app_settings/__init__.py index 4c725761b6f339f5cbf4a80cd989370f962a434c..bc55fc717ec2f04dbec268da6056da05d99b8ba4 100644 --- a/snf-cyclades-app/synnefo/app_settings/__init__.py +++ b/snf-cyclades-app/synnefo/app_settings/__init__.py @@ -9,6 +9,7 @@ synnefo_web_apps = [ 'synnefo.helpdesk', 'synnefo.ui.userdata', 'synnefo.helpdesk', + 'synnefo.quotas', ] synnefo_web_middleware = [] diff --git a/snf-cyclades-app/synnefo/quotas/management/__init__.py b/snf-cyclades-app/synnefo/quotas/management/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/__init__.py b/snf-cyclades-app/synnefo/quotas/management/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/models.py b/snf-cyclades-app/synnefo/quotas/management/commands/models.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py new file mode 100644 index 0000000000000000000000000000000000000000..d800c14a781c2b6ea14be230ea141ca219cfce14 --- /dev/null +++ b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py @@ -0,0 +1,79 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from django.core.management.base import BaseCommand +from optparse import make_option + +from synnefo.quotas import get_quota_holder +from synnefo.quotas.util import get_db_holdings + + +class Command(BaseCommand): + help = "Reconcile quotas with Quotaholder" + output_transaction = True + option_list = BaseCommand.option_list + ( + make_option("--userid", dest="userid", + default=None, + help="Verify quotas only for this user"), + make_option("--fix", dest="fix", + action='store_true', + default=False, + help="Fix pending commissions" + ), + make_option("--dry-run", dest="dry_run", + action='store_true', + default=False), + ) + + def handle(self, *args, **options): + userid = options['userid'] + + users = [userid] if userid else None + # Get info from DB + db_holdings = get_db_holdings(users) + + # Create commissions + with get_quota_holder() as qh: + for user, resources in db_holdings.items(): + if not user: + continue + reset_holding = [] + for res, val in resources.items(): + reset_holding.append((user, "cyclades." + res, "1", val, 0, 0, 0)) + if not options['dry_run']: + try: + qh.reset_holding(context={}, reset_holding=reset_holding) + except Exception as e: + self.stderr.write("Can not set up holding:%s" % e) + else: + self.stdout.write("Reseting holding: %s\n" % reset_holding) diff --git a/snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py new file mode 100644 index 0000000000000000000000000000000000000000..1dea204eb823231f4232d0c19307c8920cc51b54 --- /dev/null +++ b/snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py @@ -0,0 +1,96 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from django.core.management.base import BaseCommand +from optparse import make_option + +from synnefo.quotas.util import get_db_holdings, get_quotaholder_holdings +from synnefo.management.common import pprint_table + + +class Command(BaseCommand): + output_transaction = True + option_list = BaseCommand.option_list + ( + make_option("--userid", dest="userid", + default=None, + help="Verify quotas only for this user"), + ) + + def handle(self, *args, **options): + write = self.stdout.write + userid = options['userid'] + + users = [userid] if userid else None + # Get info from DB + db_holdings = get_db_holdings(users) + users = db_holdings.keys() + qh_holdings = get_quotaholder_holdings(users) + qh_users = qh_holdings.keys() + + if len(qh_users) < len(users): + for u in set(users) - set(qh_users): + write("Unknown entity: %s\n" % u) + users = qh_users + + headers = ("User", "Resource", "Database", "Quotaholder") + unsynced = [] + for user in users: + db = db_holdings[user] + qh = qh_holdings[user] + if not self.verify_resources(user, db.keys(), qh.keys()): + continue + + for res in db.keys(): + if db[res] != qh[res]: + unsynced.append((user, res, str(db[res]), str(qh[res]))) + + if unsynced: + pprint_table(self.stderr, unsynced, headers) + + def verify_resources(self, user, db_resources, qh_resources): + write = self.stderr.write + db_res = set(db_resources) + qh_res = set(qh_resources) + if qh_res == db_res: + return True + db_extra = db_res - qh_res + if db_extra: + for res in db_extra: + write("Resource %s exists in DB for %s but not in QH\n"\ + % (res, user)) + qh_extra = qh_res - db_res + if qh_extra: + for res in qh_extra: + write("Resource %s exists in QH for %s but not in DB\n"\ + % (res, user)) + return False diff --git a/snf-cyclades-app/synnefo/logic/management/commands/reconcile-quotas.py b/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-quotas.py similarity index 100% rename from snf-cyclades-app/synnefo/logic/management/commands/reconcile-quotas.py rename to snf-cyclades-app/synnefo/quotas/management/commands/reconcile-quotas.py diff --git a/snf-cyclades-app/synnefo/quotas/util.py b/snf-cyclades-app/synnefo/quotas/util.py new file mode 100644 index 0000000000000000000000000000000000000000..5651c5aedb2e4eec18e37a5a17fb893884cf16ca --- /dev/null +++ b/snf-cyclades-app/synnefo/quotas/util.py @@ -0,0 +1,105 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from django.db.models import Sum, Count + +from synnefo.db.models import VirtualMachine, Network +from synnefo.quotas import get_quota_holder +from synnefo.lib.quotaholder.api.exception import NoEntityError + + +def get_db_holdings(users=None): + """Get holdings from Cyclades DB.""" + holdings = {} + + vms = VirtualMachine.objects + networks = Network.objects + + if users: + assert(type(users) is list) + vms = vms.filter(userid__in=users) + networks = networks.filter(userid__in=users) + + # Get resources related with VMs + vm_resources = vms.values("userid").annotate(num=Count("id"), + ram=Sum("flavor__ram"), + cpu=Sum("flavor__cpu"), + disk=Sum("flavor__disk")) + for vm_res in vm_resources: + user = vm_res['userid'] + res = {"vm": vm_res["num"], + "cpu": vm_res["cpu"], + "disk": 1073741824 * vm_res["disk"], + "ram": 1048576 * vm_res["ram"]} + holdings[user] = res + + # Get resources related with networks + net_resources = networks.values("userid")\ + .annotate(num=Count("id")) + for net_res in net_resources: + user = net_res['userid'] + if user not in holdings: + holdings[user] = {} + holdings[user]["network.private"] = net_res["num"] + + return holdings + + +def get_quotaholder_holdings(users=[]): + """Get holdings from Quotaholder. + + If the entity for the user does not exist in quotaholder, no holding + is returned. + """ + holdings = {} + with get_quota_holder() as qh: + for user in users: + try: + (qh_holdings, _) = \ + qh.list_holdings(context={}, list_holdings=[(user, "1")]) + if not qh_holdings: + continue + qh_holdings = qh_holdings[0] + qh_holdings = filter(lambda x: x[1].startswith("cyclades."), + qh_holdings) + holdings[user] = dict(map(decode_holding, qh_holdings)) + except NoEntityError: + pass + return holdings + + +def decode_holding(holding): + entity, resource, imported, exported, returned, released = \ + holding + res = resource.replace("cyclades.", "") + return (res, imported - exported + returned - released)