Implement quotas-init and quotas-verify

Implement quotas-init and quotas-verify management commands.
- quotas-init sets quotas of Quotaholder to the state of quotas in
  Cyclades DB
- quotas-verify detects incosistencies between quotas in Quotaholder and
  Cycldes DB.

snf-cyclades-app/synnefo/app_settings/__init__.py

@@ -9,6 +9,7 @@ synnefo_web_apps = [
     'synnefo.helpdesk',
     'synnefo.ui.userdata',
     'synnefo.helpdesk',
+    'synnefo.quotas',
 ]
 
 synnefo_web_middleware = []

snf-cyclades-app/synnefo/quotas/management/commands/quotas-init.py

+# Copyright 2012 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from django.core.management.base import BaseCommand
+from optparse import make_option
+
+from synnefo.quotas import get_quota_holder
+from synnefo.quotas.util import get_db_holdings
+
+
+class Command(BaseCommand):
+    help = "Reconcile quotas with Quotaholder"
+    output_transaction = True
+    option_list = BaseCommand.option_list + (
+        make_option("--userid", dest="userid",
+                    default=None,
+                    help="Verify quotas only for this user"),
+        make_option("--fix", dest="fix",
+                    action='store_true',
+                    default=False,
+                    help="Fix pending commissions"
+                    ),
+        make_option("--dry-run", dest="dry_run",
+                    action='store_true',
+                    default=False),
+    )
+
+    def handle(self, *args, **options):
+        userid = options['userid']
+
+        users = [userid] if userid else None
+        # Get info from DB
+        db_holdings = get_db_holdings(users)
+
+        # Create commissions
+        with get_quota_holder() as qh:
+            for user, resources in db_holdings.items():
+                if not user:
+                    continue
+                reset_holding = []
+                for res, val in resources.items():
+                    reset_holding.append((user, "cyclades." + res, "1", val, 0, 0, 0))
+                if not options['dry_run']:
+                    try:
+                        qh.reset_holding(context={}, reset_holding=reset_holding)
+                    except Exception as e:
+                        self.stderr.write("Can not set up holding:%s" % e)
+                else:
+                    self.stdout.write("Reseting holding: %s\n" % reset_holding)

snf-cyclades-app/synnefo/quotas/management/commands/quotas-verify.py

+# Copyright 2012 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from django.core.management.base import BaseCommand
+from optparse import make_option
+
+from synnefo.quotas.util import get_db_holdings, get_quotaholder_holdings
+from synnefo.management.common import pprint_table
+
+
+class Command(BaseCommand):
+    output_transaction = True
+    option_list = BaseCommand.option_list + (
+        make_option("--userid", dest="userid",
+                    default=None,
+                    help="Verify quotas only for this user"),
+    )
+
+    def handle(self, *args, **options):
+        write = self.stdout.write
+        userid = options['userid']
+
+        users = [userid] if userid else None
+        # Get info from DB
+        db_holdings = get_db_holdings(users)
+        users = db_holdings.keys()
+        qh_holdings = get_quotaholder_holdings(users)
+        qh_users = qh_holdings.keys()
+
+        if len(qh_users) < len(users):
+            for u in set(users) - set(qh_users):
+                write("Unknown entity: %s\n" % u)
+                users = qh_users
+
+        headers = ("User", "Resource", "Database", "Quotaholder")
+        unsynced = []
+        for user in users:
+            db = db_holdings[user]
+            qh = qh_holdings[user]
+            if not self.verify_resources(user, db.keys(), qh.keys()):
+                continue
+
+            for res in db.keys():
+                if db[res] != qh[res]:
+                    unsynced.append((user, res, str(db[res]), str(qh[res])))
+
+        if unsynced:
+            pprint_table(self.stderr, unsynced, headers)
+
+    def verify_resources(self, user, db_resources, qh_resources):
+        write = self.stderr.write
+        db_res = set(db_resources)
+        qh_res = set(qh_resources)
+        if qh_res == db_res:
+            return True
+        db_extra = db_res - qh_res
+        if db_extra:
+            for res in db_extra:
+                write("Resource %s exists in DB for %s but not in QH\n"\
+                      % (res, user))
+        qh_extra = qh_res - db_res
+        if qh_extra:
+            for res in qh_extra:
+                write("Resource %s exists in QH for %s but not in DB\n"\
+                      % (res, user))
+        return False

snf-cyclades-app/synnefo/quotas/util.py

+# Copyright 2012 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from django.db.models import Sum, Count
+
+from synnefo.db.models import VirtualMachine, Network
+from synnefo.quotas import get_quota_holder
+from synnefo.lib.quotaholder.api.exception import NoEntityError
+
+
+def get_db_holdings(users=None):
+    """Get holdings from Cyclades DB."""
+    holdings = {}
+
+    vms = VirtualMachine.objects
+    networks = Network.objects
+
+    if users:
+        assert(type(users) is list)
+        vms = vms.filter(userid__in=users)
+        networks = networks.filter(userid__in=users)
+
+    # Get resources related with VMs
+    vm_resources = vms.values("userid").annotate(num=Count("id"),
+                                                 ram=Sum("flavor__ram"),
+                                                 cpu=Sum("flavor__cpu"),
+                                                 disk=Sum("flavor__disk"))
+    for vm_res in vm_resources:
+        user = vm_res['userid']
+        res = {"vm": vm_res["num"],
+               "cpu": vm_res["cpu"],
+               "disk": 1073741824 * vm_res["disk"],
+               "ram": 1048576 * vm_res["ram"]}
+        holdings[user] = res
+
+    # Get resources related with networks
+    net_resources = networks.values("userid")\
+                                    .annotate(num=Count("id"))
+    for net_res in net_resources:
+        user = net_res['userid']
+        if user not in holdings:
+            holdings[user] = {}
+        holdings[user]["network.private"] = net_res["num"]
+
+    return holdings
+
+
+def get_quotaholder_holdings(users=[]):
+    """Get holdings from Quotaholder.
+
+    If the entity for the user does not exist in quotaholder, no holding
+    is returned.
+    """
+    holdings = {}
+    with get_quota_holder() as qh:
+        for user in users:
+            try:
+                (qh_holdings, _) = \
+                    qh.list_holdings(context={}, list_holdings=[(user, "1")])
+                if not qh_holdings:
+                    continue
+                qh_holdings = qh_holdings[0]
+                qh_holdings = filter(lambda x: x[1].startswith("cyclades."),
+                                     qh_holdings)
+                holdings[user] = dict(map(decode_holding, qh_holdings))
+            except NoEntityError:
+                pass
+    return holdings
+
+
+def decode_holding(holding):
+    entity, resource, imported, exported, returned, released = \
+            holding
+    res = resource.replace("cyclades.", "")
+    return (res, imported - exported + returned - released)