Commit 220d507b authored by Giorgos Korfiatis's avatar Giorgos Korfiatis

cyclades: Adapt resource reconciliation

Take projects into account when computing resource usage and comparing
with the quotaholder values.
parent fc9f5243
......@@ -35,11 +35,10 @@ from datetime import datetime
from django.core.management.base import BaseCommand
from optparse import make_option
from synnefo import quotas
from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings,
transform_quotas)
from synnefo.quotas import util
from snf_django.management.utils import pprint_table
from snf_django.utils import reconcile
class Command(BaseCommand):
......@@ -53,6 +52,8 @@ class Command(BaseCommand):
make_option("--userid", dest="userid",
default=None,
help="Reconcile resources only for this user"),
make_option("--project",
help="Reconcile resources only for this project"),
make_option("--fix", dest="fix",
default=False,
action="store_true",
......@@ -67,68 +68,47 @@ class Command(BaseCommand):
def handle(self, *args, **options):
write = self.stderr.write
userid = options['userid']
project = options["project"]
# Get holdings from Cyclades DB
db_holdings = get_db_holdings(userid)
# Get holdings from QuotaHolder
qh_holdings = get_quotaholder_holdings(userid)
users = set(db_holdings.keys())
users.update(qh_holdings.keys())
# Remove 'None' user
users.discard(None)
if userid and userid not in users:
write("User '%s' does not exist in Quotaholder!", userid)
return
pending_exists = False
unknown_user_exists = False
unsynced = []
for user in users:
db = db_holdings.get(user, {})
try:
qh_all = qh_holdings[user]
except KeyError:
write("User '%s' does not exist in Quotaholder!\n" %
user)
unknown_user_exists = True
continue
db_holdings = util.get_db_holdings(userid, project)
db_project_holdings = util.get_db_project_holdings(project)
# Assuming only one source
qh = qh_all.get(quotas.DEFAULT_SOURCE, {})
qh = transform_quotas(qh)
for resource in quotas.RESOURCES:
db_value = db.pop(resource, 0)
try:
qh_value, _, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s'!\n" % (resource, user))
continue
if qh_pending:
write("Pending commission. User '%s', resource '%s'.\n" %
(user, resource))
pending_exists = True
continue
if db_value != qh_value:
data = (user, resource, db_value, qh_value)
unsynced.append(data)
headers = ("User", "Resource", "Database", "Quotaholder")
# Get holdings from QuotaHolder
qh_holdings = util.get_qh_users_holdings(
[userid] if userid is not None else None)
qh_project_holdings = util.get_qh_project_holdings(
[project] if project is not None else None)
unsynced_users, users_pending, users_unknown =\
reconcile.check_users(self.stderr, quotas.RESOURCES,
db_holdings, qh_holdings)
unsynced_projects, projects_pending, projects_unknown =\
reconcile.check_projects(self.stderr, quotas.RESOURCES,
db_project_holdings, qh_project_holdings)
pending_exists = users_pending or projects_pending
unknown_exists = users_unknown or projects_unknown
headers = ("Type", "Holder", "Source", "Resource",
"Database", "Quotaholder")
unsynced = unsynced_users + unsynced_projects
if unsynced:
pprint_table(self.stdout, unsynced, headers)
if options["fix"]:
qh = quotas.Quotaholder.get()
request = {}
request["force"] = options["force"]
request["auto_accept"] = True
request["name"] = \
("client: reconcile-resources-cyclades, time: %s"
% datetime.now())
request["provisions"] = map(create_provision, unsynced)
force = options["force"]
name = ("client: reconcile-resources-cyclades, time: %s"
% datetime.now())
user_provisions = reconcile.create_user_provisions(
unsynced_users)
project_provisions = reconcile.create_project_provisions(
unsynced_projects)
try:
qh.issue_commission(request)
qh.issue_commission_generic(
user_provisions, project_provisions,
name=name, force=force,
auto_accept=True)
except quotas.errors.QuotaLimit:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
......@@ -138,13 +118,5 @@ class Command(BaseCommand):
if pending_exists:
write("Found pending commissions. Run 'snf-manage"
" reconcile-commissions-cyclades'\n")
elif not (unsynced or unknown_user_exists):
elif not (unsynced or unknown_exists):
write("Everything in sync.\n")
def create_provision(provision_info):
user, resource, db_value, qh_value = provision_info
return {"holder": user,
"source": quotas.DEFAULT_SOURCE,
"resource": resource,
"quantity": db_value - qh_value}
......@@ -35,9 +35,14 @@ from django.db.models import Sum, Count, Q
from synnefo.db.models import VirtualMachine, Network, IPAddress
from synnefo.quotas import Quotaholder
from synnefo.util.keypath import set_path
def get_db_holdings(user=None):
MiB = 2 ** 20
GiB = 2 ** 30
def get_db_holdings(user=None, project=None):
"""Get holdings from Cyclades DB."""
holdings = {}
......@@ -50,46 +55,123 @@ def get_db_holdings(user=None):
networks = networks.filter(userid=user)
floating_ips = floating_ips.filter(userid=user)
if project is not None:
vms = vms.filter(project=project)
networks = networks.filter(project=project)
floating_ips = floating_ips.filter(project=project)
# Get resources related with VMs
vm_resources = vms.values("userid")\
.annotate(num=Count("id"),
total_ram=Sum("flavor__ram"),
total_cpu=Sum("flavor__cpu"),
disk=Sum("flavor__disk"))
vm_active_resources = \
vms.values("userid")\
.filter(Q(operstate="STARTED") | Q(operstate="BUILD") |
Q(operstate="ERROR"))\
.annotate(ram=Sum("flavor__ram"),
cpu=Sum("flavor__cpu"))
vm_resources = vms.values("userid", "project")\
.annotate(num=Count("id"),
total_ram=Sum("flavor__ram"),
total_cpu=Sum("flavor__cpu"),
disk=Sum("flavor__disk"))
for vm_res in vm_resources.iterator():
user = vm_res['userid']
project = vm_res['project']
res = {"cyclades.vm": vm_res["num"],
"cyclades.total_cpu": vm_res["total_cpu"],
"cyclades.disk": 1073741824 * vm_res["disk"],
"cyclades.total_ram": 1048576 * vm_res["total_ram"]}
holdings[user] = res
"cyclades.disk": vm_res["disk"] * GiB,
"cyclades.total_ram": vm_res["total_ram"] * MiB}
set_path(holdings, [user, project], res, createpath=True)
vm_active_resources = vms.values("userid", "project")\
.filter(Q(operstate="STARTED") | Q(operstate="BUILD") |
Q(operstate="ERROR"))\
.annotate(ram=Sum("flavor__ram"),
cpu=Sum("flavor__cpu"))
for vm_res in vm_active_resources.iterator():
user = vm_res['userid']
holdings[user]["cyclades.cpu"] = vm_res["cpu"]
holdings[user]["cyclades.ram"] = 1048576 * vm_res["ram"]
project = vm_res['project']
set_path(holdings, [user, project, "cyclades.cpu"], vm_res["cpu"],
createpath=True)
set_path(holdings, [user, project, "cyclades.ram"],
vm_res["ram"] * MiB, createpath=True)
# Get resources related with networks
net_resources = networks.values("userid")\
net_resources = networks.values("userid", "project")\
.annotate(num=Count("id"))
for net_res in net_resources.iterator():
user = net_res['userid']
holdings.setdefault(user, {})
holdings[user]["cyclades.network.private"] = net_res["num"]
if user is None:
continue
project = net_res['project']
set_path(holdings, [user, project, "cyclades.network.private"],
net_res["num"], createpath=True)
floating_ips_resources = floating_ips.values("userid")\
floating_ips_resources = floating_ips.values("userid", "project")\
.annotate(num=Count("id"))
for floating_ip_res in floating_ips_resources.iterator():
user = floating_ip_res["userid"]
holdings.setdefault(user, {})
holdings[user]["cyclades.floating_ip"] = floating_ip_res["num"]
project = floating_ip_res["project"]
set_path(holdings, [user, project, "cyclades.floating_ip"],
floating_ip_res["num"], createpath=True)
return holdings
def get_db_project_holdings(project=None):
"""Get holdings from Cyclades DB."""
holdings = {}
vms = VirtualMachine.objects.filter(deleted=False)
networks = Network.objects.filter(deleted=False)
floating_ips = IPAddress.objects.filter(deleted=False, floating_ip=True)
if project is not None:
vms = vms.filter(project=project)
networks = networks.filter(project=project)
floating_ips = floating_ips.filter(project=project)
# Get resources related with VMs
vm_resources = vms.values("project")\
.annotate(num=Count("id"),
total_ram=Sum("flavor__ram"),
total_cpu=Sum("flavor__cpu"),
disk=Sum("flavor__disk"))
for vm_res in vm_resources.iterator():
project = vm_res['project']
res = {"cyclades.vm": vm_res["num"],
"cyclades.total_cpu": vm_res["total_cpu"],
"cyclades.disk": vm_res["disk"] * GiB,
"cyclades.total_ram": vm_res["total_ram"] * MiB}
set_path(holdings, [project], res, createpath=True)
vm_active_resources = vms.values("project")\
.filter(Q(operstate="STARTED") | Q(operstate="BUILD") |
Q(operstate="ERROR"))\
.annotate(ram=Sum("flavor__ram"),
cpu=Sum("flavor__cpu"))
for vm_res in vm_active_resources.iterator():
project = vm_res['project']
set_path(holdings, [project, "cyclades.cpu"], vm_res["cpu"],
createpath=True)
set_path(holdings, [project, "cyclades.ram"],
vm_res["ram"] * MiB, createpath=True)
# Get resources related with networks
net_resources = networks.values("project").annotate(num=Count("id"))
for net_res in net_resources.iterator():
project = net_res['project']
if project is None:
continue
set_path(holdings, [project, "cyclades.network.private"],
net_res["num"], createpath=True)
floating_ips_resources = floating_ips.values("project")\
.annotate(num=Count("id"))
for floating_ip_res in floating_ips_resources.iterator():
project = floating_ip_res["project"]
set_path(holdings, [project, "cyclades.floating_ip"],
floating_ip_res["num"], createpath=True)
return holdings
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment