Commit 6f60cf44 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki Committed by Giorgos Korfiatis

pithos: Update reconcile-resources-pithos command

parent 220d507b
# Copyright 2012 GRNET S.A. All rights reserved. # Copyright 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -31,19 +31,20 @@ ...@@ -31,19 +31,20 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.core.management.base import NoArgsCommand from datetime import datetime
from django.core.management.base import NoArgsCommand, CommandError
from optparse import make_option from optparse import make_option
from pithos.api.util import get_backend from pithos.api.util import get_backend
from pithos.api.resources import resources
from pithos.backends.modular import DEFAULT_SOURCE
from snf_django.management import utils from snf_django.management import utils
from astakosclient.errors import QuotaLimit, NotFound from astakosclient.errors import QuotaLimit, NotFound
from snf_django.utils import reconcile
backend = get_backend() backend = get_backend()
RESOURCES = ['pithos.diskspace']
class Command(NoArgsCommand): class Command(NoArgsCommand):
...@@ -57,6 +58,8 @@ class Command(NoArgsCommand): ...@@ -57,6 +58,8 @@ class Command(NoArgsCommand):
make_option("--userid", dest="userid", make_option("--userid", dest="userid",
default=None, default=None,
help="Reconcile resources only for this user"), help="Reconcile resources only for this user"),
make_option("--project",
help="Reconcile resources only for this project"),
make_option("--fix", dest="fix", make_option("--fix", dest="fix",
default=False, default=False,
action="store_true", action="store_true",
...@@ -69,103 +72,78 @@ class Command(NoArgsCommand): ...@@ -69,103 +72,78 @@ class Command(NoArgsCommand):
) )
def handle_noargs(self, **options): def handle_noargs(self, **options):
write = self.stdout.write
try: try:
backend.pre_exec() backend.pre_exec()
userid = options['userid'] userid = options['userid']
project = options['project']
# Get holding from Pithos DB # Get holding from Pithos DB
db_usage = backend.node.node_account_usage(userid) db_usage = backend.node.node_account_usage(userid, project)
db_project_usage = backend.node.node_project_usage(project)
users = set(db_usage.keys()) users = set(db_usage.keys())
if userid and userid not in users: if userid and userid not in users:
if backend._lookup_account(userid) is None: if backend._lookup_account(userid) is None:
self.stdout.write("User '%s' does not exist in DB!\n" % write("User '%s' does not exist in DB!\n" % userid)
userid)
return return
# Get holding from Quotaholder # Get holding from Quotaholder
try: try:
qh_result = backend.astakosclient.service_get_quotas(userid) qh_result = backend.astakosclient.service_get_quotas(userid)
except NotFound: except NotFound:
self.stdout.write( write("User '%s' does not exist in Quotaholder!\n" % userid)
"User '%s' does not exist in Quotaholder!\n" % userid)
return return
users.update(qh_result.keys()) try:
qh_project_result = \
pending_exists = False backend.astakosclient.service_get_project_quotas(project)
unknown_user_exists = False except NotFound:
unsynced = [] write("Project '%s' does not exist in Quotaholder!\n" %
for uuid in users: project)
db_value = db_usage.get(uuid, 0)
try: unsynced_users, users_pending, users_unknown =\
qh_all = qh_result[uuid] reconcile.check_users(self.stderr, RESOURCES,
except KeyError: db_usage, qh_result)
self.stdout.write(
"User '%s' does not exist in Quotaholder!\n" % uuid) unsynced_projects, projects_pending, projects_unknown =\
unknown_user_exists = True reconcile.check_projects(self.stderr, RESOURCES,
continue db_project_usage, qh_project_result)
else: pending_exists = users_pending or projects_pending
qh = qh_all.get(DEFAULT_SOURCE, {}) unknown_exists = users_unknown or projects_unknown
for resource in [r['name'] for r in resources]:
try: headers = ("Type", "Holder", "Source", "Resource",
qh_resource = qh[resource] "Database", "Quotaholder")
except KeyError: unsynced = unsynced_users + unsynced_projects
self.stdout.write(
"Resource '%s' does not exist in Quotaholder "
"for user '%s'!\n" % (resource, uuid))
continue
if qh_resource['pending']:
self.stdout.write(
"Pending commission. "
"User '%s', resource '%s'.\n" %
(uuid, resource))
pending_exists = True
continue
qh_value = qh_resource['usage']
if db_value != qh_value:
data = (uuid, resource, db_value, qh_value)
unsynced.append(data)
if unsynced: if unsynced:
headers = ("User", "Resource", "Database", "Quotaholder")
utils.pprint_table(self.stdout, unsynced, headers) utils.pprint_table(self.stdout, unsynced, headers)
if options['fix']: if options["fix"]:
request = {} force = options["force"]
request['force'] = options['force'] name = ("client: reconcile-resources-pithos, time: %s"
request['auto_accept'] = True % datetime.now())
request['name'] = "RECONCILE" user_provisions = reconcile.create_user_provisions(
request['provisions'] = map(create_provision, unsynced) unsynced_users)
project_provisions = reconcile.create_project_provisions(
unsynced_projects)
try: try:
backend.astakosclient.issue_commission(request) backend.astakosclient.issue_commission_generic(
user_provisions, project_provisions, name=name,
force=force, auto_accept=True)
except QuotaLimit: except QuotaLimit:
self.stdout.write( write("Reconciling failed because a limit has been "
"Reconciling failed because a limit has been " "reached. Use --force to ignore the check.\n")
"reached. Use --force to ignore the check.\n")
return return
self.stdout.write("Fixed unsynced resources\n") write("Fixed unsynced resources\n")
if pending_exists: if pending_exists:
self.stdout.write( write("Found pending commissions. Run 'snf-manage"
"Found pending commissions. Run 'snf-manage" " reconcile-commissions-pithos'\n")
" reconcile-commissions-pithos'\n") elif not (unsynced or unknown_exists):
elif not (unsynced or unknown_user_exists): write("Everything in sync.\n")
self.stdout.write("Everything in sync.\n")
except BaseException as e: except BaseException as e:
backend.post_exec(False) backend.post_exec(False)
self.stdout.write(str(e) + "\n") raise CommandError(e)
else: else:
backend.post_exec(True) backend.post_exec(True)
finally: finally:
backend.close() backend.close()
def create_provision(provision_info):
user, resource, db_value, qh_value = provision_info
return {"holder": user,
"source": DEFAULT_SOURCE,
"resource": resource,
"quantity": int(db_value - qh_value)}
# Copyright 2011-2012 GRNET S.A. All rights reserved. # Copyright 2011, 2012, 2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
from time import time from time import time
from operator import itemgetter from operator import itemgetter
from itertools import groupby from itertools import groupby
from collections import defaultdict
from sqlalchemy import (Table, Integer, BigInteger, DECIMAL, Boolean, from sqlalchemy import (Table, Integer, BigInteger, DECIMAL, Boolean,
Column, String, MetaData, ForeignKey) Column, String, MetaData, ForeignKey)
...@@ -45,7 +46,7 @@ from dbworker import DBWorker, ESCAPE_CHAR ...@@ -45,7 +46,7 @@ from dbworker import DBWorker, ESCAPE_CHAR
from pithos.backends.filter import parse_filters from pithos.backends.filter import parse_filters
DEFAULT_DISKSPACE_RESOURCE = 'pithos.diskspace'
ROOTNODE = 0 ROOTNODE = 0
(SERIAL, NODE, HASH, SIZE, TYPE, SOURCE, MTIME, MUSER, UUID, CHECKSUM, (SERIAL, NODE, HASH, SIZE, TYPE, SOURCE, MTIME, MUSER, UUID, CHECKSUM,
...@@ -499,11 +500,12 @@ class Node(DBWorker): ...@@ -499,11 +500,12 @@ class Node(DBWorker):
r.close() r.close()
return dict(rows) return dict(rows)
def node_account_usage(self, account=None, cluster=0): def node_account_usage(self, account=None, project=None, cluster=0):
"""Return usage for a specific account. """Return a dict of dicts with the project usage for a specific account.
Keyword arguments: Keyword arguments:
account -- (default None: list usage for all the accounts) account -- (default None: list usage for all accounts)
project -- (default None: list usage for all projects)
cluster -- list current, history or deleted usage (default 0: normal) cluster -- list current, history or deleted usage (default 0: normal)
""" """
...@@ -511,20 +513,61 @@ class Node(DBWorker): ...@@ -511,20 +513,61 @@ class Node(DBWorker):
n2 = self.nodes.alias('n2') n2 = self.nodes.alias('n2')
n3 = self.nodes.alias('n3') n3 = self.nodes.alias('n3')
s = select([n3.c.path, func.sum(self.versions.c.size)]) s = select([n3.c.path, self.policy.c.value,
func.sum(self.versions.c.size)])
s = s.where(self.policy.c.key == 'project')
s = s.where(self.policy.c.node == n2.c.node)
s = s.where(n1.c.node == self.versions.c.node) s = s.where(n1.c.node == self.versions.c.node)
s = s.where(self.versions.c.cluster == cluster) s = s.where(self.versions.c.cluster == cluster)
s = s.where(n1.c.parent == n2.c.node) s = s.where(n1.c.parent == n2.c.node)
s = s.where(n2.c.parent == n3.c.node) s = s.where(n2.c.parent == n3.c.node)
s = s.where(n3.c.parent == 0) s = s.where(n3.c.parent == 0)
s = s.where(n3.c.node != 0) s = s.where(n3.c.node != 0)
s = s.group_by(n3.c.path, self.policy.c.value)
if account: if account:
s = s.where(n3.c.path == account) s = s.where(n3.c.path == account)
s = s.group_by(n3.c.path) if project:
s = s.where(self.policy.c.value == project)
r = self.conn.execute(s) r = self.conn.execute(s)
usage = r.fetchall() rows = r.fetchall()
r.close() r.close()
return dict(usage) d = defaultdict(dict)
for account, project, usage in rows:
d[account][project][DEFAULT_DISKSPACE_RESOURCE] = usage
return d
def node_project_usage(self, project=None, cluster=0):
"""Return a dict of dicts with the project usage for a specific account.
Keyword arguments:
project -- (default None: list usage for all projects)
cluster -- list current, history or deleted usage (default 0: normal)
"""
n1 = self.nodes.alias('n1')
n2 = self.nodes.alias('n2')
n3 = self.nodes.alias('n3')
s = select([self.policy.c.value,
func.sum(self.versions.c.size)])
s = s.where(self.policy.c.key == 'project')
s = s.where(self.policy.c.node == n2.c.node)
s = s.where(n1.c.node == self.versions.c.node)
s = s.where(self.versions.c.cluster == cluster)
s = s.where(n1.c.parent == n2.c.node)
s = s.where(n2.c.parent == n3.c.node)
# s = s.where(n3.c.parent == 0)
# s = s.where(n3.c.node != 0)
s = s.group_by(self.policy.c.value)
if project:
s = s.where(self.policy.c.value == project)
r = self.conn.execute(s)
rows = r.fetchall()
r.close()
d = defaultdict(dict)
for project, usage in rows:
d[project][DEFAULT_DISKSPACE_RESOURCE] = usage
return d
def policy_get(self, node): def policy_get(self, node):
s = select([self.policy.c.key, self.policy.c.value], s = select([self.policy.c.key, self.policy.c.value],
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment