Commit ae2ad61f authored by Giorgos Korfiatis's avatar Giorgos Korfiatis

astakos: Improve printing in quota-verify

Also add option to delete project and user holdings that do not correspond
to actual astakos entities.
parent 42304537
...@@ -34,75 +34,104 @@ ...@@ -34,75 +34,104 @@
from optparse import make_option from optparse import make_option
from django.db import transaction from django.db import transaction
from import AstakosUser from import Project
from import ( from import quotas
qh_sync_users_diffs,) from import pprint_table
from import get_user_by_uuid
from import SynnefoCommand from import SynnefoCommand
from import _common as common
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def differences(local_quotas, qh_quotas):
unsynced = []
unexpected = []
for holder, h_quotas in local_quotas.iteritems():
qh_h_quotas = qh_quotas.pop(holder, {})
for source, s_quotas in h_quotas.iteritems():
qh_s_quotas = qh_h_quotas.pop(source, {})
for resource, value in s_quotas.iteritems():
qh_value = qh_s_quotas.pop(resource, None)
if value != qh_value:
data = (holder, source, resource, value, qh_value)
unexpected += unexpected_resources(holder, source, qh_s_quotas)
unexpected += unexpected_sources(holder, qh_h_quotas)
unexpected += unexpected_holders(qh_quotas)
return unsynced, unexpected
def unexpected_holders(qh_quotas):
unexpected = []
for holder, qh_h_quotas in qh_quotas.iteritems():
unexpected += unexpected_sources(holder, qh_h_quotas)
return unexpected
def unexpected_sources(holder, qh_h_quotas):
unexpected = []
for source, qh_s_quotas in qh_h_quotas.iteritems():
unexpected += unexpected_resources(holder, source, qh_s_quotas)
return unexpected
def unexpected_resources(holder, source, qh_s_quotas):
unexpected = []
for resource, qh_value in qh_s_quotas.iteritems():
data = (holder, source, resource, None, qh_value)
return unexpected
class Command(SynnefoCommand): class Command(SynnefoCommand):
help = "Check the integrity of user quota" help = "Check the integrity of user and project quota"
option_list = SynnefoCommand.option_list + ( option_list = SynnefoCommand.option_list + (
make_option('--sync', make_option("--include-unexpected-holdings",
action='store_true', default=False,
dest='sync', action="store_true",
help=("Also check for holdings that do not correspond "
"to Astakos projects or user. Note that fixing such "
"inconsistencies will permanently delete these "
make_option("--fix", dest="fix",
default=False, default=False,
help="Sync quotaholder"), action="store_true",
make_option('--user', help="Synchronize Quotaholder with Astakos DB."),
metavar='<uuid or email>',
help="Check for a specified user"),
) )
@transaction.commit_on_success @transaction.commit_on_success
def handle(self, *args, **options): def handle(self, *args, **options):
sync = options['sync'] write = self.stderr.write
user_ident = options['user'] fix = options['fix']
check_unexpected = options["include_unexpected_holdings"]
if user_ident is not None:
users = [common.get_accepted_user(user_ident)] projects = Project.objects.all()
else: local_proj_quotas, local_user_quotas = \
users = AstakosUser.objects.accepted() quotas.astakos_project_quotas(projects)
qh_proj_quotas, qh_user_quotas = \
qh_limits, diff_q = qh_sync_users_diffs(users, sync=sync) quotas.get_projects_quota_limits()
if sync: unsynced, unexpected = differences(local_proj_quotas, qh_proj_quotas)
self.print_sync(diff_q) unsync_u, unexpect_u = differences(local_user_quotas, qh_user_quotas)
else: unsynced += unsync_u
self.print_verify(qh_limits, diff_q) unexpected += unexpect_u
def print_sync(self, diff_quotas): headers = ("Holder", "Source", "Resource", "Astakos", "Quotaholder")
size = len(diff_quotas) if not unsynced and (not check_unexpected or not unexpected):
if size == 0: write("Everything in sync.\n")
self.stderr.write("No sync needed.\n") return
self.stderr.write("Synced %s users:\n" % size) printable = (unsynced if not check_unexpected
uuids = diff_quotas.keys() else unsynced + unexpected)
users = AstakosUser.objects.filter(uuid__in=uuids) pprint_table(self.stdout, printable, headers, title="Inconsistencies")
for user in users: if fix:
self.stderr.write("%s (%s)\n" % (user.uuid, user.username)) to_sync = []
for holder, source, resource, value, qh_value in unsynced:
def print_verify(self, qh_limits, diff_quotas): to_sync.append(((holder, source, resource), value))
for holder, local in diff_quotas.iteritems(): quotas.qh.set_quota(to_sync)
registered = qh_limits.pop(holder, None)
user = get_user_by_uuid(holder) if check_unexpected:
if registered is None: to_del = []
self.stderr.write( for holder, source, resource, value, qh_value in unexpected:
"No quota for %s (%s) in quotaholder.\n" % to_del.append((holder, source, resource))
(holder, user.username)) quotas.qh.delete_quota(to_del)
self.stdout.write("Quota differ for %s (%s):\n" %
(holder, user.username))
self.stdout.write("Quota according to quotaholder:\n")
self.stdout.write("%s\n" % (registered))
self.stdout.write("Quota according to astakos:\n")
self.stdout.write("%s\n\n" % (local))
diffs = len(diff_quotas)
if diffs:
self.stderr.write("Quota differ for %d users.\n" % (diffs))
...@@ -320,31 +320,6 @@ def qh_sync_membership(membership): ...@@ -320,31 +320,6 @@ def qh_sync_membership(membership):
set_quota(quota) set_quota(quota)
def compute_diff_quota(local, registered):
diff = {}
for holder, h_quota in local.iteritems():
for source, value in h_quota.iteritems():
reg_h_quota = registered.get(holder, {})
reg_value = reg_h_quota.get(source, {})
if value != reg_value:
set_path(diff, [holder, source], value, createpath=True)
return diff
def qh_sync_projects_diffs(projects, sync=True):
local_project, local_user = astakos_project_quotas(projects)
registered_project, registered_user = get_projects_quota_limits(projects)
diff_quotas = compute_diff_quota(local_project, registered_project)
diff_quotas.update(compute_diff_quota(local_user, registered_user))
if sync:
all_registered = registered_project
return all_registered, diff_quotas
def pick_limit_scheme(project, resource): def pick_limit_scheme(project, resource):
return resource.uplimit if project.is_base else resource.project_default return resource.uplimit if project.is_base else resource.project_default
...@@ -75,6 +75,13 @@ def get_quota(holders=None, sources=None, resources=None, flt=None): ...@@ -75,6 +75,13 @@ def get_quota(holders=None, sources=None, resources=None, flt=None):
return quotas return quotas
def delete_quota(keys):
for holder, source, resource in keys:
def _get_holdings_for_update(holding_keys, resource=None, delete=False): def _get_holdings_for_update(holding_keys, resource=None, delete=False):
flt = Q(resource=resource) if resource is not None else Q() flt = Q(resource=resource) if resource is not None else Q()
holders = set(holder for (holder, source, resource) in holding_keys) holders = set(holder for (holder, source, resource) in holding_keys)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment