Commit 15e08fdf authored by Giorgos Korfiatis's avatar Giorgos Korfiatis
Browse files

Sync quotas efficiently when importing resources

When changing the default base quota (uplimit) for a preexisting resource,
do a bulk update of the related holdings.
When importing a new resource, create the new related holdings.

Prompt the admin for the uplimit in the management command, if a config
file is not given.
parent a125b747
...@@ -38,7 +38,7 @@ from django.db.utils import IntegrityError ...@@ -38,7 +38,7 @@ from django.db.utils import IntegrityError
from django.utils import simplejson as json from django.utils import simplejson as json
from synnefo.lib.db.transaction import commit_on_success_strict from synnefo.lib.db.transaction import commit_on_success_strict
from astakos.im.resources import add_resources from astakos.im.resources import add_resource
class Command(BaseCommand): class Command(BaseCommand):
...@@ -50,33 +50,66 @@ class Command(BaseCommand): ...@@ -50,33 +50,66 @@ class Command(BaseCommand):
dest='json', dest='json',
metavar='<json.file>', metavar='<json.file>',
help="Load resource info from a json file"), help="Load resource info from a json file"),
make_option('--service',
dest='service_id',
metavar='<service_id>',
help=("Automatically load resource info for a given "
"service")),
make_option('--conf', make_option('--conf',
dest='conf', dest='conf',
metavar='<conf.json>', metavar='<conf.json>',
help="Limit configuration file"), help="Limit configuration file"),
) )
@commit_on_success_strict()
def handle(self, *args, **options): def handle(self, *args, **options):
config = {}
conf_file = options['conf'] conf_file = options['conf']
if not conf_file: if conf_file is not None:
m = "Please provide a configuation file." with open(conf_file) as file_data:
raise CommandError(m) config = json.load(file_data)
with open(conf_file) as file_data:
config = json.load(file_data)
json_file = options['json'] json_file = options['json']
service_id = options['service_id']
if bool(json_file) == bool(service_id):
m = "Please provide either --service or --json option."
raise CommandError(m)
if service_id:
raise NotImplementedError()
if json_file: if json_file:
with open(json_file) as file_data: with open(json_file) as file_data:
data = json.load(file_data) data = json.load(file_data)
service = data.get('service')
resources = data.get('resources')
if service is None or resources is None:
m = "JSON file should contain service and resource data."
raise CommandError(m)
self.add_resources(service, resources, config)
@commit_on_success_strict()
def add_resources(self, service, resources, config):
for resource in resources:
name = resource['name']
uplimit = config.get(name)
if uplimit is None:
desc = resource['desc']
unit = resource.get('unit')
self.stdout.write(
"Provide default base quota for resource '%s' (%s)" %
(name, desc))
m = (" in %s: " % unit) if unit else ": "
self.stdout.write(m)
uplimit = raw_input()
service = data.get('service') try:
resources = data.get('resources') uplimit = int(uplimit)
if service is None or resources is None: except ValueError:
m = "JSON file should contain service and resource data." m = "Limit for resource %s is not an integer." % (name)
raise CommandError(m) raise CommandError(m)
add_resources(service, resources, config) add_resource(service, resource, uplimit)
...@@ -236,3 +236,21 @@ def sync_users(users, sync=True): ...@@ -236,3 +236,21 @@ def sync_users(users, sync=True):
def sync_all_users(sync=True): def sync_all_users(sync=True):
users = AstakosUser.objects.verified() users = AstakosUser.objects.verified()
return sync_users(users, sync) return sync_users(users, sync)
def qh_add_resource_limit(resource, diff):
users = AstakosUser.forupdate.all().select_for_update()
qh.add_resource_limit(SYSTEM, resource, diff)
def qh_sync_new_resource(resource, limit):
users = AstakosUser.forupdate.filter(
email_verified=True).select_for_update()
data = []
for user in users:
uuid = user.uuid
key = uuid, SYSTEM, resource
data.append((key, limit))
qh.set_quota(data)
...@@ -33,35 +33,41 @@ ...@@ -33,35 +33,41 @@
from astakos.im.models import Service, Resource from astakos.im.models import Service, Resource
from astakos.im.functions import qh_sync_all_users from astakos.im.functions import qh_sync_all_users
from astakos.im.quotas import qh_add_resource_limit, qh_sync_new_resource
import logging
logger = logging.getLogger(__name__)
def add_resources(service, resources, conf):
def add_resource(service, resource, uplimit):
try: try:
s = Service.objects.get(name=service) s = Service.objects.get(name=service)
except Service.DoesNotExist: except Service.DoesNotExist:
raise Exception("Service %s is not registered." % (service)) raise Exception("Service %s is not registered." % (service))
names = [resource['name'] for resource in resources] name = resource['name']
rs = Resource.objects.filter(name__in=names).select_for_update() try:
rs = dict((r.name, r) for r in rs) r = Resource.objects.get_for_update(name=name)
old_uplimit = r.uplimit
for resource in resources: except Resource.DoesNotExist:
name = resource['name'] r = Resource()
existing = rs.get(name) old_uplimit = None
r = existing if existing is not None else Resource()
uplimit = conf.get(name) r.uplimit = uplimit
if uplimit is None: r.service = s
raise Exception("Limit for resource %s is missing." % (name)) for key, value in resource.iteritems():
setattr(r, key, value)
if not isinstance(uplimit, (int, long)): r.save()
raise Exception("Limit for resource %s is not an integer." %
(name))
r.uplimit = uplimit if old_uplimit is not None:
r.service = s logger.info("Updated resource %s with limit %s." % (name, uplimit))
for key, value in resource.iteritems(): else:
setattr(r, key, value) logger.info("Added resource %s with limit %s." % (name, uplimit))
r.save() if old_uplimit is not None:
qh_sync_all_users() diff = uplimit - old_uplimit
if diff != 0:
qh_add_resource_limit(name, diff)
else:
qh_sync_new_resource(name, uplimit)
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.db.models import F
from astakos.quotaholder.exception import ( from astakos.quotaholder.exception import (
QuotaholderError, QuotaholderError,
NoCommissionError, NoCommissionError,
...@@ -131,6 +132,10 @@ class QuotaholderDjangoDBCallpoint(object): ...@@ -131,6 +132,10 @@ class QuotaholderDjangoDBCallpoint(object):
h.save() h.save()
holdings[key] = h holdings[key] = h
def add_resource_limit(self, source, resource, diff):
objs = Holding.objects.filter(source=source, resource=resource)
objs.update(limit=F('limit')+diff)
def issue_commission(self, def issue_commission(self,
context=None, context=None,
clientkey=None, clientkey=None,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment