quotas.py 10.8 KB
Newer Older
1
# Copyright (C) 2010-2016 GRNET S.A.
2
#
Vangelis Koukis's avatar
Vangelis Koukis committed
3 4 5 6
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
7
#
Vangelis Koukis's avatar
Vangelis Koukis committed
8 9 10 11
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
12
#
Vangelis Koukis's avatar
Vangelis Koukis committed
13 14
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
15

16
from astakos.im.models import (
17
    Resource, AstakosUser, Service,
18
    Project, ProjectMembership, ProjectResourceQuota)
19
import astakos.quotaholder_app.callpoint as qh
20
from astakos.quotaholder_app.exception import NoCapacityError
21
from django.db.models import Q
22
from collections import defaultdict
23 24


25 26
QuotaDict = lambda: defaultdict(lambda: defaultdict(dict))

27 28 29
PROJECT_TAG = "project:"
USER_TAG = "user:"

Giorgos Korfiatis's avatar
Giorgos Korfiatis committed
30

31
def project_ref(value):
Giorgos Korfiatis's avatar
Giorgos Korfiatis committed
32
    return PROJECT_TAG + value
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47


def get_project_ref(project):
    return project_ref(project.uuid)


def user_ref(value):
    return USER_TAG + value


def get_user_ref(user):
    return user_ref(user.uuid)


def from_holding(holding, is_project=False):
48
    limit, usage_min, usage_max = holding
49 50 51 52
    prefix = 'project_' if is_project else ''
    body = {prefix+'limit':   limit,
            prefix+'usage':   usage_max,
            prefix+'pending': usage_max-usage_min,
53 54 55 56
            }
    return body


57 58 59 60 61 62
def get_user_counters(users, resources=None, sources=None, flt=None):
    holders = [get_user_ref(user) for user in users]
    return qh.get_quota(holders=holders,
                        resources=resources,
                        sources=sources,
                        flt=flt)
63 64


65 66 67 68 69
def get_project_counters(projects, resources=None, sources=None):
    holders = [get_project_ref(project) for project in projects]
    return qh.get_quota(holders=holders,
                        resources=resources,
                        sources=sources)
70 71


72 73 74 75 76 77 78 79 80 81
def strip_names(counters):
    stripped = {}
    for ((holder, source, resource), value) in counters.iteritems():
        prefix, sep, holder = holder.partition(":")
        assert prefix in ["user", "project"]
        if source is not None:
            prefix, sep, source = source.partition(":")
            assert prefix == "project"
        stripped[(holder, source, resource)] = value
    return stripped
82 83


84 85 86 87 88
def get_related_sources(counters):
    projects = set()
    for (holder, source, resource) in counters.iterkeys():
        projects.add(source)
    return list(projects)
89 90


91
def mk_quota_dict(users_counters, project_counters):
92
    quota = QuotaDict()
93 94 95 96
    for (holder, source, resource), u_value in users_counters.iteritems():
        p_value = project_counters[(source, None, resource)]
        values_dict = from_holding(u_value)
        values_dict.update(from_holding(p_value, is_project=True))
97
        quota[holder][source][resource] = values_dict
98 99
    return quota

100

101 102 103 104 105
def get_users_quotas_counters(users, resources=None, sources=None, flt=None):
    user_counters = get_user_counters(users, resources, sources, flt=flt)
    projects = get_related_sources(user_counters)
    project_counters = qh.get_quota(holders=projects, resources=resources)
    return strip_names(user_counters), strip_names(project_counters)
106

107 108 109 110

def get_users_quotas(users, resources=None, sources=None, flt=None):
    u_c, p_c = get_users_quotas_counters(users, resources, sources, flt=flt)
    return mk_quota_dict(u_c, p_c)
111 112 113 114


def get_user_quotas(user, resources=None, sources=None):
    quotas = get_users_quotas([user], resources, sources)
115
    return quotas.get(user.uuid, {})
116 117


118
def service_get_quotas(component, users=None, sources=None):
119 120 121 122
    name_values = Service.objects.filter(
        component=component).values_list('name')
    service_names = [t for (t,) in name_values]
    resources = Resource.objects.filter(service_origin__in=service_names)
123
    resource_names = [r.name for r in resources]
124 125 126
    astakosusers = AstakosUser.objects.verified()
    if users is not None:
        astakosusers = astakosusers.filter(uuid__in=users)
127 128 129 130
    if sources is not None:
        sources = [project_ref(s) for s in sources]
    return get_users_quotas(astakosusers, resources=resource_names,
                            sources=sources)
131 132 133


def mk_limits_dict(counters):
134 135 136
    quota = QuotaDict()
    for (holder, source, resource), (limit, _, _) in counters.iteritems():
        quota[holder][source][resource] = limit
137 138 139 140
    return quota


def mk_project_quota_dict(project_counters):
141
    quota = QuotaDict()
142 143
    for (holder, _, resource), p_value in project_counters.iteritems():
        values_dict = from_holding(p_value, is_project=True)
144
        quota[holder][resource] = values_dict
145 146 147 148 149 150 151 152
    return quota


def get_projects_quota(projects, resources=None, sources=None):
    project_counters = get_project_counters(projects, resources, sources)
    return mk_project_quota_dict(strip_names(project_counters))


153 154 155 156 157 158 159 160 161 162 163 164
def service_get_project_quotas(component, projects=None):
    name_values = Service.objects.filter(
        component=component).values_list('name')
    service_names = [t for (t,) in name_values]
    resources = Resource.objects.filter(service_origin__in=service_names)
    resource_names = [r.name for r in resources]
    ps = Project.objects.initialized()
    if projects is not None:
        ps = ps.filter(uuid__in=projects)
    return get_projects_quota(ps, resources=resource_names)


165 166 167 168 169 170 171 172 173
def get_project_quota(project, resources=None, sources=None):
    quotas = get_projects_quota([project], resources, sources)
    return quotas.get(project.uuid, {})


def get_projects_quota_limits():
    project_counters = qh.get_quota(flt=Q(holder__startswith=PROJECT_TAG))
    user_counters = qh.get_quota(flt=Q(holder__startswith=USER_TAG))
    return mk_limits_dict(project_counters), mk_limits_dict(user_counters)
174 175


176 177 178 179 180 181 182 183 184 185
def _level_quota_dict(quotas):
    lst = []
    for holder, holder_quota in quotas.iteritems():
        for source, source_quota in holder_quota.iteritems():
            for resource, limit in source_quota.iteritems():
                key = (holder, source, resource)
                lst.append((key, limit))
    return lst


186
def set_quota(quotas, resource=None):
187
    q = _level_quota_dict(quotas)
188
    qh.set_quota(q, resource=resource)
189 190


191
PENDING_APP_RESOURCE = 'astakos.pending_app'
192 193


194 195 196 197 198 199 200 201 202 203 204
def mk_user_provision(user, source, resource, quantity):
    holder = user_ref(user)
    source = project_ref(source)
    return (holder, source, resource), quantity


def mk_project_provision(project, resource, quantity):
    holder = project_ref(project)
    return (holder, None, resource), quantity


205 206 207 208 209 210
def _mk_provisions(values):
    provisions = []
    for (holder, source, resource, quantity) in values:
        provisions += [((holder, source, resource), quantity),
                       ((source, None, resource), quantity)]
    return provisions
211 212


213 214 215 216 217 218
def register_pending_apps(triples, force=False):
    values = [(get_user_ref(user), get_project_ref(project),
               PENDING_APP_RESOURCE, quantity)
              for (user, project, quantity) in triples]

    provisions = _mk_provisions(values)
219 220 221
    try:
        s = qh.issue_commission(clientkey='astakos',
                                force=force,
222
                                provisions=provisions)
223 224 225
    except NoCapacityError as e:
        limit = e.data['limit']
        return False, limit
226
    qh.resolve_pending_commission('astakos', s)
227
    return True, None
228 229


230 231
def get_pending_app_quota(user):
    quota = get_user_quotas(user)
232
    source = user.get_base_project().uuid
233
    return quota[source][PENDING_APP_RESOURCE]
234 235


236 237 238 239 240 241 242 243 244 245
def _partition_by(f, l):
    d = {}
    for x in l:
        group = f(x)
        group_l = d.get(group, [])
        group_l.append(x)
        d[group] = group_l
    return d


246
def astakos_project_quotas(projects, resource=None):
247
    objs = ProjectResourceQuota.objects.select_related()
248 249 250 251 252 253 254 255 256
    flt = Q(resource__name=resource) if resource is not None else Q()
    grants = objs.filter(project__in=projects).filter(flt)
    grants_d = _partition_by(lambda g: g.project_id, grants)

    objs = ProjectMembership.objects
    memberships = objs.initialized(projects).select_related(
        "person", "project")
    memberships_d = _partition_by(lambda m: m.project_id, memberships)

257 258
    user_quota = QuotaDict()
    project_quota = QuotaDict()
259 260 261 262 263 264 265 266 267 268 269 270

    for project in projects:
        pr_ref = get_project_ref(project)
        state = project.state
        if state not in Project.INITIALIZED_STATES:
            continue

        project_grants = grants_d.get(project.id, [])
        project_memberships = memberships_d.get(project.id, [])
        for grant in project_grants:
            resource = grant.resource.name
            val = grant.project_capacity if state == Project.NORMAL else 0
271
            project_quota[pr_ref][None][resource] = val
272 273 274
            for membership in project_memberships:
                u_ref = get_user_ref(membership.person)
                val = grant.member_capacity if membership.is_active() else 0
275
                user_quota[u_ref][pr_ref][resource] = val
276 277 278 279 280

    return project_quota, user_quota


def list_user_quotas(users, qhflt=None):
281
    qh_quotas = get_users_quotas(users, flt=qhflt)
282
    return qh_quotas
283 284


285 286 287 288
def qh_sync_projects(projects, resource=None):
    p_quota, u_quota = astakos_project_quotas(projects, resource=resource)
    p_quota.update(u_quota)
    set_quota(p_quota, resource=resource)
289 290


291 292
def qh_sync_project(project):
    qh_sync_projects([project])
293

294

295 296 297 298 299 300
def membership_quota(membership):
    project = membership.project
    pr_ref = get_project_ref(project)
    u_ref = get_user_ref(membership.person)
    objs = ProjectResourceQuota.objects.select_related()
    grants = objs.filter(project=project)
301
    user_quota = QuotaDict()
302 303 304 305
    is_active = membership.is_active()
    for grant in grants:
        resource = grant.resource.name
        value = grant.member_capacity if is_active else 0
306
        user_quota[u_ref][pr_ref][resource] = value
307 308 309 310 311 312 313 314
    return user_quota


def qh_sync_membership(membership):
    quota = membership_quota(membership)
    set_quota(quota)


315 316 317 318
def pick_limit_scheme(project, resource):
    return resource.uplimit if project.is_base else resource.project_default


319
def qh_sync_new_resource(resource):
320 321
    projects = Project.objects.filter(state__in=Project.INITIALIZED_STATES).\
        select_for_update()
322

323
    entries = []
324 325
    for project in projects:
        limit = pick_limit_scheme(project, resource)
326
        entries.append(
327 328 329 330 331 332 333
            ProjectResourceQuota(
                project=project,
                resource=resource,
                project_capacity=limit,
                member_capacity=limit))
    ProjectResourceQuota.objects.bulk_create(entries)
    qh_sync_projects(projects, resource=resource.name)