quotas.py 11.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

34
from astakos.im.models import (
35
    Resource, AstakosUserQuota, AstakosUser, Service,
36
    Project, ProjectMembership, ProjectResourceQuota)
37
import astakos.quotaholder_app.callpoint as qh
38
from astakos.quotaholder_app.exception import NoCapacityError
39
from django.db.models import Q
40
from synnefo.util.keypath import set_path
41 42


43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
PROJECT_TAG = "project:"
USER_TAG = "user:"

def project_ref(value):
    return  PROJECT_TAG + value


def get_project_ref(project):
    return project_ref(project.uuid)


def user_ref(value):
    return USER_TAG + value


def get_user_ref(user):
    return user_ref(user.uuid)


def from_holding(holding, is_project=False):
63
    limit, usage_min, usage_max = holding
64 65 66 67
    prefix = 'project_' if is_project else ''
    body = {prefix+'limit':   limit,
            prefix+'usage':   usage_max,
            prefix+'pending': usage_max-usage_min,
68 69 70 71
            }
    return body


72 73 74 75 76 77
def get_user_counters(users, resources=None, sources=None, flt=None):
    holders = [get_user_ref(user) for user in users]
    return qh.get_quota(holders=holders,
                        resources=resources,
                        sources=sources,
                        flt=flt)
78 79


80 81 82 83 84
def get_project_counters(projects, resources=None, sources=None):
    holders = [get_project_ref(project) for project in projects]
    return qh.get_quota(holders=holders,
                        resources=resources,
                        sources=sources)
85 86


87 88 89 90 91 92 93 94 95 96
def strip_names(counters):
    stripped = {}
    for ((holder, source, resource), value) in counters.iteritems():
        prefix, sep, holder = holder.partition(":")
        assert prefix in ["user", "project"]
        if source is not None:
            prefix, sep, source = source.partition(":")
            assert prefix == "project"
        stripped[(holder, source, resource)] = value
    return stripped
97 98


99 100 101 102 103
def get_related_sources(counters):
    projects = set()
    for (holder, source, resource) in counters.iterkeys():
        projects.add(source)
    return list(projects)
104 105


106 107 108 109 110 111 112 113 114 115
def mk_quota_dict(users_counters, project_counters):
    quota = {}
    for (holder, source, resource), u_value in users_counters.iteritems():
        p_value = project_counters[(source, None, resource)]
        values_dict = from_holding(u_value)
        values_dict.update(from_holding(p_value, is_project=True))
        set_path(quota, [holder, source, resource], values_dict,
                 createpath=True)
    return quota

116

117 118 119 120 121
def get_users_quotas_counters(users, resources=None, sources=None, flt=None):
    user_counters = get_user_counters(users, resources, sources, flt=flt)
    projects = get_related_sources(user_counters)
    project_counters = qh.get_quota(holders=projects, resources=resources)
    return strip_names(user_counters), strip_names(project_counters)
122

123 124 125 126

def get_users_quotas(users, resources=None, sources=None, flt=None):
    u_c, p_c = get_users_quotas_counters(users, resources, sources, flt=flt)
    return mk_quota_dict(u_c, p_c)
127 128 129 130


def get_user_quotas(user, resources=None, sources=None):
    quotas = get_users_quotas([user], resources, sources)
131
    return quotas.get(user.uuid, {})
132 133


134
def service_get_quotas(component, users=None):
135 136 137 138
    name_values = Service.objects.filter(
        component=component).values_list('name')
    service_names = [t for (t,) in name_values]
    resources = Resource.objects.filter(service_origin__in=service_names)
139
    resource_names = [r.name for r in resources]
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
    astakosusers = AstakosUser.objects.verified()
    if users is not None:
        astakosusers = astakosusers.filter(uuid__in=users)
    return get_users_quotas(astakosusers, resources=resource_names)


def mk_limits_dict(counters):
    quota = {}
    for key, (limit, _, _) in counters.iteritems():
        path = list(key)
        set_path(quota, path, limit, createpath=True)
    return quota


def mk_project_quota_dict(project_counters):
    quota = {}
    for (holder, _, resource), p_value in project_counters.iteritems():
        values_dict = from_holding(p_value, is_project=True)
        set_path(quota, [holder, resource], values_dict,
                 createpath=True)
    return quota


def get_projects_quota(projects, resources=None, sources=None):
    project_counters = get_project_counters(projects, resources, sources)
    return mk_project_quota_dict(strip_names(project_counters))


def get_project_quota(project, resources=None, sources=None):
    quotas = get_projects_quota([project], resources, sources)
    return quotas.get(project.uuid, {})


def get_projects_quota_limits():
    project_counters = qh.get_quota(flt=Q(holder__startswith=PROJECT_TAG))
    user_counters = qh.get_quota(flt=Q(holder__startswith=USER_TAG))
    return mk_limits_dict(project_counters), mk_limits_dict(user_counters)
177 178


179 180 181 182 183 184 185 186 187 188
def _level_quota_dict(quotas):
    lst = []
    for holder, holder_quota in quotas.iteritems():
        for source, source_quota in holder_quota.iteritems():
            for resource, limit in source_quota.iteritems():
                key = (holder, source, resource)
                lst.append((key, limit))
    return lst


189
def set_quota(quotas, resource=None):
190
    q = _level_quota_dict(quotas)
191
    qh.set_quota(q, resource=resource)
192 193


194
PENDING_APP_RESOURCE = 'astakos.pending_app'
195 196


197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
def mk_user_provision(user, source, resource, quantity):
    holder = user_ref(user)
    source = project_ref(source)
    return (holder, source, resource), quantity


def mk_project_provision(project, resource, quantity):
    holder = project_ref(project)
    return (holder, None, resource), quantity


def _mk_provisions(holder, source, resource, quantity):
    return [((holder, source, resource), quantity),
            ((source, None, resource), quantity)]


def register_pending_apps(user, project, quantity, force=False):
    provisions = _mk_provisions(get_user_ref(user), get_project_ref(project),
                                PENDING_APP_RESOURCE, quantity)
216 217 218
    try:
        s = qh.issue_commission(clientkey='astakos',
                                force=force,
219
                                provisions=provisions)
220 221 222
    except NoCapacityError as e:
        limit = e.data['limit']
        return False, limit
223
    qh.resolve_pending_commission('astakos', s)
224
    return True, None
225 226


227 228
def get_pending_app_quota(user):
    quota = get_user_quotas(user)
229 230
    source = user.base_project.uuid
    return quota[source][PENDING_APP_RESOURCE]
231 232


233 234 235 236 237
def update_base_quota(users, resource, value):
    userids = [user.pk for user in users]
    AstakosUserQuota.objects.\
        filter(resource__name=resource, user__pk__in=userids).\
        update(capacity=value)
238
    qh_sync_locked_users(users, resource=resource)
239

240

241 242 243 244 245 246 247 248 249 250
def _partition_by(f, l):
    d = {}
    for x in l:
        group = f(x)
        group_l = d.get(group, [])
        group_l.append(x)
        d[group] = group_l
    return d


251
def astakos_project_quotas(projects, resource=None):
252
    objs = ProjectResourceQuota.objects.select_related()
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
    flt = Q(resource__name=resource) if resource is not None else Q()
    grants = objs.filter(project__in=projects).filter(flt)
    grants_d = _partition_by(lambda g: g.project_id, grants)

    objs = ProjectMembership.objects
    memberships = objs.initialized(projects).select_related(
        "person", "project")
    memberships_d = _partition_by(lambda m: m.project_id, memberships)

    user_quota = {}
    project_quota = {}

    for project in projects:
        pr_ref = get_project_ref(project)
        state = project.state
        if state not in Project.INITIALIZED_STATES:
            continue

        project_grants = grants_d.get(project.id, [])
        project_memberships = memberships_d.get(project.id, [])
        for grant in project_grants:
            resource = grant.resource.name
            path = [pr_ref, None, resource]
            val = grant.project_capacity if state == Project.NORMAL else 0
            set_path(project_quota, path, val, createpath=True)
            for membership in project_memberships:
                u_ref = get_user_ref(membership.person)
                path = [u_ref, pr_ref, resource]
                val = grant.member_capacity if membership.is_active() else 0
                set_path(user_quota, path, val, createpath=True)

    return project_quota, user_quota


def list_user_quotas(users, qhflt=None):
288
    qh_quotas = get_users_quotas(users, flt=qhflt)
289
    return qh_quotas
290 291


292 293 294 295
def qh_sync_projects(projects, resource=None):
    p_quota, u_quota = astakos_project_quotas(projects, resource=resource)
    p_quota.update(u_quota)
    set_quota(p_quota, resource=resource)
296 297


298 299
def qh_sync_project(project):
    qh_sync_projects([project])
300

301

302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
def membership_quota(membership):
    project = membership.project
    pr_ref = get_project_ref(project)
    u_ref = get_user_ref(membership.person)
    objs = ProjectResourceQuota.objects.select_related()
    grants = objs.filter(project=project)
    user_quota = {}
    is_active = membership.is_active()
    for grant in grants:
        resource = grant.resource.name
        path = [u_ref, pr_ref, resource]
        value = grant.member_capacity if is_active else 0
        set_path(user_quota, path, value, createpath=True)
    return user_quota


def qh_sync_membership(membership):
    quota = membership_quota(membership)
    set_quota(quota)


323 324 325 326
def pick_limit_scheme(project, resource):
    return resource.uplimit if project.is_base else resource.project_default


327
def qh_sync_new_resource(resource):
328 329
    projects = Project.objects.filter(state__in=Project.INITIALIZED_STATES).\
        select_for_update()
330

331
    entries = []
332 333
    for project in projects:
        limit = pick_limit_scheme(project, resource)
334
        entries.append(
335 336 337 338 339 340 341 342
            ProjectResourceQuota(
                project=project,
                resource=resource,
                project_capacity=limit,
                member_capacity=limit))

    ProjectResourceQuota.objects.bulk_create(entries)
    qh_sync_projects(projects, resource=resource.name)