Commit 8d477a6d authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis

Merge pull request #28 from cstavr/feature-unicode-handling

This patch-set fixes a number of issues relative with handling unicode strings.
These commits try to avoid mixing bytestrings and unicode objects, by decoding
bytestrings as early as possible (Input) and encoding unicodes as late as
possible (Ouput). In the API the encoding that is used is always 'UTF-8'. In
the CLI the encoding that is used is the user's preferred encoding.

Besides unicode issues, this patch-set fixes handling of image metadata
(Plankton) which are views as HTTP headers. Since, image metadata must be valid
HTTP headers, metadata keys and values which contain user defined values must
be properly quoted and unquoted where needed.
parents 98838c30 490861c8
......@@ -48,6 +48,17 @@ identity credentials.
<http://docs.openstack.org/developer/glance/glanceapi.html#authentication>`_,
with the only difference being the suggested identity manager.
Image Metadata Format
---------------------
In Cyclades Image API all image metadata are viewed as HTTP headers that are
starting with the `x-image-meta-` prefix. All metadata must be encoded with the
`UTF-8` encoding. Since the image metadata must be valid HTTP headers, user
defined metadata like the image's name or properties must also be properly
quoted. Finally, image properties that are viewed as HTTP headers and are
starting with the `x-image-meta-property-` prefix, are not case-sensitive and
all punctuation characters will be replaced with underscore.
List Available Images
---------------------
......
......@@ -42,7 +42,8 @@ from astakos.api.util import json_response
from snf_django.lib import api
from snf_django.lib.api import faults
from .util import user_from_token, invert_dict, read_json_body
from snf_django.lib.api import utils
from .util import user_from_token, invert_dict, check_is_dict
from astakos.im import functions
from astakos.im.models import (
......@@ -319,8 +320,7 @@ def _get_projects(query, mode="default", request_user=None):
@transaction.commit_on_success
def create_project(request):
user = request.user
data = request.body
app_data = json.loads(data)
app_data = utils.get_json_body(request)
return submit_new_project(app_data, user)
......@@ -357,8 +357,7 @@ def _get_project(project_id, request_user=None):
@transaction.commit_on_success
def modify_project(request, project_id):
user = request.user
data = request.body
app_data = json.loads(data)
app_data = utils.get_json_body(request)
return submit_modification(app_data, user, project_id=project_id)
......@@ -548,6 +547,7 @@ def submit_modification(app_data, user, project_id):
def get_action(actions, input_data):
action = None
data = None
check_is_dict(input_data)
for option in actions.keys():
if option in input_data:
if action:
......@@ -586,8 +586,7 @@ APP_ACTION_FUNCS = APPLICATION_ACTION.values()
@transaction.commit_on_success
def project_action(request, project_id):
user = request.user
data = request.body
input_data = json.loads(data)
input_data = utils.get_json_body(request)
func, action_data = get_action(PROJECT_ACTION, input_data)
with ExceptionHandler():
......@@ -707,7 +706,7 @@ MEMBERSHIP_ACTION = {
@transaction.commit_on_success
def membership_action(request, memb_id):
user = request.user
input_data = read_json_body(request, default={})
input_data = utils.get_json_body(request)
func, action_data = get_action(MEMBERSHIP_ACTION, input_data)
with ExceptionHandler():
func(memb_id, user, reason=action_data)
......
......@@ -31,13 +31,13 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.utils import simplejson as json
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from django.db import transaction
from snf_django.lib import api
from snf_django.lib.api.faults import BadRequest, ItemNotFound
from snf_django.lib.api import utils
from django.core.cache import cache
from astakos.im import settings
......@@ -48,7 +48,7 @@ from astakos.im.quotas import get_user_quotas, service_get_quotas, \
import astakos.quotaholder_app.exception as qh_exception
import astakos.quotaholder_app.callpoint as qh
from .util import (json_response, is_integer, are_integer,
from .util import (json_response, is_integer, are_integer, check_is_dict,
user_from_token, component_from_token)
......@@ -120,7 +120,7 @@ def commissions(request):
@api.api_method(http_method='GET', token_required=True, user_required=False)
@component_from_token
def get_pending_commissions(request):
client_key = str(request.component_instance)
client_key = unicode(request.component_instance)
result = qh.get_pending_commissions(clientkey=client_key)
return json_response(result)
......@@ -139,7 +139,7 @@ def _provisions_to_list(provisions):
if not is_integer(quantity):
raise ValueError()
except (TypeError, KeyError, ValueError):
raise BadRequest("Malformed provision %s" % str(provision))
raise BadRequest("Malformed provision %s" % unicode(provision))
return lst
......@@ -147,13 +147,10 @@ def _provisions_to_list(provisions):
@api.api_method(http_method='POST', token_required=True, user_required=False)
@component_from_token
def issue_commission(request):
data = request.body
try:
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
input_data = utils.get_json_body(request)
check_is_dict(input_data)
client_key = str(request.component_instance)
client_key = unicode(request.component_instance)
provisions = input_data.get('provisions')
if provisions is None:
raise BadRequest("Provisions are missing.")
......@@ -237,13 +234,10 @@ def conflictingCF(serial):
@component_from_token
@transaction.commit_on_success
def resolve_pending_commissions(request):
data = request.body
try:
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
input_data = utils.get_json_body(request)
check_is_dict(input_data)
client_key = str(request.component_instance)
client_key = unicode(request.component_instance)
accept = input_data.get('accept', [])
reject = input_data.get('reject', [])
......@@ -273,7 +267,7 @@ def resolve_pending_commissions(request):
@component_from_token
def get_commission(request, serial):
data = request.GET
client_key = str(request.component_instance)
client_key = unicode(request.component_instance)
try:
serial = int(serial)
except ValueError:
......@@ -293,18 +287,15 @@ def get_commission(request, serial):
@component_from_token
@transaction.commit_on_success
def serial_action(request, serial):
data = request.body
try:
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
input_data = utils.get_json_body(request)
check_is_dict(input_data)
try:
serial = int(serial)
except ValueError:
raise BadRequest("Serial should be an integer.")
client_key = str(request.component_instance)
client_key = unicode(request.component_instance)
accept = 'accept' in input_data
reject = 'reject' in input_data
......
# Copyright 2011-2013 GRNET S.A. All rights reserved.
# Copyright 2011-2014 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -91,7 +91,7 @@ def authenticate(request):
d = defaultdict(dict)
if not public_mode:
req = utils.get_request_dict(request)
req = utils.get_json_body(request)
uuid = None
try:
......
# Copyright 2013 GRNET S.A. All rights reserved.
# Copyright 2013-2014 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -81,16 +81,9 @@ def xml_response(content, template, status_code=None):
return response
def read_json_body(request, default=None):
body = request.body
if not body and request.method == "GET":
body = request.GET.get("body")
if not body:
return default
try:
return json.loads(body)
except json.JSONDecodeError:
raise faults.BadRequest("Request body should be in json format.")
def check_is_dict(obj):
if not isinstance(obj, dict):
raise faults.BadRequest("Request should be a JSON dict")
def is_integer(x):
......
......@@ -83,20 +83,20 @@ class ActivationBackend(object):
ActivationBackend handles user verification/activation.
Example usage::
>>> # it is wise to not instantiate a backend class directly but use
>>> # get_backend method instead.
>>> backend = get_backend()
>>> formCls = backend.get_signup_form(request.POST)
>>> if form.is_valid():
>>> user = form.create_user()
>>> activation = backend.handle_registration(user)
>>> # activation.status is one of backend.Result.{*} activation result
>>> # types
>>>
>>> # sending activation notifications is not done automatically
>>> # we need to call send_result_notifications
>>> backend.send_result_notifications(activation)
>>> return HttpResponse(activation.message)
#>>> # it is wise to not instantiate a backend class directly but use
#>>> # get_backend method instead.
#>>> backend = get_backend()
#>>> formCls = backend.get_signup_form(request.POST)
#>>> if form.is_valid():
#>>> user = form.create_user()
#>>> activation = backend.handle_registration(user)
#>>> # activation.status is one of backend.Result.{*} activation result
#>>> # types
#>>>
#>>> # sending activation notifications is not done automatically
#>>> # we need to call send_result_notifications
#>>> backend.send_result_notifications(activation)
#>>> return HttpResponse(activation.message)
"""
verification_template_name = 'im/activation_email.txt'
......@@ -251,7 +251,7 @@ class ActivationBackend(object):
user.moderated_at = datetime.datetime.now()
user.moderated_data = json.dumps(user.__dict__,
default=lambda obj:
str(obj))
unicode(obj))
user.save()
functions.enable_base_project(user)
......@@ -323,7 +323,7 @@ class ActivationBackend(object):
user.moderated_at = datetime.datetime.now()
user.moderated_data = json.dumps(user.__dict__,
default=lambda obj:
str(obj))
unicode(obj))
user.is_rejected = True
user.rejected_reason = reason
user.save()
......
......@@ -748,7 +748,7 @@ def modify_project(project_id, request):
main_fields = modifies_main_fields(request)
if main_fields:
m = (_(astakos_messages.BASE_NO_MODIFY_FIELDS)
% ", ".join(map(str, main_fields)))
% ", ".join(map(unicode, main_fields)))
raise ProjectBadRequest(m)
new_name = request.get("realname")
......@@ -765,7 +765,7 @@ def modify_projects_in_bulk(flt, request):
main_fields = modifies_main_fields(request)
if main_fields:
raise ProjectBadRequest("Cannot modify field(s) '%s' in bulk" %
", ".join(map(str, main_fields)))
", ".join(map(unicode, main_fields)))
projects = Project.objects.initialized(flt).select_for_update()
_modify_projects(projects, request)
......
......@@ -36,7 +36,7 @@ import uuid
from django.core.validators import validate_email
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType
from django.core.management import CommandError
from snf_django.management.commands import CommandError
from synnefo.util import units
from astakos.im.models import AstakosUser
......
# Copyright 2013 GRNET S.A. All rights reserved.
# Copyright 2013-2014 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -32,7 +32,7 @@
# or implied, of GRNET S.A.
from synnefo.util import units
from django.core.management import CommandError
from snf_django.management.commands import CommandError
from django.db.models import Q
......
......@@ -35,10 +35,9 @@ import string
from optparse import make_option
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile
from snf_django.management.commands import SynnefoCommand
option_list = list(SynnefoCommand.option_list) + [
make_option('--update',
......
......@@ -31,10 +31,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......
......@@ -34,9 +34,7 @@
from optparse import make_option
from django.db import transaction
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile
from astakos.im.models import AstakosUser, Group
......
# Copyright 2012, 2013 GRNET S.A. All rights reserved.
# Copyright 2012-2014 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -31,11 +31,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.core.management.base import CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile
from synnefo.lib.ordereddict import OrderedDict
from snf_django.management.commands import SynnefoCommand
from snf_django.management.commands import SynnefoCommand, CommandError
from snf_django.management import utils
......
......@@ -32,10 +32,8 @@
# or implied, of GRNET S.A.
from optparse import make_option
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import Component
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......
......@@ -32,11 +32,8 @@
# or implied, of GRNET S.A.
from optparse import make_option
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import Component
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......
......@@ -31,11 +31,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from django.db import transaction
from astakos.im.models import Component
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......
# Copyright 2013 GRNET S.A. All rights reserved.
# Copyright 2013-2014 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -31,10 +31,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.core.management.base import CommandError
from astakos.im.models import Component
from synnefo.lib.ordereddict import OrderedDict
from snf_django.management.commands import SynnefoCommand
from snf_django.management.commands import SynnefoCommand, CommandError
from snf_django.management import utils
......
......@@ -32,10 +32,9 @@
# or implied, of GRNET S.A.
from django.db import transaction
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.auth import fix_superusers
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......
......@@ -31,10 +31,8 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import Group
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......
......@@ -34,12 +34,11 @@
from optparse import make_option
from django.db import transaction
from django.core.management.base import CommandError
from snf_django.management import utils
from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.functions import (terminate, suspend, unsuspend,
reinstate, check_expiration,
approve_application, deny_application)
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......@@ -91,6 +90,7 @@ class Command(SynnefoCommand):
@transaction.commit_on_success
def handle(self, *args, **options):
self.output_format = options["output_format"]
message = options['message']
actions = {
......@@ -122,28 +122,15 @@ class Command(SynnefoCommand):
length = len(projects)
if length == 0:
s = 'No expired projects.\n'
elif length == 1:
s = '1 expired project:\n'
else:
s = '%d expired projects:\n' % (length,)
self.stderr.write(s)
if length > 0:
labels = ('Project', 'Name', 'Status', 'Expiration date')
columns = (10, 30, 14, 30)
line = ' '.join(l.rjust(w) for l, w in zip(labels, columns))
self.stderr.write(line + '\n')
sep = '-' * len(line)
self.stderr.write(sep + '\n')
for project in projects:
line = ' '.join(f.rjust(w) for f, w in zip(project, columns))
self.stderr.write(line + '\n')
if execute:
self.stderr.write('%d projects have been terminated.\n' % (
length,))
self.stderr.write(s)
return
labels = ('Project', 'Name', 'Status', 'Expiration date')
utils.pprint_table(self.stdout, projects, labels,
self.output_format, title="Expired projects")
if execute:
self.stderr.write('%d projects have been terminated.\n' %
(length,))
def expire(self, execute=False):
projects = check_expiration(execute=execute)
......
......@@ -34,7 +34,7 @@
from optparse import make_option
from django.db.models import Q
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from django.db import transaction
from synnefo.util import units
from astakos.im import functions
......@@ -43,7 +43,6 @@ import astakos.api.projects as api
import synnefo.util.date as date_util
from snf_django.management import utils
from astakos.im.management.commands import _common
from snf_django.management.commands import SynnefoCommand
def make_policies(limits):
......
......@@ -32,10 +32,9 @@
# or implied, of GRNET S.A.
from optparse import make_option
from django.core.management.base import CommandError
from synnefo.lib.ordereddict import OrderedDict
from snf_django.management.commands import SynnefoCommand
from snf_django.management.commands import SynnefoCommand, CommandError
from snf_django.management import utils
from astakos.im.models import ProjectApplication, Project
from astakos.im import quotas
......@@ -191,8 +190,6 @@ def app_fields(app):
def project_fields(project):
app = project.last_application
d = OrderedDict([
('project id', project.uuid),
('name', project.realname),
......
......@@ -34,9 +34,8 @@
from optparse import make_option
from datetime import datetime
from django.core.management.base import CommandError
from snf_django.management.commands import SynnefoCommand, CommandError
from django.db import transaction
from snf_django.utils import reconcile
from snf_django.management.utils import pprint_table
from astakos.im.models import Component, AstakosUser
......@@ -44,7 +43,6 @@ from astakos.im import quotas
from astakos.im.functions import count_pending_app
import astakos.quotaholder_app.callpoint as qh
import astakos.quotaholder_app.exception as qh_exception
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand):
......