Commit 8d477a6d authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis
Browse files

Merge pull request #28 from cstavr/feature-unicode-handling

This patch-set fixes a number of issues relative with handling unicode strings.
These commits try to avoid mixing bytestrings and unicode objects, by decoding
bytestrings as early as possible (Input) and encoding unicodes as late as
possible (Ouput). In the API the encoding that is used is always 'UTF-8'. In
the CLI the encoding that is used is the user's preferred encoding.

Besides unicode issues, this patch-set fixes handling of image metadata
(Plankton) which are views as HTTP headers. Since, image metadata must be valid
HTTP headers, metadata keys and values which contain user defined values must
be properly quoted and unquoted where needed.
parents 98838c30 490861c8
...@@ -48,6 +48,17 @@ identity credentials. ...@@ -48,6 +48,17 @@ identity credentials.
<http://docs.openstack.org/developer/glance/glanceapi.html#authentication>`_, <http://docs.openstack.org/developer/glance/glanceapi.html#authentication>`_,
with the only difference being the suggested identity manager. with the only difference being the suggested identity manager.
Image Metadata Format
---------------------
In Cyclades Image API all image metadata are viewed as HTTP headers that are
starting with the `x-image-meta-` prefix. All metadata must be encoded with the
`UTF-8` encoding. Since the image metadata must be valid HTTP headers, user
defined metadata like the image's name or properties must also be properly
quoted. Finally, image properties that are viewed as HTTP headers and are
starting with the `x-image-meta-property-` prefix, are not case-sensitive and
all punctuation characters will be replaced with underscore.
List Available Images List Available Images
--------------------- ---------------------
......
...@@ -42,7 +42,8 @@ from astakos.api.util import json_response ...@@ -42,7 +42,8 @@ from astakos.api.util import json_response
from snf_django.lib import api from snf_django.lib import api
from snf_django.lib.api import faults from snf_django.lib.api import faults
from .util import user_from_token, invert_dict, read_json_body from snf_django.lib.api import utils
from .util import user_from_token, invert_dict, check_is_dict
from astakos.im import functions from astakos.im import functions
from astakos.im.models import ( from astakos.im.models import (
...@@ -319,8 +320,7 @@ def _get_projects(query, mode="default", request_user=None): ...@@ -319,8 +320,7 @@ def _get_projects(query, mode="default", request_user=None):
@transaction.commit_on_success @transaction.commit_on_success
def create_project(request): def create_project(request):
user = request.user user = request.user
data = request.body app_data = utils.get_json_body(request)
app_data = json.loads(data)
return submit_new_project(app_data, user) return submit_new_project(app_data, user)
...@@ -357,8 +357,7 @@ def _get_project(project_id, request_user=None): ...@@ -357,8 +357,7 @@ def _get_project(project_id, request_user=None):
@transaction.commit_on_success @transaction.commit_on_success
def modify_project(request, project_id): def modify_project(request, project_id):
user = request.user user = request.user
data = request.body app_data = utils.get_json_body(request)
app_data = json.loads(data)
return submit_modification(app_data, user, project_id=project_id) return submit_modification(app_data, user, project_id=project_id)
...@@ -548,6 +547,7 @@ def submit_modification(app_data, user, project_id): ...@@ -548,6 +547,7 @@ def submit_modification(app_data, user, project_id):
def get_action(actions, input_data): def get_action(actions, input_data):
action = None action = None
data = None data = None
check_is_dict(input_data)
for option in actions.keys(): for option in actions.keys():
if option in input_data: if option in input_data:
if action: if action:
...@@ -586,8 +586,7 @@ APP_ACTION_FUNCS = APPLICATION_ACTION.values() ...@@ -586,8 +586,7 @@ APP_ACTION_FUNCS = APPLICATION_ACTION.values()
@transaction.commit_on_success @transaction.commit_on_success
def project_action(request, project_id): def project_action(request, project_id):
user = request.user user = request.user
data = request.body input_data = utils.get_json_body(request)
input_data = json.loads(data)
func, action_data = get_action(PROJECT_ACTION, input_data) func, action_data = get_action(PROJECT_ACTION, input_data)
with ExceptionHandler(): with ExceptionHandler():
...@@ -707,7 +706,7 @@ MEMBERSHIP_ACTION = { ...@@ -707,7 +706,7 @@ MEMBERSHIP_ACTION = {
@transaction.commit_on_success @transaction.commit_on_success
def membership_action(request, memb_id): def membership_action(request, memb_id):
user = request.user user = request.user
input_data = read_json_body(request, default={}) input_data = utils.get_json_body(request)
func, action_data = get_action(MEMBERSHIP_ACTION, input_data) func, action_data = get_action(MEMBERSHIP_ACTION, input_data)
with ExceptionHandler(): with ExceptionHandler():
func(memb_id, user, reason=action_data) func(memb_id, user, reason=action_data)
......
...@@ -31,13 +31,13 @@ ...@@ -31,13 +31,13 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.utils import simplejson as json
from django.views.decorators.csrf import csrf_exempt from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse from django.http import HttpResponse
from django.db import transaction from django.db import transaction
from snf_django.lib import api from snf_django.lib import api
from snf_django.lib.api.faults import BadRequest, ItemNotFound from snf_django.lib.api.faults import BadRequest, ItemNotFound
from snf_django.lib.api import utils
from django.core.cache import cache from django.core.cache import cache
from astakos.im import settings from astakos.im import settings
...@@ -48,7 +48,7 @@ from astakos.im.quotas import get_user_quotas, service_get_quotas, \ ...@@ -48,7 +48,7 @@ from astakos.im.quotas import get_user_quotas, service_get_quotas, \
import astakos.quotaholder_app.exception as qh_exception import astakos.quotaholder_app.exception as qh_exception
import astakos.quotaholder_app.callpoint as qh import astakos.quotaholder_app.callpoint as qh
from .util import (json_response, is_integer, are_integer, from .util import (json_response, is_integer, are_integer, check_is_dict,
user_from_token, component_from_token) user_from_token, component_from_token)
...@@ -120,7 +120,7 @@ def commissions(request): ...@@ -120,7 +120,7 @@ def commissions(request):
@api.api_method(http_method='GET', token_required=True, user_required=False) @api.api_method(http_method='GET', token_required=True, user_required=False)
@component_from_token @component_from_token
def get_pending_commissions(request): def get_pending_commissions(request):
client_key = str(request.component_instance) client_key = unicode(request.component_instance)
result = qh.get_pending_commissions(clientkey=client_key) result = qh.get_pending_commissions(clientkey=client_key)
return json_response(result) return json_response(result)
...@@ -139,7 +139,7 @@ def _provisions_to_list(provisions): ...@@ -139,7 +139,7 @@ def _provisions_to_list(provisions):
if not is_integer(quantity): if not is_integer(quantity):
raise ValueError() raise ValueError()
except (TypeError, KeyError, ValueError): except (TypeError, KeyError, ValueError):
raise BadRequest("Malformed provision %s" % str(provision)) raise BadRequest("Malformed provision %s" % unicode(provision))
return lst return lst
...@@ -147,13 +147,10 @@ def _provisions_to_list(provisions): ...@@ -147,13 +147,10 @@ def _provisions_to_list(provisions):
@api.api_method(http_method='POST', token_required=True, user_required=False) @api.api_method(http_method='POST', token_required=True, user_required=False)
@component_from_token @component_from_token
def issue_commission(request): def issue_commission(request):
data = request.body input_data = utils.get_json_body(request)
try: check_is_dict(input_data)
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
client_key = str(request.component_instance) client_key = unicode(request.component_instance)
provisions = input_data.get('provisions') provisions = input_data.get('provisions')
if provisions is None: if provisions is None:
raise BadRequest("Provisions are missing.") raise BadRequest("Provisions are missing.")
...@@ -237,13 +234,10 @@ def conflictingCF(serial): ...@@ -237,13 +234,10 @@ def conflictingCF(serial):
@component_from_token @component_from_token
@transaction.commit_on_success @transaction.commit_on_success
def resolve_pending_commissions(request): def resolve_pending_commissions(request):
data = request.body input_data = utils.get_json_body(request)
try: check_is_dict(input_data)
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
client_key = str(request.component_instance) client_key = unicode(request.component_instance)
accept = input_data.get('accept', []) accept = input_data.get('accept', [])
reject = input_data.get('reject', []) reject = input_data.get('reject', [])
...@@ -273,7 +267,7 @@ def resolve_pending_commissions(request): ...@@ -273,7 +267,7 @@ def resolve_pending_commissions(request):
@component_from_token @component_from_token
def get_commission(request, serial): def get_commission(request, serial):
data = request.GET data = request.GET
client_key = str(request.component_instance) client_key = unicode(request.component_instance)
try: try:
serial = int(serial) serial = int(serial)
except ValueError: except ValueError:
...@@ -293,18 +287,15 @@ def get_commission(request, serial): ...@@ -293,18 +287,15 @@ def get_commission(request, serial):
@component_from_token @component_from_token
@transaction.commit_on_success @transaction.commit_on_success
def serial_action(request, serial): def serial_action(request, serial):
data = request.body input_data = utils.get_json_body(request)
try: check_is_dict(input_data)
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
try: try:
serial = int(serial) serial = int(serial)
except ValueError: except ValueError:
raise BadRequest("Serial should be an integer.") raise BadRequest("Serial should be an integer.")
client_key = str(request.component_instance) client_key = unicode(request.component_instance)
accept = 'accept' in input_data accept = 'accept' in input_data
reject = 'reject' in input_data reject = 'reject' in input_data
......
# Copyright 2011-2013 GRNET S.A. All rights reserved. # Copyright 2011-2014 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -91,7 +91,7 @@ def authenticate(request): ...@@ -91,7 +91,7 @@ def authenticate(request):
d = defaultdict(dict) d = defaultdict(dict)
if not public_mode: if not public_mode:
req = utils.get_request_dict(request) req = utils.get_json_body(request)
uuid = None uuid = None
try: try:
......
# Copyright 2013 GRNET S.A. All rights reserved. # Copyright 2013-2014 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -81,16 +81,9 @@ def xml_response(content, template, status_code=None): ...@@ -81,16 +81,9 @@ def xml_response(content, template, status_code=None):
return response return response
def read_json_body(request, default=None): def check_is_dict(obj):
body = request.body if not isinstance(obj, dict):
if not body and request.method == "GET": raise faults.BadRequest("Request should be a JSON dict")
body = request.GET.get("body")
if not body:
return default
try:
return json.loads(body)
except json.JSONDecodeError:
raise faults.BadRequest("Request body should be in json format.")
def is_integer(x): def is_integer(x):
......
...@@ -83,20 +83,20 @@ class ActivationBackend(object): ...@@ -83,20 +83,20 @@ class ActivationBackend(object):
ActivationBackend handles user verification/activation. ActivationBackend handles user verification/activation.
Example usage:: Example usage::
>>> # it is wise to not instantiate a backend class directly but use #>>> # it is wise to not instantiate a backend class directly but use
>>> # get_backend method instead. #>>> # get_backend method instead.
>>> backend = get_backend() #>>> backend = get_backend()
>>> formCls = backend.get_signup_form(request.POST) #>>> formCls = backend.get_signup_form(request.POST)
>>> if form.is_valid(): #>>> if form.is_valid():
>>> user = form.create_user() #>>> user = form.create_user()
>>> activation = backend.handle_registration(user) #>>> activation = backend.handle_registration(user)
>>> # activation.status is one of backend.Result.{*} activation result #>>> # activation.status is one of backend.Result.{*} activation result
>>> # types #>>> # types
>>> #>>>
>>> # sending activation notifications is not done automatically #>>> # sending activation notifications is not done automatically
>>> # we need to call send_result_notifications #>>> # we need to call send_result_notifications
>>> backend.send_result_notifications(activation) #>>> backend.send_result_notifications(activation)
>>> return HttpResponse(activation.message) #>>> return HttpResponse(activation.message)
""" """
verification_template_name = 'im/activation_email.txt' verification_template_name = 'im/activation_email.txt'
...@@ -251,7 +251,7 @@ class ActivationBackend(object): ...@@ -251,7 +251,7 @@ class ActivationBackend(object):
user.moderated_at = datetime.datetime.now() user.moderated_at = datetime.datetime.now()
user.moderated_data = json.dumps(user.__dict__, user.moderated_data = json.dumps(user.__dict__,
default=lambda obj: default=lambda obj:
str(obj)) unicode(obj))
user.save() user.save()
functions.enable_base_project(user) functions.enable_base_project(user)
...@@ -323,7 +323,7 @@ class ActivationBackend(object): ...@@ -323,7 +323,7 @@ class ActivationBackend(object):
user.moderated_at = datetime.datetime.now() user.moderated_at = datetime.datetime.now()
user.moderated_data = json.dumps(user.__dict__, user.moderated_data = json.dumps(user.__dict__,
default=lambda obj: default=lambda obj:
str(obj)) unicode(obj))
user.is_rejected = True user.is_rejected = True
user.rejected_reason = reason user.rejected_reason = reason
user.save() user.save()
......
...@@ -748,7 +748,7 @@ def modify_project(project_id, request): ...@@ -748,7 +748,7 @@ def modify_project(project_id, request):
main_fields = modifies_main_fields(request) main_fields = modifies_main_fields(request)
if main_fields: if main_fields:
m = (_(astakos_messages.BASE_NO_MODIFY_FIELDS) m = (_(astakos_messages.BASE_NO_MODIFY_FIELDS)
% ", ".join(map(str, main_fields))) % ", ".join(map(unicode, main_fields)))
raise ProjectBadRequest(m) raise ProjectBadRequest(m)
new_name = request.get("realname") new_name = request.get("realname")
...@@ -765,7 +765,7 @@ def modify_projects_in_bulk(flt, request): ...@@ -765,7 +765,7 @@ def modify_projects_in_bulk(flt, request):
main_fields = modifies_main_fields(request) main_fields = modifies_main_fields(request)
if main_fields: if main_fields:
raise ProjectBadRequest("Cannot modify field(s) '%s' in bulk" % raise ProjectBadRequest("Cannot modify field(s) '%s' in bulk" %
", ".join(map(str, main_fields))) ", ".join(map(unicode, main_fields)))
projects = Project.objects.initialized(flt).select_for_update() projects = Project.objects.initialized(flt).select_for_update()
_modify_projects(projects, request) _modify_projects(projects, request)
......
...@@ -36,7 +36,7 @@ import uuid ...@@ -36,7 +36,7 @@ import uuid
from django.core.validators import validate_email from django.core.validators import validate_email
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.management import CommandError from snf_django.management.commands import CommandError
from synnefo.util import units from synnefo.util import units
from astakos.im.models import AstakosUser from astakos.im.models import AstakosUser
......
# Copyright 2013 GRNET S.A. All rights reserved. # Copyright 2013-2014 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -32,7 +32,7 @@ ...@@ -32,7 +32,7 @@
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from synnefo.util import units from synnefo.util import units
from django.core.management import CommandError from snf_django.management.commands import CommandError
from django.db.models import Q from django.db.models import Q
......
...@@ -35,10 +35,9 @@ import string ...@@ -35,10 +35,9 @@ import string
from optparse import make_option from optparse import make_option
from django.core.management.base import CommandError from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile from astakos.im.models import AuthProviderPolicyProfile as Profile
from snf_django.management.commands import SynnefoCommand
option_list = list(SynnefoCommand.option_list) + [ option_list = list(SynnefoCommand.option_list) + [
make_option('--update', make_option('--update',
......
...@@ -31,10 +31,9 @@ ...@@ -31,10 +31,9 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.core.management.base import CommandError from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile from astakos.im.models import AuthProviderPolicyProfile as Profile
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand): class Command(SynnefoCommand):
......
...@@ -34,9 +34,7 @@ ...@@ -34,9 +34,7 @@
from optparse import make_option from optparse import make_option
from django.db import transaction from django.db import transaction
from django.core.management.base import CommandError from snf_django.management.commands import SynnefoCommand, CommandError
from snf_django.management.commands import SynnefoCommand
from astakos.im.models import AuthProviderPolicyProfile as Profile from astakos.im.models import AuthProviderPolicyProfile as Profile
from astakos.im.models import AstakosUser, Group from astakos.im.models import AstakosUser, Group
......
# Copyright 2012, 2013 GRNET S.A. All rights reserved. # Copyright 2012-2014 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -31,11 +31,9 @@ ...@@ -31,11 +31,9 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.core.management.base import CommandError
from astakos.im.models import AuthProviderPolicyProfile as Profile from astakos.im.models import AuthProviderPolicyProfile as Profile
from synnefo.lib.ordereddict import OrderedDict from synnefo.lib.ordereddict import OrderedDict
from snf_django.management.commands import SynnefoCommand from snf_django.management.commands import SynnefoCommand, CommandError
from snf_django.management import utils from snf_django.management import utils
......
...@@ -32,10 +32,8 @@ ...@@ -32,10 +32,8 @@
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from optparse import make_option from optparse import make_option
from django.core.management.base import CommandError from snf_django.management.commands import SynnefoCommand, CommandError
from astakos.im.models import Component from astakos.im.models import Component
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand): class Command(SynnefoCommand):
......
...@@ -32,11 +32,8 @@ ...@@ -32,11 +32,8 @@
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from optparse import make_option from optparse import make_option
from snf_django.management.commands import SynnefoCommand, CommandError
from django.core.management.base import CommandError
from astakos.im.models import Component from astakos.im.models import Component
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand): class Command(SynnefoCommand):
......
...@@ -31,11 +31,9 @@ ...@@ -31,11 +31,9 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.core.management.base import CommandError from snf_django.management.commands import SynnefoCommand, CommandError
from django.db import transaction from django.db import transaction
from astakos.im.models import Component from astakos.im.models import Component
from snf_django.management.commands import SynnefoCommand
class Command(SynnefoCommand): class Command(SynnefoCommand):
......
# Copyright 2013 GRNET S.A. All rights reserved. # Copyright 2013-2014 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or # Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following # without modification, are permitted provided that the following
...@@ -31,10 +31,9 @@ ...@@ -31,10 +31,9 @@
# interpreted as representing official policies, either expressed # interpreted as representing official policies, either expressed
# or implied, of GRNET S.A. # or implied, of GRNET S.A.
from django.core.management.base import CommandError
from astakos.im.models import Component from astakos.im.models import Component