functions.py

# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

from xml.dom import minidom

from django.http import HttpResponse
from django.template.loader import render_to_string
from django.utils import simplejson as json
from django.utils.http import parse_etags
from django.utils.encoding import smart_str
from django.views.decorators.csrf import csrf_exempt

from snf_django.lib.astakos import get_uuids as _get_uuids

from snf_django.lib import api
from snf_django.lib.api import faults

from pithos.api.util import (
    json_encode_decimal, rename_meta_key, format_header_key,
    printable_header_dict, get_account_headers, put_account_headers,
    get_container_headers, put_container_headers, get_object_headers,
    put_object_headers, update_manifest_meta, update_sharing_meta,
    update_public_meta, validate_modification_preconditions,
    validate_matching_preconditions, split_container_object_string,
    copy_or_move_object, get_int_parameter, get_content_length,
    get_content_range, socket_read_iterator, SaveToBackendHandler,
    object_data_response, put_object_block, hashmap_md5, simple_list_response,
    api_method, is_uuid,
    retrieve_uuid, retrieve_uuids, retrieve_displaynames,
    get_pithos_usage
)

from pithos.api.settings import (UPDATE_MD5, TRANSLATE_UUIDS,
                                 SERVICE_TOKEN, ASTAKOS_URL)

from pithos.backends.base import (
    NotAllowedError, QuotaError, ContainerNotEmpty, ItemNotExists,
    VersionNotExists, ContainerExists)

from pithos.backends.filter import parse_filters

import hashlib

import logging
logger = logging.getLogger(__name__)


def get_uuids(names):
    try:
        url = ASTAKOS_URL + "/service/api/user_catalogs"
        uuids = _get_uuids(SERVICE_TOKEN, names, url=url)
    except Exception, e:
        logger.exception(e)
        return {}

    return uuids


@csrf_exempt
def top_demux(request):
    if request.method == 'GET':
        try:
            request.GET['X-Auth-Token']
        except KeyError:
            try:
                request.META['HTTP_X_AUTH_TOKEN']
            except KeyError:
                return authenticate(request)
        return account_list(request)
    else:
        return api.method_not_allowed(request)


@csrf_exempt
def account_demux(request, v_account):
    if TRANSLATE_UUIDS:
        if not is_uuid(v_account):
            uuids = get_uuids([v_account])
            if not uuids or not v_account in uuids:
                return HttpResponse(status=404)
            v_account = uuids[v_account]

    if request.method == 'HEAD':
        return account_meta(request, v_account)
    elif request.method == 'POST':
        return account_update(request, v_account)
    elif request.method == 'GET':
        return container_list(request, v_account)
    else:
        return api.method_not_allowed(request)


@csrf_exempt
def container_demux(request, v_account, v_container):
    if TRANSLATE_UUIDS:
        if not is_uuid(v_account):
            uuids = get_uuids([v_account])
            if not uuids or not v_account in uuids:
                return HttpResponse(status=404)
            v_account = uuids[v_account]

    if request.method == 'HEAD':
        return container_meta(request, v_account, v_container)
    elif request.method == 'PUT':
        return container_create(request, v_account, v_container)
    elif request.method == 'POST':
        return container_update(request, v_account, v_container)
    elif request.method == 'DELETE':
        return container_delete(request, v_account, v_container)
    elif request.method == 'GET':
        return object_list(request, v_account, v_container)
    else:
        return api.method_not_allowed(request)


@csrf_exempt
def object_demux(request, v_account, v_container, v_object):
    # Helper to avoid placing the token in the URL
    # when loading objects from a browser.
    if TRANSLATE_UUIDS:
        if not is_uuid(v_account):
            uuids = get_uuids([v_account])
            if not uuids or not v_account in uuids:
                return HttpResponse(status=404)
            v_account = uuids[v_account]

    if request.method == 'HEAD':
        return object_meta(request, v_account, v_container, v_object)
    elif request.method == 'GET':
        return object_read(request, v_account, v_container, v_object)
    elif request.method == 'PUT':
        return object_write(request, v_account, v_container, v_object)
    elif request.method == 'COPY':
        return object_copy(request, v_account, v_container, v_object)
    elif request.method == 'MOVE':
        return object_move(request, v_account, v_container, v_object)
    elif request.method == 'POST':
        if request.META.get(
                'CONTENT_TYPE', '').startswith('multipart/form-data'):
            return object_write_form(request, v_account, v_container, v_object)
        return object_update(request, v_account, v_container, v_object)
    elif request.method == 'DELETE':
        return object_delete(request, v_account, v_container, v_object)
    else:
        return api.method_not_allowed(request)


@api_method('GET', user_required=False, logger=logger)
def authenticate(request):
    # Normal Response Codes: 204
    # Error Response Codes: internalServerError (500),
    #                       forbidden (403),
    #                       badRequest (400)

    x_auth_user = request.META.get('HTTP_X_AUTH_USER')
    x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
    if not x_auth_user or not x_auth_key:
        raise faults.BadRequest('Missing X-Auth-User or X-Auth-Key header')
    response = HttpResponse(status=204)

    uri = request.build_absolute_uri()
    if '?' in uri:
        uri = uri[:uri.find('?')]

    response['X-Auth-Token'] = x_auth_key
    response['X-Storage-Url'] = uri + ('' if uri.endswith('/')
                                       else '/') + x_auth_user
    return response


@api_method('GET', format_allowed=True, user_required=True, logger=logger)
def account_list(request):
    # Normal Response Codes: 200, 204
    # Error Response Codes: internalServerError (500),
    #                       badRequest (400)
    response = HttpResponse()

    marker = request.GET.get('marker')
    limit = get_int_parameter(request.GET.get('limit'))
    if not limit:
        limit = 10000

    accounts = request.backend.list_accounts(request.user_uniq, marker, limit)

    if request.serialization == 'text':
        if TRANSLATE_UUIDS:
            accounts = retrieve_displaynames(
                getattr(request, 'token', None), accounts)
        if len(accounts) == 0:
            # The cloudfiles python bindings expect 200 if json/xml.
            response.status_code = 204
            return response
        response.status_code = 200
        response.content = '\n'.join(accounts) + '\n'
        return response

    account_meta = []
    for x in accounts:
        if x == request.user_uniq:
            continue
        usage = get_pithos_usage(request.x_auth_token)
        try:
            meta = request.backend.get_account_meta(
                request.user_uniq, x, 'pithos', include_user_defined=False,
                external_quota=usage)
            groups = request.backend.get_account_groups(request.user_uniq, x)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        else:
            rename_meta_key(meta, 'modified', 'last_modified')
            rename_meta_key(
                meta, 'until_timestamp', 'x_account_until_timestamp')
            if groups:
                meta['X-Account-Group'] = printable_header_dict(
                    dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
            account_meta.append(printable_header_dict(meta))

    if TRANSLATE_UUIDS:
        uuids = list(d['name'] for d in account_meta)
        catalog = retrieve_displaynames(
            getattr(request, 'token', None), uuids, return_dict=True)
        for meta in account_meta:
            meta['name'] = catalog.get(meta.get('name'))

    if request.serialization == 'xml':
        data = render_to_string('accounts.xml', {'accounts': account_meta})
    elif request.serialization == 'json':
        data = json.dumps(account_meta)
    response.status_code = 200
    response.content = data
    return response


@api_method('HEAD', user_required=True, logger=logger)
def account_meta(request, v_account):
    # Normal Response Codes: 204
    # Error Response Codes: internalServerError (500),
    #                       forbidden (403),
    #                       badRequest (400)

    until = get_int_parameter(request.GET.get('until'))
    usage = get_pithos_usage(request.x_auth_token)
    try:
        meta = request.backend.get_account_meta(
            request.user_uniq, v_account, 'pithos', until,
            external_quota=usage)
        groups = request.backend.get_account_groups(
            request.user_uniq, v_account)

        if TRANSLATE_UUIDS:
            for k in groups:
                groups[k] = retrieve_displaynames(
                    getattr(request, 'token', None), groups[k])
        policy = request.backend.get_account_policy(
            request.user_uniq, v_account, external_quota=usage)
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')

    validate_modification_preconditions(request, meta)

    response = HttpResponse(status=204)
    put_account_headers(response, meta, groups, policy)
    return response


@api_method('POST', user_required=True, logger=logger)
def account_update(request, v_account):
    # Normal Response Codes: 202
    # Error Response Codes: internalServerError (500),
    #                       forbidden (403),
    #                       badRequest (400)

    meta, groups = get_account_headers(request)
    for k in groups:
        if TRANSLATE_UUIDS:
            try:
                groups[k] = retrieve_uuids(
                    getattr(request, 'token', None),
                    groups[k],
                    fail_silently=False)
            except ItemNotExists, e:
                raise faults.BadRequest(
                    'Bad X-Account-Group header value: %s' % e)
        else:
            try:
                retrieve_displaynames(
                    getattr(request, 'token', None),
                    groups[k],
                    fail_silently=False)
            except ItemNotExists, e:
                raise faults.BadRequest(
                    'Bad X-Account-Group header value: %s' % e)
    replace = True
    if 'update' in request.GET:
        replace = False
    if groups:
        try:
            request.backend.update_account_groups(request.user_uniq, v_account,
                                                  groups, replace)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except ValueError:
            raise faults.BadRequest('Invalid groups header')
    if meta or replace:
        try:
            request.backend.update_account_meta(request.user_uniq, v_account,
                                                'pithos', meta, replace)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
    return HttpResponse(status=202)


@api_method('GET', format_allowed=True, user_required=True, logger=logger)
def container_list(request, v_account):
    # Normal Response Codes: 200, 204
    # Error Response Codes: internalServerError (500),
    #                       itemNotFound (404),
    #                       forbidden (403),
    #                       badRequest (400)

    until = get_int_parameter(request.GET.get('until'))
    usage = get_pithos_usage(request.x_auth_token)
    try:
        meta = request.backend.get_account_meta(
            request.user_uniq, v_account, 'pithos', until,
            external_quota=usage)
        groups = request.backend.get_account_groups(
            request.user_uniq, v_account)
        policy = request.backend.get_account_policy(
            request.user_uniq, v_account, external_quota=usage)
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')

    validate_modification_preconditions(request, meta)

    response = HttpResponse()
    put_account_headers(response, meta, groups, policy)

    marker = request.GET.get('marker')
    limit = get_int_parameter(request.GET.get('limit'))
    if not limit:
        limit = 10000

    shared = False
    if 'shared' in request.GET:
        shared = True
    public = False
    if request.user_uniq == v_account and 'public' in request.GET:
        public = True

    try:
        containers = request.backend.list_containers(
            request.user_uniq, v_account,
            marker, limit, shared, until, public)
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')
    except NameError:
        containers = []

    if request.serialization == 'text':
        if len(containers) == 0:
            # The cloudfiles python bindings expect 200 if json/xml.
            response.status_code = 204
            return response
        response.status_code = 200
        response.content = '\n'.join(containers) + '\n'
        return response

    container_meta = []
    for x in containers:
        try:
            meta = request.backend.get_container_meta(
                request.user_uniq, v_account,
                x, 'pithos', until, include_user_defined=False)
            policy = request.backend.get_container_policy(request.user_uniq,
                                                          v_account, x)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except NameError:
            pass
        else:
            rename_meta_key(meta, 'modified', 'last_modified')
            rename_meta_key(
                meta, 'until_timestamp', 'x_container_until_timestamp')
            if policy:
                meta['X-Container-Policy'] = printable_header_dict(
                    dict([(k, v) for k, v in policy.iteritems()]))
            container_meta.append(printable_header_dict(meta))
    if request.serialization == 'xml':
        data = render_to_string('containers.xml', {'account':
                                v_account, 'containers': container_meta})
    elif request.serialization == 'json':
        data = json.dumps(container_meta)
    response.status_code = 200
    response.content = data
    return response


@api_method('HEAD', user_required=True, logger=logger)
def container_meta(request, v_account, v_container):
    # Normal Response Codes: 204
    # Error Response Codes: internalServerError (500),
    #                       itemNotFound (404),
    #                       forbidden (403),
    #                       badRequest (400)

    until = get_int_parameter(request.GET.get('until'))
    try:
        meta = request.backend.get_container_meta(request.user_uniq, v_account,
                                                  v_container, 'pithos', until)
        meta['object_meta'] = \
            request.backend.list_container_meta(request.user_uniq,
                                                v_account, v_container,
                                                'pithos', until)
        policy = request.backend.get_container_policy(
            request.user_uniq, v_account,
            v_container)
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')
    except ItemNotExists:
        raise faults.ItemNotFound('Container does not exist')

    validate_modification_preconditions(request, meta)

    response = HttpResponse(status=204)
    put_container_headers(request, response, meta, policy)
    return response


@api_method('PUT', user_required=True, logger=logger)
def container_create(request, v_account, v_container):
    # Normal Response Codes: 201, 202
    # Error Response Codes: internalServerError (500),
    #                       itemNotFound (404),
    #                       forbidden (403),
    #                       badRequest (400)

    meta, policy = get_container_headers(request)

    try:
        request.backend.put_container(
            request.user_uniq, v_account, v_container, policy)
        ret = 201
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')
    except ValueError:
        raise faults.BadRequest('Invalid policy header')
    except ContainerExists:
        ret = 202

    if ret == 202 and policy:
        try:
            request.backend.update_container_policy(
                request.user_uniq, v_account,
                v_container, policy, replace=False)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except ItemNotExists:
            raise faults.ItemNotFound('Container does not exist')
        except ValueError:
            raise faults.BadRequest('Invalid policy header')
    if meta:
        try:
            request.backend.update_container_meta(request.user_uniq, v_account,
                                                  v_container, 'pithos',
                                                  meta, replace=False)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except ItemNotExists:
            raise faults.ItemNotFound('Container does not exist')

    return HttpResponse(status=ret)


@api_method('POST', format_allowed=True, user_required=True, logger=logger)
def container_update(request, v_account, v_container):
    # Normal Response Codes: 202
    # Error Response Codes: internalServerError (500),
    #                       itemNotFound (404),
    #                       forbidden (403),
    #                       badRequest (400)

    meta, policy = get_container_headers(request)
    replace = True
    if 'update' in request.GET:
        replace = False
    if policy:
        try:
            request.backend.update_container_policy(
                request.user_uniq, v_account,
                v_container, policy, replace)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except ItemNotExists:
            raise faults.ItemNotFound('Container does not exist')
        except ValueError:
            raise faults.BadRequest('Invalid policy header')
    if meta or replace:
        try:
            request.backend.update_container_meta(request.user_uniq, v_account,
                                                  v_container, 'pithos',
                                                  meta, replace)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except ItemNotExists:
            raise faults.ItemNotFound('Container does not exist')

    content_length = -1
    if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
        content_length = get_int_parameter(
            request.META.get('CONTENT_LENGTH', 0))
    content_type = request.META.get('CONTENT_TYPE')
    hashmap = []
    if (content_type
            and content_type == 'application/octet-stream'
            and content_length != 0):
        for data in socket_read_iterator(request, content_length,
                                         request.backend.block_size):
            # TODO: Raise 408 (Request Timeout) if this takes too long.
            # TODO: Raise 499 (Client Disconnect) if a length is defined
            #       and we stop before getting this much data.
            hashmap.append(request.backend.put_block(data))

    response = HttpResponse(status=202)
    if hashmap:
        response.content = simple_list_response(request, hashmap)
    return response


@api_method('DELETE', user_required=True, logger=logger)
def container_delete(request, v_account, v_container):
    # Normal Response Codes: 204
    # Error Response Codes: internalServerError (500),
    #                       conflict (409),
    #                       itemNotFound (404),
    #                       forbidden (403),
    #                       badRequest (400)
    #                       requestentitytoolarge (413)

    until = get_int_parameter(request.GET.get('until'))

    delimiter = request.GET.get('delimiter')

    try:
        request.backend.delete_container(
            request.user_uniq, v_account, v_container,
            until, delimiter=delimiter)
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')
    except ItemNotExists:
        raise faults.ItemNotFound('Container does not exist')
    except ContainerNotEmpty:
        raise faults.Conflict('Container is not empty')
    except QuotaError, e:
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
    return HttpResponse(status=204)


@api_method('GET', format_allowed=True, user_required=True, logger=logger)
def object_list(request, v_account, v_container):
    # Normal Response Codes: 200, 204
    # Error Response Codes: internalServerError (500),
    #                       itemNotFound (404),
    #                       forbidden (403),
    #                       badRequest (400)

    until = get_int_parameter(request.GET.get('until'))
    try:
        meta = request.backend.get_container_meta(request.user_uniq, v_account,
                                                  v_container, 'pithos', until)
        meta['object_meta'] = \
            request.backend.list_container_meta(request.user_uniq,
                                                v_account, v_container,
                                                'pithos', until)
        policy = request.backend.get_container_policy(
            request.user_uniq, v_account,
            v_container)
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')
    except ItemNotExists:
        raise faults.ItemNotFound('Container does not exist')

    validate_modification_preconditions(request, meta)

    response = HttpResponse()
    put_container_headers(request, response, meta, policy)

    path = request.GET.get('path')
    prefix = request.GET.get('prefix')
    delimiter = request.GET.get('delimiter')

    # Path overrides prefix and delimiter.
    virtual = True
    if path:
        prefix = path
        delimiter = '/'
        virtual = False

    # Naming policy.
    if prefix and delimiter and not prefix.endswith(delimiter):
        prefix = prefix + delimiter
    if not prefix:
        prefix = ''
    prefix = prefix.lstrip('/')

    marker = request.GET.get('marker')
    limit = get_int_parameter(request.GET.get('limit'))
    if not limit:
        limit = 10000

    keys = request.GET.get('meta')
    if keys:
        keys = [smart_str(x.strip()) for x in keys.split(',')
                if x.strip() != '']
        included, excluded, opers = parse_filters(keys)
        keys = []
        keys += [format_header_key('X-Object-Meta-' + x) for x in included]
        keys += ['!' + format_header_key('X-Object-Meta-' + x)
                 for x in excluded]
        keys += ['%s%s%s' % (format_header_key(
            'X-Object-Meta-' + k), o, v) for k, o, v in opers]
    else:
        keys = []

    shared = False
    if 'shared' in request.GET:
        shared = True

    public_requested = 'public' in request.GET
    public_granted = public_requested and request.user_uniq == v_account

    if request.serialization == 'text':
        try:
            objects = request.backend.list_objects(
                request.user_uniq, v_account,
                v_container, prefix, delimiter, marker,
                limit, virtual, 'pithos', keys, shared,
                until, None, public_granted)
        except NotAllowedError:
            raise faults.Forbidden('Not allowed')
        except ItemNotExists:
            raise faults.ItemNotFound('Container does not exist')

        if len(objects) == 0:
            # The cloudfiles python bindings expect 200 if json/xml.
            response.status_code = 204
            return response
        response.status_code = 200
        response.content = '\n'.join([x[0] for x in objects]) + '\n'
        return response

    try:
        objects = request.backend.list_object_meta(
            request.user_uniq, v_account,
            v_container, prefix, delimiter, marker,
            limit, virtual, 'pithos', keys, shared, until, None, public_granted)
        object_permissions = {}
        object_public = {}
        if until is None:
            name = '/'.join((v_account, v_container, ''))
            name_idx = len(name)
            for x in request.backend.list_object_permissions(
                    request.user_uniq, v_account, v_container, prefix):

                # filter out objects which are not under the container
                if name != x[:name_idx]:
                    continue

                object = x[name_idx:]
                object_permissions[object] = \
                    request.backend.get_object_permissions(
                        request.user_uniq, v_account, v_container, object)

            if public_granted:
                for k, v in request.backend.list_object_public(
                        request.user_uniq, v_account,
                        v_container, prefix).iteritems():
                    object_public[k[name_idx:]] = v
    except NotAllowedError:
        raise faults.Forbidden('Not allowed')
    except ItemNotExists:
        raise faults.ItemNotFound('Container does not exist')

    object_meta = []
    for meta in objects:
        if TRANSLATE_UUIDS:
            modified_by = meta.get('modified_by')
            if modified_by:
                l = retrieve_displaynames(
                    getattr(request, 'token', None), [meta['modified_by']])
                if l is not None and len(l) == 1:
                    meta['modified_by'] = l[0]

        if len(meta) == 1:
            # Virtual objects/directories.
            object_meta.append(meta)
        else:
            rename_meta_key(
                meta, 'hash', 'x_object_hash')  # Will be replaced by checksum.
            rename_meta_key(meta, 'checksum', 'hash')
            rename_meta_key(meta, 'type', 'content_type')
            rename_meta_key(meta, 'uuid', 'x_object_uuid')
            if until is not None and 'modified' in meta:
                del(meta['modified'])
            else:
                rename_meta_key(meta, 'modified', 'last_modified')
            rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
            rename_meta_key(meta, 'version', 'x_object_version')
            rename_meta_key(
                meta, 'version_timestamp', 'x_object_version_timestamp')
            permissions = object_permissions.get(meta['name'], None)
            if permissions:
                update_sharing_meta(request, permissions, v_account,
                                    v_container, meta['name'], meta)
            public_url = object_public.get(meta['name'], None)
            if public_granted:
                update_public_meta(public_url, meta)
            object_meta.append(printable_header_dict(meta))

    if request.serialization == 'xml':