util.py 35.7 KB
Newer Older
Antony Chazapis's avatar
Antony Chazapis committed
1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
Antony Chazapis's avatar
Antony Chazapis committed
3 4 5
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
6
#
Antony Chazapis's avatar
Antony Chazapis committed
7 8 9
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
10
#
Antony Chazapis's avatar
Antony Chazapis committed
11 12 13 14
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
15
#
Antony Chazapis's avatar
Antony Chazapis committed
16 17 18 19 20 21 22 23 24 25 26 27
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
28
#
Antony Chazapis's avatar
Antony Chazapis committed
29 30 31 32 33
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

34 35 36 37
from functools import wraps
from time import time
from traceback import format_exc
from wsgiref.handlers import format_date_time
Antony Chazapis's avatar
Antony Chazapis committed
38
from binascii import hexlify, unhexlify
39
from datetime import datetime, tzinfo, timedelta
40
from urllib import quote, unquote
41 42 43

from django.conf import settings
from django.http import HttpResponse
44
from django.template.loader import render_to_string
45
from django.utils import simplejson as json
46
from django.utils.http import http_date, parse_etags
Antony Chazapis's avatar
Antony Chazapis committed
47
from django.utils.encoding import smart_unicode, smart_str
48 49
from django.core.files.uploadhandler import FileUploadHandler
from django.core.files.uploadedfile import UploadedFile
50

Antony Chazapis's avatar
Antony Chazapis committed
51
from synnefo.lib.parsedate import parse_http_date_safe, parse_http_date
52
from synnefo.lib.astakos import get_user
53

54 55 56 57
from pithos.api.faults import (
    Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound,
    Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge,
    RangeNotSatisfiable, InternalServerError, NotImplemented)
Antony Chazapis's avatar
Antony Chazapis committed
58
from pithos.api.short_url import encode_url
Antony Chazapis's avatar
Antony Chazapis committed
59
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
60 61 62 63 64 65
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
                                 BACKEND_BLOCK_UMASK,
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_CONNECTION,
                                 BACKEND_QUOTA, BACKEND_VERSIONING,
                                 AUTHENTICATION_URL, AUTHENTICATION_USERS,
                                 SERVICE_TOKEN, COOKIE_NAME)
66

67
from pithos.backends import connect_backend
68
from pithos.backends.base import NotAllowedError, QuotaError, ItemNotExists, VersionNotExists
69 70

import logging
71
import re
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
72
import hashlib
73
import uuid
74
import decimal
75

76 77 78 79

logger = logging.getLogger(__name__)


80
class UTC(tzinfo):
81 82
    def utcoffset(self, dt):
        return timedelta(0)
83

84 85 86 87 88
    def tzname(self, dt):
        return 'UTC'

    def dst(self, dt):
        return timedelta(0)
89 90


91 92 93 94 95
def json_encode_decimal(obj):
    if isinstance(obj, decimal.Decimal):
        return str(obj)
    raise TypeError(repr(obj) + " is not JSON serializable")

96

97
def isoformat(d):
98 99 100
    """Return an ISO8601 date string that includes a timezone."""

    return d.replace(tzinfo=UTC()).isoformat()
101 102


103 104 105 106 107 108
def rename_meta_key(d, old, new):
    if old not in d:
        return
    d[new] = d[old]
    del(d[old])

109

110
def printable_header_dict(d):
111
    """Format a meta dictionary for printing out json/xml.
112

113 114
    Convert all keys to lower case and replace dashes with underscores.
    Format 'last_modified' timestamp.
115
    """
116

117
    if 'last_modified' in d and d['last_modified']:
118 119
        d['last_modified'] = isoformat(
            datetime.fromtimestamp(d['last_modified']))
120 121
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])

122

123
def format_header_key(k):
124
    """Convert underscores to dashes and capitalize intra-dash strings."""
125 126
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])

127

128 129
def get_header_prefix(request, prefix):
    """Get all prefix-* request headers in a dict. Reformat keys with format_header_key()."""
130

131
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
Antony Chazapis's avatar
Antony Chazapis committed
132 133
    # TODO: Document or remove '~' replacing.
    return dict([(format_header_key(k[5:]), v.replace('~', '')) for k, v in request.META.iteritems() if k.startswith(prefix) and len(k) > len(prefix)])
134

135

136 137 138 139 140 141 142 143 144
def check_meta_headers(meta):
    if len(meta) > 90:
        raise BadRequest('Too many headers.')
    for k, v in meta.iteritems():
        if len(k) > 128:
            raise BadRequest('Header name too large.')
        if len(v) > 256:
            raise BadRequest('Header value too large.')

145

146 147
def get_account_headers(request):
    meta = get_header_prefix(request, 'X-Account-Meta-')
148
    check_meta_headers(meta)
149 150 151 152 153 154
    groups = {}
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
        n = k[16:].lower()
        if '-' in n or '_' in n:
            raise BadRequest('Bad characters in group name')
        groups[n] = v.replace(' ', '').split(',')
Antony Chazapis's avatar
Antony Chazapis committed
155
        while '' in groups[n]:
156 157 158
            groups[n].remove('')
    return meta, groups

159

160
def put_account_headers(response, meta, groups, policy):
161 162 163 164
    if 'count' in meta:
        response['X-Account-Container-Count'] = meta['count']
    if 'bytes' in meta:
        response['X-Account-Bytes-Used'] = meta['bytes']
165
    response['Last-Modified'] = http_date(int(meta['modified']))
166
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
167 168
        response[smart_str(
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
169
    if 'until_timestamp' in meta:
170 171
        response['X-Account-Until-Timestamp'] = http_date(
            int(meta['until_timestamp']))
172
    for k, v in groups.iteritems():
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
173 174 175 176
        k = smart_str(k, strings_only=True)
        k = format_header_key('X-Account-Group-' + k)
        v = smart_str(','.join(v), strings_only=True)
        response[k] = v
177 178 179
    for k, v in policy.iteritems():
        response[smart_str(format_header_key('X-Account-Policy-' + k), strings_only=True)] = smart_str(v, strings_only=True)

180

181 182
def get_container_headers(request):
    meta = get_header_prefix(request, 'X-Container-Meta-')
183
    check_meta_headers(meta)
Antony Chazapis's avatar
Antony Chazapis committed
184 185
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in get_header_prefix(request, 'X-Container-Policy-').iteritems()])
    return meta, policy
186

187

188
def put_container_headers(request, response, meta, policy):
189 190 191 192
    if 'count' in meta:
        response['X-Container-Object-Count'] = meta['count']
    if 'bytes' in meta:
        response['X-Container-Bytes-Used'] = meta['bytes']
193
    response['Last-Modified'] = http_date(int(meta['modified']))
194
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
195 196 197 198
        response[smart_str(
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
         if x.startswith('X-Object-Meta-')]
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
199
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
200 201
    response['X-Container-Block-Size'] = request.backend.block_size
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
202
    if 'until_timestamp' in meta:
203 204
        response['X-Container-Until-Timestamp'] = http_date(
            int(meta['until_timestamp']))
Antony Chazapis's avatar
Antony Chazapis committed
205
    for k, v in policy.iteritems():
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
206
        response[smart_str(format_header_key('X-Container-Policy-' + k), strings_only=True)] = smart_str(v, strings_only=True)
207

208

209
def get_object_headers(request):
Antony Chazapis's avatar
Antony Chazapis committed
210
    content_type = request.META.get('CONTENT_TYPE', None)
211
    meta = get_header_prefix(request, 'X-Object-Meta-')
212
    check_meta_headers(meta)
213 214
    if request.META.get('HTTP_CONTENT_ENCODING'):
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
215 216
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
217 218
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
Antony Chazapis's avatar
Antony Chazapis committed
219
    return content_type, meta, get_sharing(request), get_public(request)
220

221

Antony Chazapis's avatar
Antony Chazapis committed
222
def put_object_headers(response, meta, restricted=False):
223
    response['ETag'] = meta['checksum']
224
    response['Content-Length'] = meta['bytes']
Antony Chazapis's avatar
Antony Chazapis committed
225
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
226
    response['Last-Modified'] = http_date(int(meta['modified']))
Antony Chazapis's avatar
Antony Chazapis committed
227
    if not restricted:
228
        response['X-Object-Hash'] = meta['hash']
229
        response['X-Object-UUID'] = meta['uuid']
230 231
        response['X-Object-Modified-By'] = smart_str(
            meta['modified_by'], strings_only=True)
232
        response['X-Object-Version'] = meta['version']
233 234
        response['X-Object-Version-Timestamp'] = http_date(
            int(meta['version_timestamp']))
235
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
236 237 238 239 240 241
            response[smart_str(
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
        for k in (
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
                'X-Object-Public'):
242
            if k in meta:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
243
                response[k] = smart_str(meta[k], strings_only=True)
244
    else:
245
        for k in ('Content-Encoding', 'Content-Disposition'):
246
            if k in meta:
247
                response[k] = smart_str(meta[k], strings_only=True)
248

249

Antony Chazapis's avatar
Antony Chazapis committed
250 251
def update_manifest_meta(request, v_account, meta):
    """Update metadata if the object has an X-Object-Manifest."""
252

Antony Chazapis's avatar
Antony Chazapis committed
253
    if 'X-Object-Manifest' in meta:
254
        etag = ''
Antony Chazapis's avatar
Antony Chazapis committed
255 256
        bytes = 0
        try:
257 258 259 260 261
            src_container, src_name = split_container_object_string(
                '/' + meta['X-Object-Manifest'])
            objects = request.backend.list_objects(
                request.user_uniq, v_account,
                src_container, prefix=src_name, virtual=False)
Antony Chazapis's avatar
Antony Chazapis committed
262
            for x in objects:
263
                src_meta = request.backend.get_object_meta(request.user_uniq,
264
                                                           v_account, src_container, x[0], 'pithos', x[1])
265
                etag += src_meta['checksum']
Antony Chazapis's avatar
Antony Chazapis committed
266 267 268 269 270 271
                bytes += src_meta['bytes']
        except:
            # Ignore errors.
            return
        meta['bytes'] = bytes
        md5 = hashlib.md5()
272
        md5.update(etag)
273
        meta['checksum'] = md5.hexdigest().lower()
Antony Chazapis's avatar
Antony Chazapis committed
274

275

276
def update_sharing_meta(request, permissions, v_account, v_container, v_object, meta):
Antony Chazapis's avatar
Antony Chazapis committed
277 278
    if permissions is None:
        return
279
    allowed, perm_path, perms = permissions
Antony Chazapis's avatar
Antony Chazapis committed
280 281
    if len(perms) == 0:
        return
Antony Chazapis's avatar
Antony Chazapis committed
282
    ret = []
Antony Chazapis's avatar
Antony Chazapis committed
283
    r = ','.join(perms.get('read', []))
Antony Chazapis's avatar
Antony Chazapis committed
284 285
    if r:
        ret.append('read=' + r)
Antony Chazapis's avatar
Antony Chazapis committed
286
    w = ','.join(perms.get('write', []))
Antony Chazapis's avatar
Antony Chazapis committed
287 288
    if w:
        ret.append('write=' + w)
Antony Chazapis's avatar
Antony Chazapis committed
289 290 291
    meta['X-Object-Sharing'] = '; '.join(ret)
    if '/'.join((v_account, v_container, v_object)) != perm_path:
        meta['X-Object-Shared-By'] = perm_path
292
    if request.user_uniq != v_account:
293
        meta['X-Object-Allowed-To'] = allowed
Antony Chazapis's avatar
Antony Chazapis committed
294

295

296 297 298
def update_public_meta(public, meta):
    if not public:
        return
Antony Chazapis's avatar
Antony Chazapis committed
299
    meta['X-Object-Public'] = '/public/' + encode_url(public)
300

301

302
def validate_modification_preconditions(request, meta):
303
    """Check that the modified timestamp conforms with the preconditions set."""
304

305
    if 'modified' not in meta:
306 307
        return  # TODO: Always return?

308 309 310
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
    if if_modified_since is not None:
        if_modified_since = parse_http_date_safe(if_modified_since)
311
    if if_modified_since is not None and int(meta['modified']) <= if_modified_since:
312
        raise NotModified('Resource has not been modified')
313

314 315 316
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
    if if_unmodified_since is not None:
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
317
    if if_unmodified_since is not None and int(meta['modified']) > if_unmodified_since:
318
        raise PreconditionFailed('Resource has been modified')
319

320

321
def validate_matching_preconditions(request, meta):
322
    """Check that the ETag conforms with the preconditions set."""
323

324 325 326
    etag = meta['checksum']
    if not etag:
        etag = None
327

328
    if_match = request.META.get('HTTP_IF_MATCH')
329
    if if_match is not None:
330
        if etag is None:
331
            raise PreconditionFailed('Resource does not exist')
332
        if if_match != '*' and etag not in [x.lower() for x in parse_etags(if_match)]:
333
            raise PreconditionFailed('Resource ETag does not match')
334

335 336
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
    if if_none_match is not None:
337
        # TODO: If this passes, must ignore If-Modified-Since header.
338 339
        if etag is not None:
            if if_none_match == '*' or etag in [x.lower() for x in parse_etags(if_none_match)]:
340 341 342 343
                # TODO: Continue if an If-Modified-Since header is present.
                if request.method in ('HEAD', 'GET'):
                    raise NotModified('Resource ETag matches')
                raise PreconditionFailed('Resource exists or ETag matches')
344

345

346
def split_container_object_string(s):
347 348 349
    if not len(s) > 0 or s[0] != '/':
        raise ValueError
    s = s[1:]
Antony Chazapis's avatar
Antony Chazapis committed
350
    pos = s.find('/')
351
    if pos == -1 or pos == len(s) - 1:
352
        raise ValueError
Antony Chazapis's avatar
Antony Chazapis committed
353
    return s[:pos], s[(pos + 1):]
354

355

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
356
def copy_or_move_object(request, src_account, src_container, src_name, dest_account, dest_container, dest_name, move=False, delimiter=None):
357
    """Copy or move an object."""
358

359 360
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
        del(request.META['CONTENT_TYPE'])
Antony Chazapis's avatar
Antony Chazapis committed
361
    content_type, meta, permissions, public = get_object_headers(request)
362
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
363 364
    try:
        if move:
365 366 367 368
            version_id = request.backend.move_object(
                request.user_uniq, src_account, src_container, src_name,
                dest_account, dest_container, dest_name,
                content_type, 'pithos', meta, False, permissions, delimiter)
369
        else:
370 371 372 373
            version_id = request.backend.copy_object(
                request.user_uniq, src_account, src_container, src_name,
                dest_account, dest_container, dest_name,
                content_type, 'pithos', meta, False, permissions, src_version, delimiter)
Antony Chazapis's avatar
Antony Chazapis committed
374
    except NotAllowedError:
375
        raise Forbidden('Not allowed')
376
    except (ItemNotExists, VersionNotExists):
377
        raise ItemNotFound('Container or object does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
378 379
    except ValueError:
        raise BadRequest('Invalid sharing header')
380 381
    except QuotaError:
        raise RequestEntityTooLarge('Quota exceeded')
382 383
    if public is not None:
        try:
384
            request.backend.update_object_public(request.user_uniq, dest_account, dest_container, dest_name, public)
385
        except NotAllowedError:
386
            raise Forbidden('Not allowed')
387
        except ItemNotExists:
388
            raise ItemNotFound('Object does not exist')
389
    return version_id
390

391

Antony Chazapis's avatar
Antony Chazapis committed
392
def get_int_parameter(p):
393
    if p is not None:
394
        try:
395
            p = int(p)
396 397
        except ValueError:
            return None
398
        if p < 0:
399
            return None
400
    return p
401

402

403
def get_content_length(request):
Antony Chazapis's avatar
Antony Chazapis committed
404 405 406
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
    if content_length is None:
        raise LengthRequired('Missing or invalid Content-Length header')
407 408
    return content_length

409

410
def get_range(request, size):
411
    """Parse a Range header from the request.
412

413 414
    Either returns None, when the header is not existent or should be ignored,
    or a list of (offset, length) tuples - should be further checked.
415
    """
416

417 418
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
    if not ranges.startswith('bytes='):
419
        return None
420

421 422 423 424 425 426 427 428 429
    ret = []
    for r in (x.strip() for x in ranges[6:].split(',')):
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
        m = p.match(r)
        if not m:
            return None
        offset = m.group('offset')
        upto = m.group('upto')
        if offset == '' and upto == '':
430
            return None
431

432 433 434
        if offset != '':
            offset = int(offset)
            if upto != '':
435
                upto = int(upto)
436 437 438 439 440
                if offset > upto:
                    return None
                ret.append((offset, upto - offset + 1))
            else:
                ret.append((offset, size - offset))
441
        else:
442 443
            length = int(upto)
            ret.append((size - length, length))
444

445 446
    return ret

447

448
def get_content_range(request):
449
    """Parse a Content-Range header from the request.
450

451 452 453 454
    Either returns None, when the header is not existent or should be ignored,
    or an (offset, length, total) tuple - check as length, total may be None.
    Returns (None, None, None) if the provided range is '*/*'.
    """
455

456 457 458
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
    if not ranges:
        return None
459

460 461 462 463 464 465 466 467 468 469 470
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
    m = p.match(ranges)
    if not m:
        if ranges == 'bytes */*':
            return (None, None, None)
        return None
    offset = int(m.group('offset'))
    upto = m.group('upto')
    total = m.group('total')
    if upto != '':
        upto = int(upto)
471
    else:
472 473 474 475 476
        upto = None
    if total != '*':
        total = int(total)
    else:
        total = None
Antony Chazapis's avatar
Antony Chazapis committed
477 478
    if (upto is not None and offset > upto) or \
        (total is not None and offset >= total) or \
479
            (total is not None and upto is not None and upto >= total):
480
        return None
481

Antony Chazapis's avatar
Antony Chazapis committed
482
    if upto is None:
483 484 485 486
        length = None
    else:
        length = upto - offset + 1
    return (offset, length, total)
487

488

Antony Chazapis's avatar
Antony Chazapis committed
489 490
def get_sharing(request):
    """Parse an X-Object-Sharing header from the request.
491

Antony Chazapis's avatar
Antony Chazapis committed
492 493
    Raises BadRequest on error.
    """
494

Antony Chazapis's avatar
Antony Chazapis committed
495
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
Antony Chazapis's avatar
Antony Chazapis committed
496
    if permissions is None:
Antony Chazapis's avatar
Antony Chazapis committed
497
        return None
498

499 500
    # TODO: Document or remove '~' replacing.
    permissions = permissions.replace('~', '')
501

Antony Chazapis's avatar
Antony Chazapis committed
502
    ret = {}
Antony Chazapis's avatar
Antony Chazapis committed
503 504 505 506 507
    permissions = permissions.replace(' ', '')
    if permissions == '':
        return ret
    for perm in (x for x in permissions.split(';')):
        if perm.startswith('read='):
508 509
            ret['read'] = list(set(
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
510 511
            if '' in ret['read']:
                ret['read'].remove('')
512 513
            if '*' in ret['read']:
                ret['read'] = ['*']
Antony Chazapis's avatar
Antony Chazapis committed
514 515 516
            if len(ret['read']) == 0:
                raise BadRequest('Bad X-Object-Sharing header value')
        elif perm.startswith('write='):
517 518
            ret['write'] = list(set(
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
519 520
            if '' in ret['write']:
                ret['write'].remove('')
521 522
            if '*' in ret['write']:
                ret['write'] = ['*']
Antony Chazapis's avatar
Antony Chazapis committed
523 524 525 526
            if len(ret['write']) == 0:
                raise BadRequest('Bad X-Object-Sharing header value')
        else:
            raise BadRequest('Bad X-Object-Sharing header value')
527

528
    # Keep duplicates only in write list.
529 530
    dups = [x for x in ret.get(
        'read', []) if x in ret.get('write', []) and x != '*']
531 532 533 534 535
    if dups:
        for x in dups:
            ret['read'].remove(x)
        if len(ret['read']) == 0:
            del(ret['read'])
536

Antony Chazapis's avatar
Antony Chazapis committed
537 538
    return ret

539

Antony Chazapis's avatar
Antony Chazapis committed
540 541
def get_public(request):
    """Parse an X-Object-Public header from the request.
542

Antony Chazapis's avatar
Antony Chazapis committed
543 544
    Raises BadRequest on error.
    """
545

Antony Chazapis's avatar
Antony Chazapis committed
546 547 548
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
    if public is None:
        return None
549

Antony Chazapis's avatar
Antony Chazapis committed
550 551 552 553 554 555 556
    public = public.replace(' ', '').lower()
    if public == 'true':
        return True
    elif public == 'false' or public == '':
        return False
    raise BadRequest('Bad X-Object-Public header value')

557

558
def raw_input_socket(request):
559
    """Return the socket for reading the rest of the request."""
560

561
    server_software = request.META.get('SERVER_SOFTWARE')
562
    if server_software and server_software.startswith('mod_python'):
563
        return request._req
564 565
    if 'wsgi.input' in request.environ:
        return request.environ['wsgi.input']
566
    raise NotImplemented('Unknown server software')
567

568 569
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB

570

571
def socket_read_iterator(request, length=0, blocksize=4096):
572
    """Return a maximum of blocksize data read from the socket in each iteration.
573

574
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
575 576
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
    """
577

578
    sock = raw_input_socket(request)
579
    if length < 0:  # Chunked transfers
580
        # Small version (server does the dechunking).
581
        if request.environ.get('mod_wsgi.input_chunked', None) or request.META['SERVER_SOFTWARE'].startswith('gunicorn'):
582 583 584 585 586 587
            while length < MAX_UPLOAD_SIZE:
                data = sock.read(blocksize)
                if data == '':
                    return
                yield data
            raise BadRequest('Maximum size is reached')
588

589
        # Long version (do the dechunking).
590
        data = ''
591
        while length < MAX_UPLOAD_SIZE:
592 593 594 595 596 597 598 599
            # Get chunk size.
            if hasattr(sock, 'readline'):
                chunk_length = sock.readline()
            else:
                chunk_length = ''
                while chunk_length[-1:] != '\n':
                    chunk_length += sock.read(1)
                chunk_length.strip()
600 601 602 603 604 605
            pos = chunk_length.find(';')
            if pos >= 0:
                chunk_length = chunk_length[:pos]
            try:
                chunk_length = int(chunk_length, 16)
            except Exception, e:
606 607
                raise BadRequest('Bad chunk size')
                                 # TODO: Change to something more appropriate.
608
            # Check if done.
609
            if chunk_length == 0:
610 611
                if len(data) > 0:
                    yield data
612
                return
613
            # Get the actual data.
614
            while chunk_length > 0:
615 616
                chunk = sock.read(min(chunk_length, blocksize))
                chunk_length -= len(chunk)
617 618
                if length > 0:
                    length += len(chunk)
619 620 621 622 623
                data += chunk
                if len(data) >= blocksize:
                    ret = data[:blocksize]
                    data = data[blocksize:]
                    yield ret
624
            sock.read(2)  # CRLF
625
        raise BadRequest('Maximum size is reached')
626 627
    else:
        if length > MAX_UPLOAD_SIZE:
628
            raise BadRequest('Maximum size is reached')
629 630
        while length > 0:
            data = sock.read(min(length, blocksize))
631 632
            if not data:
                raise BadRequest()
633 634 635
            length -= len(data)
            yield data

636

637 638
class SaveToBackendHandler(FileUploadHandler):
    """Handle a file from an HTML form the django way."""
639

640 641 642
    def __init__(self, request=None):
        super(SaveToBackendHandler, self).__init__(request)
        self.backend = request.backend
643

644 645 646 647 648 649
    def put_data(self, length):
        if len(self.data) >= length:
            block = self.data[:length]
            self.file.hashmap.append(self.backend.put_block(block))
            self.md5.update(block)
            self.data = self.data[length:]
650

651
    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
652
        self.md5 = hashlib.md5()
653
        self.data = ''
654 655
        self.file = UploadedFile(
            name=file_name, content_type=content_type, charset=charset)
656 657
        self.file.size = 0
        self.file.hashmap = []
658

659 660 661 662 663
    def receive_data_chunk(self, raw_data, start):
        self.data += raw_data
        self.file.size += len(raw_data)
        self.put_data(self.request.backend.block_size)
        return None
664

665 666 667 668 669 670 671
    def file_complete(self, file_size):
        l = len(self.data)
        if l > 0:
            self.put_data(l)
        self.file.etag = self.md5.hexdigest().lower()
        return self.file

672

673
class ObjectWrapper(object):
674
    """Return the object's data block-per-block in each iteration.
675

676 677
    Read from the object using the offset and length provided in each entry of the range list.
    """
678

679 680
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
        self.backend = backend
681
        self.ranges = ranges
Antony Chazapis's avatar
Antony Chazapis committed
682 683
        self.sizes = sizes
        self.hashmaps = hashmaps
684
        self.boundary = boundary
Antony Chazapis's avatar
Antony Chazapis committed
685
        self.size = sum(self.sizes)
686

Antony Chazapis's avatar
Antony Chazapis committed
687 688 689
        self.file_index = 0
        self.block_index = 0
        self.block_hash = -1
690
        self.block = ''
691

692 693
        self.range_index = -1
        self.offset, self.length = self.ranges[0]
694

695 696
    def __iter__(self):
        return self
697

698 699
    def part_iterator(self):
        if self.length > 0:
Antony Chazapis's avatar
Antony Chazapis committed
700 701 702 703 704 705
            # Get the file for the current offset.
            file_size = self.sizes[self.file_index]
            while self.offset >= file_size:
                self.offset -= file_size
                self.file_index += 1
                file_size = self.sizes[self.file_index]
706

Antony Chazapis's avatar
Antony Chazapis committed
707
            # Get the block for the current position.
708
            self.block_index = int(self.offset / self.backend.block_size)
Antony Chazapis's avatar
Antony Chazapis committed
709
            if self.block_hash != self.hashmaps[self.file_index][self.block_index]:
710 711
                self.block_hash = self.hashmaps[
                    self.file_index][self.block_index]
712
                try:
713
                    self.block = self.backend.get_block(self.block_hash)
714
                except ItemNotExists:
715
                    raise ItemNotFound('Block does not exist')
716

717
            # Get the data from the block.
718
            bo = self.offset % self.backend.block_size
719
            bs = self.backend.block_size
720
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
721
                    self.sizes[self.file_index] % self.backend.block_size):
722 723
                bs = self.sizes[self.file_index] % self.backend.block_size
            bl = min(self.length, bs - bo)
724 725 726 727 728 729
            data = self.block[bo:bo + bl]
            self.offset += bl
            self.length -= bl
            return data
        else:
            raise StopIteration
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
    def next(self):
        if len(self.ranges) == 1:
            return self.part_iterator()
        if self.range_index == len(self.ranges):
            raise StopIteration
        try:
            if self.range_index == -1:
                raise StopIteration
            return self.part_iterator()
        except StopIteration:
            self.range_index += 1
            out = []
            if self.range_index < len(self.ranges):
                # Part header.
                self.offset, self.length = self.ranges[self.range_index]
Antony Chazapis's avatar
Antony Chazapis committed
746
                self.file_index = 0
747 748 749
                if self.range_index > 0:
                    out.append('')
                out.append('--' + self.boundary)
750 751
                out.append('Content-Range: bytes %d-%d/%d' % (
                    self.offset, self.offset + self.length - 1, self.size))
752 753 754 755 756 757 758 759 760 761 762
                out.append('Content-Transfer-Encoding: binary')
                out.append('')
                out.append('')
                return '\r\n'.join(out)
            else:
                # Footer.
                out.append('')
                out.append('--' + self.boundary + '--')
                out.append('')
                return '\r\n'.join(out)

763

Antony Chazapis's avatar
Antony Chazapis committed
764
def object_data_response(request, sizes, hashmaps, meta, public=False):
765
    """Get the HttpResponse object for replying with the object's data."""
766

767
    # Range handling.
Antony Chazapis's avatar
Antony Chazapis committed
768
    size = sum(sizes)