Commit fda14525 authored by Antony Chazapis's avatar Antony Chazapis
Browse files

Clean up, sort out logging.

parent ecefb5c1
# Copyright (c) Django Software Foundation and individual contributors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of Django nor the names of its contributors may be used
# to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import re
import datetime
import calendar
__D = r'(?P<day>\d{2})'
__D2 = r'(?P<day>[ \d]\d)'
__M = r'(?P<mon>\w{3})'
__Y = r'(?P<year>\d{4})'
__Y2 = r'(?P<year>\d{2})'
__T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
def parse_http_date(date):
"""
Parses a date format as specified by HTTP RFC2616 section 3.3.1.
The three formats allowed by the RFC are accepted, even if only the first
one is still in widespread use.
Returns an floating point number expressed in seconds since the epoch, in
UTC.
"""
# emails.Util.parsedate does the job for RFC1123 dates; unfortunately
# RFC2616 makes it mandatory to support RFC850 dates too. So we roll
# our own RFC-compliant parsing.
for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
m = regex.match(date)
if m is not None:
break
else:
raise ValueError("%r is not in a valid HTTP date format" % date)
try:
year = int(m.group('year'))
if year < 100:
if year < 70:
year += 2000
else:
year += 1900
month = MONTHS.index(m.group('mon').lower()) + 1
day = int(m.group('day'))
hour = int(m.group('hour'))
min = int(m.group('min'))
sec = int(m.group('sec'))
result = datetime.datetime(year, month, day, hour, min, sec)
return calendar.timegm(result.utctimetuple())
except Exception:
raise ValueError("%r is not a valid date" % date)
def parse_http_date_safe(date):
"""
Same as parse_http_date, but returns None if the input is invalid.
"""
try:
return parse_http_date(date)
except Exception:
pass
......@@ -10,20 +10,20 @@ from django.utils.http import http_date, parse_etags
try:
from django.utils.http import parse_http_date_safe
except:
from pithos.api.util import parse_http_date_safe
from pithos.api.compat import parse_http_date_safe
from pithos.api.faults import Fault, NotModified, BadRequest, Unauthorized, ItemNotFound, Conflict, LengthRequired, PreconditionFailed, RangeNotSatisfiable, UnprocessableEntity
from pithos.api.util import get_meta, get_range, api_method
from settings import PROJECT_PATH
from os import path
STORAGE_PATH = path.join(PROJECT_PATH, 'data')
from pithos.backends.dummy import BackEnd
import os
import datetime
import logging
logging.basicConfig(level=logging.INFO)
from settings import PROJECT_PATH
STORAGE_PATH = os.path.join(PROJECT_PATH, 'data')
logger = logging.getLogger(__name__)
@api_method('GET')
def authenticate(request):
......@@ -39,9 +39,8 @@ def authenticate(request):
raise BadRequest('Missing auth user or key.')
response = HttpResponse(status = 204)
response['X-Auth-Token'] = 'eaaafd18-0fed-4b3a-81b4-663c99ec1cbb'
# TODO: Must support X-Storage-Url to be compatible.
response['X-Storage-Url'] = 'http://127.0.0.1:8000/v1/asdf'
response['X-Auth-Token'] = '0000'
response['X-Storage-Url'] = os.path.join(request.build_absolute_uri(), 'demo')
return response
def account_demux(request, v_account):
......@@ -141,11 +140,11 @@ def container_list(request, v_account):
containers = be.list_containers(request.user, marker, limit)
except NameError:
containers = []
# TODO: The cloudfiles python bindings expect 200 if json/xml.
if len(containers) == 0:
return HttpResponse(status = 204)
if request.serialization == 'text':
if len(containers) == 0:
# The cloudfiles python bindings expect 200 if json/xml.
return HttpResponse(status = 204)
return HttpResponse('\n'.join(containers), status = 200)
# TODO: Do this with a backend parameter?
......@@ -153,7 +152,6 @@ def container_list(request, v_account):
containers = [be.get_container_meta(request.user, x) for x in containers]
except NameError:
raise ItemNotFound()
# TODO: Format dates.
if request.serialization == 'xml':
data = render_to_string('containers.xml', {'account': request.user, 'containers': containers})
elif request.serialization == 'json':
......@@ -233,18 +231,11 @@ def container_delete(request, v_account, v_container):
be = BackEnd(STORAGE_PATH)
try:
info = be.get_container_meta(request.user, v_container)
be.delete_container(request.user, v_container)
except NameError:
raise ItemNotFound()
if info['count'] > 0:
except IndexError:
raise Conflict()
# TODO: Handle both exceptions.
try:
be.delete_container(request.user, v_container)
except:
raise ItemNotFound()
return HttpResponse(status = 204)
@api_method('GET', format_allowed = True)
......@@ -259,6 +250,7 @@ def object_list(request, v_account, v_container):
prefix = request.GET.get('prefix')
delimiter = request.GET.get('delimiter')
# TODO: Check if the cloudfiles python bindings expect the results with the prefix.
# Path overrides prefix and delimiter.
if path:
prefix = path
......@@ -282,11 +274,11 @@ def object_list(request, v_account, v_container):
objects = be.list_objects(request.user, v_container, prefix, delimiter, marker, limit)
except NameError:
raise ItemNotFound()
# TODO: The cloudfiles python bindings expect 200 if json/xml.
if len(objects) == 0:
return HttpResponse(status = 204)
if request.serialization == 'text':
if len(objects) == 0:
# The cloudfiles python bindings expect 200 if json/xml.
return HttpResponse(status = 204)
return HttpResponse('\n'.join(objects), status = 200)
# TODO: Do this with a backend parameter?
......@@ -294,7 +286,10 @@ def object_list(request, v_account, v_container):
objects = [be.get_object_meta(request.user, v_container, x) for x in objects]
except NameError:
raise ItemNotFound()
# TODO: Format dates.
# Format dates.
for x in objects:
if x.has_key('last_modified'):
x['last_modified'] = datetime.datetime.fromtimestamp(x['last_modified']).isoformat()
if request.serialization == 'xml':
data = render_to_string('objects.xml', {'container': v_container, 'objects': objects})
elif request.serialization == 'json':
......@@ -320,7 +315,6 @@ def object_meta(request, v_account, v_container, v_object):
response['Content-Length'] = info['bytes']
response['Content-Type'] = info['content_type']
response['Last-Modified'] = http_date(info['last_modified'])
# TODO: How should these be encoded for non-ascii?
for k in [x for x in info.keys() if x.startswith('X-Object-Meta-')]:
response[k.encode('utf-8')] = info[k].encode('utf-8')
......
......@@ -5,7 +5,6 @@
from django.conf.urls.defaults import *
# TODO: This only works when in this order.
# TODO: Define which characters can be used in each "path" component.
urlpatterns = patterns('pithos.api.functions',
(r'^$', 'authenticate'),
(r'^(?P<v_account>.+?)/(?P<v_container>.+?)/(?P<v_object>.+?)$', 'object_demux'),
......
......@@ -2,101 +2,32 @@
# Copyright (c) 2011 Greek Research and Technology Network
#
from datetime import timedelta, tzinfo
from functools import wraps
from random import choice
from string import ascii_letters, digits
from time import time
from traceback import format_exc
from wsgiref.handlers import format_date_time
from django.conf import settings
from django.http import HttpResponse
from django.template.loader import render_to_string
from django.utils import simplejson as json
from pithos.api.faults import Fault, BadRequest, ItemNotFound, ServiceUnavailable
#from synnefo.db.models import SynnefoUser, Image, ImageMetadata, VirtualMachine, VirtualMachineMetadata
from pithos.api.faults import Fault, BadRequest, ServiceUnavailable
import datetime
import dateutil.parser
import logging
import re
import calendar
# Part of newer Django versions.
__D = r'(?P<day>\d{2})'
__D2 = r'(?P<day>[ \d]\d)'
__M = r'(?P<mon>\w{3})'
__Y = r'(?P<year>\d{4})'
__Y2 = r'(?P<year>\d{2})'
__T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
def parse_http_date(date):
"""
Parses a date format as specified by HTTP RFC2616 section 3.3.1.
The three formats allowed by the RFC are accepted, even if only the first
one is still in widespread use.
Returns an floating point number expressed in seconds since the epoch, in
UTC.
"""
# emails.Util.parsedate does the job for RFC1123 dates; unfortunately
# RFC2616 makes it mandatory to support RFC850 dates too. So we roll
# our own RFC-compliant parsing.
for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
m = regex.match(date)
if m is not None:
break
else:
raise ValueError("%r is not in a valid HTTP date format" % date)
try:
year = int(m.group('year'))
if year < 100:
if year < 70:
year += 2000
else:
year += 1900
month = MONTHS.index(m.group('mon').lower()) + 1
day = int(m.group('day'))
hour = int(m.group('hour'))
min = int(m.group('min'))
sec = int(m.group('sec'))
result = datetime.datetime(year, month, day, hour, min, sec)
return calendar.timegm(result.utctimetuple())
except Exception:
raise ValueError("%r is not a valid date" % date)
def parse_http_date_safe(date):
def format_meta_key(k):
"""
Same as parse_http_date, but returns None if the input is invalid.
Convert underscores to dashes and capitalize intra-dash strings.
"""
try:
return parse_http_date(date)
except Exception:
pass
# Metadata handling.
def format_meta_key(k):
return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
def get_meta(request, prefix):
"""
Get all prefix-* request headers in a dict.
All underscores are converted to dashes.
Get all prefix-* request headers in a dict. Reformat keys with format_meta_key().
"""
prefix = 'HTTP_' + prefix.upper().replace('-', '_')
return dict([(format_meta_key(k[5:]), v) for k, v in request.META.iteritems() if k.startswith(prefix)])
# Range parsing.
def get_range(request):
"""
Parse a Range header from the request.
......@@ -139,57 +70,6 @@ def get_range(request):
return (offset, length)
# def get_vm(server_id):
# """Return a VirtualMachine instance or raise ItemNotFound."""
#
# try:
# server_id = int(server_id)
# return VirtualMachine.objects.get(id=server_id)
# except ValueError:
# raise BadRequest('Invalid server ID.')
# except VirtualMachine.DoesNotExist:
# raise ItemNotFound('Server not found.')
#
# def get_vm_meta(server_id, key):
# """Return a VirtualMachineMetadata instance or raise ItemNotFound."""
#
# try:
# server_id = int(server_id)
# return VirtualMachineMetadata.objects.get(meta_key=key, vm=server_id)
# except VirtualMachineMetadata.DoesNotExist:
# raise ItemNotFound('Metadata key not found.')
#
# def get_image(image_id):
# """Return an Image instance or raise ItemNotFound."""
#
# try:
# image_id = int(image_id)
# return Image.objects.get(id=image_id)
# except Image.DoesNotExist:
# raise ItemNotFound('Image not found.')
#
# def get_image_meta(image_id, key):
# """Return a ImageMetadata instance or raise ItemNotFound."""
#
# try:
# image_id = int(image_id)
# return ImageMetadata.objects.get(meta_key=key, image=image_id)
# except ImageMetadata.DoesNotExist:
# raise ItemNotFound('Metadata key not found.')
#
#
# def get_request_dict(request):
# """Returns data sent by the client as a python dict."""
#
# data = request.raw_post_data
# if request.META.get('CONTENT_TYPE').startswith('application/json'):
# try:
# return json.loads(data)
# except ValueError:
# raise BadRequest('Invalid JSON data.')
# else:
# raise BadRequest('Unsupported Content-Type.')
def update_response_headers(request, response):
if request.serialization == 'xml':
response['Content-Type'] = 'application/xml; charset=UTF-8'
......@@ -202,19 +82,9 @@ def update_response_headers(request, response):
response['Date'] = format_date_time(time())
def render_fault(request, fault):
if settings.DEBUG or settings.TEST:
fault.details = format_exc(fault)
# if request.serialization == 'xml':
# data = render_to_string('fault.xml', {'fault': fault})
# else:
# d = {fault.name: {'code': fault.code, 'message': fault.message, 'details': fault.details}}
# data = json.dumps(d)
# resp = HttpResponse(data, status=fault.code)
resp = HttpResponse(status = fault.code)
update_response_headers(request, resp)
return resp
response = HttpResponse(status = fault.code)
update_response_headers(request, response)
return response
def request_serialization(request, format_allowed=False):
"""
......@@ -232,13 +102,14 @@ def request_serialization(request, format_allowed=False):
elif format == 'xml':
return 'xml'
# TODO: Do we care of Accept headers?
# for item in request.META.get('HTTP_ACCEPT', '').split(','):
# accept, sep, rest = item.strip().partition(';')
# if accept == 'application/json':
# return 'json'
# elif accept == 'application/xml':
# return 'xml'
for item in request.META.get('HTTP_ACCEPT', '').split(','):
accept, sep, rest = item.strip().partition(';')
if accept == 'text/plain':
return 'text'
elif accept == 'application/json':
return 'json'
elif accept == 'application/xml' or accept == 'text/xml':
return 'xml'
return 'text'
......@@ -251,18 +122,23 @@ def api_method(http_method = None, format_allowed = False):
@wraps(func)
def wrapper(request, *args, **kwargs):
try:
if http_method and request.method != http_method:
raise BadRequest('Method not allowed.')
# The args variable may contain up to (account, container, object).
if len(args) > 1 and len(args[1]) > 256:
raise BadRequest('Container name too large.')
if len(args) > 2 and len(args[2]) > 1024:
raise BadRequest('Object name too large.')
# Fill in custom request variables.
request.serialization = request_serialization(request, format_allowed)
# TODO: Authenticate.
# TODO: Return 401/404 when the account is not found.
request.user = "test"
# TODO: Check parameter sizes.
if http_method and request.method != http_method:
raise BadRequest('Method not allowed.')
resp = func(request, *args, **kwargs)
update_response_headers(request, resp)
return resp
response = func(request, *args, **kwargs)
update_response_headers(request, response)
return response
except Fault, fault:
return render_fault(request, fault)
except BaseException, e:
......
......@@ -7,21 +7,12 @@ import hashlib
import shutil
logger = logging.getLogger(__name__)
formatter = logging.Formatter('[%(levelname)s] %(message)s')
handler = logging.FileHandler('backend.out')
handler.setFormatter(formatter)
logger.addHandler(handler)
class BackEnd:
logger = None
def __init__(self, basepath, log_file='backend.out', log_level=logging.DEBUG):
def __init__(self, basepath):
self.basepath = basepath
# TODO: Manage log_file.
logger.setLevel(log_level)
if not os.path.exists(basepath):
os.makedirs(basepath)
db = os.path.join(basepath, 'db')
......@@ -85,7 +76,7 @@ class BackEnd:
if not os.path.exists(fullname):
raise NameError('Container does not exist')
if os.listdir(fullname):
raise Exception('Container is not empty')
raise IndexError('Container is not empty')
else:
os.rmdir(fullname)
self.__del_dbpath(os.path.join(account, name))
......@@ -155,15 +146,16 @@ class BackEnd:
c = self.con.execute('select * from objects where name like ''?'' order by name', (os.path.join(prefix, '%'),))
objects = [x[0][len(prefix):] for x in c.fetchall()]
if delimiter:
pseudo_objects = {}
pseudo_objects = []
for x in objects:
pseudo_name = x
i = pseudo_name.find(delimiter)
if i != -1:
pseudo_name = pseudo_name[:i]
if pseudo_name not in pseudo_objects:
pseudo_objects.append(pseudo_name)
# TODO: Virtual directories.
pseudo_objects[pseudo_name] = x
objects = pseudo_objects.keys()
objects = pseudo_objects
start = 0
if marker:
......
from django.conf import settings
from django.core.exceptions import MiddlewareNotUsed
import logging
import logging.handlers
import logging.config
__all__ = ('LoggingConfigMiddleware',)
class LoggingConfigMiddleware:
def __init__(self):
'''Initialise the logging setup from settings, called on first request.'''
if getattr(settings, 'DEBUG', False):
logging.basicConfig(level = logging.DEBUG, format = '%(asctime)s [%(levelname)s] %(name)s %(message)s', datefmt = '%Y-%m-%d %H:%M:%S')
else:
logging.basicConfig(level = logging.INFO, format = '%(asctime)s [%(levelname)s] %(name)s %(message)s', datefmt = '%Y-%m-%d %H:%M:%S')
raise MiddlewareNotUsed('Logging setup only.')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment