Commit af486b4c authored by Antony Chazapis's avatar Antony Chazapis
Browse files

Allow for account/container metadata.

parent 21b2eeb0
......@@ -13,7 +13,7 @@ except:
from pithos.api.util import parse_http_date_safe
from pithos.api.faults import Fault, NotModified, BadRequest, Unauthorized, ItemNotFound, LengthRequired, PreconditionFailed, RangeNotSatisfiable, UnprocessableEntity
from pithos.api.util import get_object_meta, get_range, api_method
from pithos.api.util import get_meta, get_range, api_method
from settings import PROJECT_PATH
from os import path
......@@ -50,6 +50,8 @@ def account_demux(request, v_account):
return account_meta(request, v_account)
elif request.method == 'GET':
return container_list(request, v_account)
elif request.method == 'POST':
return account_update(request, v_account)
else:
return method_not_allowed(request)
......@@ -60,6 +62,8 @@ def container_demux(request, v_account, v_container):
return object_list(request, v_account, v_container)
elif request.method == 'PUT':
return container_create(request, v_account, v_container)
elif request.method == 'POST':
return container_update(request, v_account, v_container)
elif request.method == 'DELETE':
return container_delete(request, v_account, v_container)
else:
......@@ -93,12 +97,30 @@ def account_meta(request, v_account):
info = be.get_account_meta(request.user)
except NameError:
info = {'count': 0, 'bytes': 0}
response = HttpResponse(status = 204)
response['X-Account-Container-Count'] = info['count']
response['X-Account-Bytes-Used'] = info['bytes']
for k in [x for x in info.keys() if x.startswith('X-Account-Meta-')]:
response[k] = info[k]
return response
@api_method('POST')
def account_update(request, v_account):
# Normal Response Codes: 202
# Error Response Codes: serviceUnavailable (503),
# itemNotFound (404),
# unauthorized (401),
# badRequest (400)
meta = get_meta(request, 'X-Account-Meta-')
be = BackEnd(STORAGE_PATH)
be.update_account_meta(request.user, meta)
return HttpResponse(status = 202)
@api_method('GET', format_allowed = True)
def container_list(request, v_account):
# Normal Response Codes: 200, 204
......@@ -120,6 +142,7 @@ def container_list(request, v_account):
containers = be.list_containers(request.user, marker, limit)
except NameError:
containers = []
# TODO: The cloudfiles python bindings expect 200 if json/xml.
if len(containers) == 0:
return HttpResponse(status = 204)
......@@ -134,7 +157,7 @@ def container_list(request, v_account):
if request.serialization == 'xml':
data = render_to_string('containers.xml', {'account': request.user, 'containers': containers})
elif request.serialization == 'json':
data = json.dumps(containers)
data = json.dumps(containers)
return HttpResponse(data, status = 200)
@api_method('HEAD')
......@@ -154,6 +177,9 @@ def container_meta(request, v_account, v_container):
response = HttpResponse(status = 204)
response['X-Container-Object-Count'] = info['count']
response['X-Container-Bytes-Used'] = info['bytes']
for k in [x for x in info.keys() if x.startswith('X-Container-Meta-')]:
response[k] = info[k]
return response
@api_method('PUT')
......@@ -163,13 +189,38 @@ def container_create(request, v_account, v_container):
# itemNotFound (404),
# unauthorized (401),
# badRequest (400)
meta = get_meta(request, 'X-Container-Meta-')
be = BackEnd(STORAGE_PATH)
try:
be.create_container(request.user, v_container)
return HttpResponse(status = 201)
ret = 201
except NameError:
return HttpResponse(status = 202)
ret = 202
if len(meta) > 0:
be.update_container_meta(request.user, v_container, meta)
return HttpResponse(status = ret)
@api_method('POST')
def container_update(request, v_account, v_container):
# Normal Response Codes: 202
# Error Response Codes: serviceUnavailable (503),
# itemNotFound (404),
# unauthorized (401),
# badRequest (400)
meta = get_meta(request, 'X-Container-Meta-')
be = BackEnd(STORAGE_PATH)
try:
be.update_container_meta(request.user, v_container, meta)
except NameError:
raise ItemNotFound()
return HttpResponse(status = 202)
@api_method('DELETE')
def container_delete(request, v_account, v_container):
......@@ -195,7 +246,7 @@ def container_delete(request, v_account, v_container):
raise ItemNotFound()
return HttpResponse(status = 204)
# --- UP TO HERE ---
# --- MERGED UP TO HERE ---
@api_method('GET', format_allowed = True)
def object_list(request, v_account, v_container):
......@@ -362,7 +413,7 @@ def object_write(request, v_account, v_container, v_object):
if not content_length or not content_type:
raise LengthRequired()
meta = get_object_meta(request)
meta = get_meta(request, 'X-Object-Meta-')
info = {'bytes': content_length, 'content_type': content_type, 'meta': meta}
etag = request.META.get('HTTP_ETAG')
......@@ -400,14 +451,14 @@ def object_copy(request, v_account, v_container, v_object):
raise BadRequest('Bad Destination path.')
dest_container = parts[1]
dest_name = '/'.join(parts[2:])
info = get_object_meta(request.user, v_container, v_object)
content_type = request.META.get('CONTENT_TYPE')
if content_type:
info['content_type'] = content_type
meta = get_object_meta(request)
meta = get_meta(request, 'X-Object-Meta-')
for k, v in meta.iteritems():
info['meta'][k] = v
......@@ -424,7 +475,7 @@ def object_update(request, v_account, v_container, v_object):
# unauthorized (401),
# badRequest (400)
meta = get_object_meta(request)
meta = get_meta(request, 'X-Object-Meta-')
update_object_meta(request.user, v_container, v_object, meta)
return HttpResponse(status = 202)
......
......@@ -82,13 +82,21 @@ def parse_http_date_safe(date):
except Exception:
pass
def get_object_meta(request):
# Metadata handling.
def format_meta_key(k):
return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
def get_meta(request, prefix):
"""
Get all X-Object-Meta-* headers in a dict.
Get all prefix-* request headers in a dict.
All underscores are converted to dashes.
"""
prefix = 'HTTP_X_OBJECT_META_'
return dict([(k[len(prefix):].lower(), v) for k, v in request.META.iteritems() if k.startswith(prefix)])
prefix = 'HTTP_' + prefix.upper().replace('-', '_')
return dict([(format_meta_key(k[5:]), v) for k, v in request.META.iteritems() if k.startswith(prefix)])
# Range parsing.
def get_range(request):
"""
Parse a Range header from the request.
......
......@@ -3,17 +3,23 @@ import sqlite3
import json
import logging
import types
import hashlib
logger = logging.getLogger(__name__)
formatter = logging.Formatter('[%(levelname)s] %(message)s')
handler = logging.FileHandler('backend.out')
handler.setFormatter(formatter)
logger.addHandler(handler)
class BackEnd:
logger = None
def __init__(self, basepath, log_file='backend.out', log_level=logging.DEBUG):
self.basepath = basepath
self.logger = logging.getLogger(__name__)
self.logger.setLevel(log_level)
formatter = logging.Formatter('[%(levelname)s] %(message)s')
handler = logging.FileHandler(log_file)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
# TODO: Manage log_file.
logger.setLevel(log_level)
if not os.path.exists(basepath):
os.makedirs(basepath)
......@@ -25,39 +31,54 @@ class BackEnd:
sql = '''create table if not exists metadata(object_id int, name text, value text)'''
self.con.execute(sql)
self.con.commit()
# TODO: Create/delete account?
def get_account_meta(self, account):
"""
returns a dictionary with the container metadata
returns a dictionary with the account metadata
"""
self.logger.debug("get_account_meta: %s", account)
logger.debug("get_account_meta: %s", account)
fullname = os.path.join(self.basepath, account)
if not os.path.exists(fullname):
raise NameError('Account does not exist')
contents = os.listdir(fullname)
contents = os.listdir(fullname)
count = len(contents)
size = sum(os.path.getsize(os.path.join(self.basepath, account, objectname)) for objectname in contents)
return {'name': account, 'count': count, 'bytes': size}
meta = self.__get_metadata(account)
meta.update({'name': account, 'count': count, 'bytes': size})
return meta
def update_account_meta(self, account, meta):
"""
updates the metadata associated with the account
"""
logger.debug("update_account_meta: %s %s", account, meta)
fullname = os.path.join(self.basepath, account)
if not os.path.exists(fullname):
os.makedirs(fullname)
self.__put_metadata(account, meta)
return
def create_container(self, account, name):
"""
creates a new container with the given name
if it doesn't exist under the basepath
"""
self.logger.debug("create_container: %s %s", account, name)
logger.debug("create_container: %s %s", account, name)
fullname = os.path.join(self.basepath, account, name)
if not os.path.exists(fullname):
os.makedirs(fullname)
else:
raise NameError('Container already exists')
return
def delete_container(self, account, name):
"""
deletes the container with the given name
if it exists under the basepath and is empty
"""
self.logger.debug("delete_container: %s %s", account, name)
logger.debug("delete_container: %s %s", account, name)
fullname = os.path.join(self.basepath, account, name)
if not os.path.exists(fullname):
raise NameError('Container does not exist')
......@@ -65,31 +86,45 @@ class BackEnd:
raise Exception('Container is not empty')
else:
os.rmdir(fullname)
self.__del_dbpath(os.path.join(account, name))
return
def get_container_meta(self, account, name):
"""
returns a dictionary with the container metadata
"""
self.logger.debug("get_container_meta: %s %s", account, name)
logger.debug("get_container_meta: %s %s", account, name)
fullname = os.path.join(self.basepath, account, name)
if not os.path.exists(fullname):
raise NameError('Container does not exist')
contents = os.listdir(fullname)
contents = os.listdir(fullname)
count = len(contents)
size = sum(os.path.getsize(os.path.join(self.basepath, account, name, objectname)) for objectname in contents)
return {'name': name, 'count': count, 'bytes': size}
meta = self.__get_metadata(os.path.join(account, name))
meta.update({'name': name, 'count': count, 'bytes': size})
return meta
def update_container_meta(self, account, name, meta):
"""
updates the metadata associated with the container
"""
logger.debug("update_container_meta: %s %s %s", account, name, meta)
fullname = os.path.join(self.basepath, account, name)
if not os.path.exists(fullname):
raise NameError('Container does not exist')
self.__put_metadata(os.path.join(account, name), meta)
return
def list_containers(self, account, marker = None, limit = 10000):
"""
returns a list of at most limit (default = 10000) containers
starting from the next item after the optional marker
"""
self.logger.debug("list_containers: %s %s %s", account, marker, limit)
logger.debug("list_containers: %s %s %s", account, marker, limit)
fullname = os.path.join(self.basepath, account)
if not os.path.exists(fullname):
raise NameError('Account does not exist')
containers = os.listdir(fullname)
containers = os.listdir(fullname)
start = 0
if marker:
try:
......@@ -98,13 +133,63 @@ class BackEnd:
pass
return containers[start:limit]
# --- UP TO HERE ---
# def __get_linkinfo(self, path):
# c = self.con.execute('select rowid from objects where name=''?''', (path,))
# row = c.fetchone()
# if row:
# return str(row[0])
# else:
# raise NameError('Requested path not found')
#
# def __put_linkinfo(self, path):
# id = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid
# self.con.commit()
# return id
def __del_dbpath(self, path):
self.con.execute('delete from metadata where object_id in (select rowid from objects where name = ''?'')', (path,))
self.con.execute('delete from objects where name = ''?''', (path,))
self.con.commit()
return
def __get_metadata(self, path):
c = self.con.execute('select m.name, m.value from metadata m, objects o where o.rowid = m.object_id and o.name = ''?''', (path,))
return dict(c.fetchall())
def __put_metadata(self, path, meta):
c = self.con.execute('select rowid from objects where name=''?''', (path,))
row = c.fetchone()
if row:
link = str(row[0])
else:
link = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid
for k, v in meta.iteritems():
self.con.execute('insert or replace into metadata (object_id, name, value) values (?, ?, ?)', (link, k, v))
self.con.commit()
return
def __object_hash(self, location, block_size = 8192):
md5 = hashlib.md5()
f = open(location, 'r')
while True:
data = f.read(block_size)
if not data:
break
md5.update(data)
f.close()
return md5.hexdigest()
# --- MERGED UP TO HERE ---
def list_objects(self, account, container, prefix='', delimiter=None, marker = None, limit = 10000):
"""
returns a list of the objects existing under a specific account container
"""
self.logger.info("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
logger.info("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
dir = os.path.join(self.basepath, account, container)
if not os.path.exists(dir):
raise NameError('Container does not exist')
......@@ -230,7 +315,7 @@ class BackEnd:
f = open(location, 'w')
f.write(data)
f.close()
def __delete_data(self, location, account, container):
file = os.path.join(self.basepath, account, container, location)
if not os.path.exists(dir):
......
......@@ -75,13 +75,13 @@ SECRET_KEY = '$j0cdrfm*0sc2j+e@@2f-&3-_@2=^!z#+b-8o4_i10@2%ev7si'
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
# 'django.template.loaders.eggs.Loader',
# 'django.template.loaders.eggs.Loader',
)
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
# 'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment