Commit a29e82b5 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki

Merge branch 'master' of https://code.grnet.gr/git/pithos

parents 66fa6e34 eac8a9ed
......@@ -41,6 +41,7 @@ from django.utils.http import parse_etags
from django.utils.encoding import smart_str
from django.views.decorators.csrf import csrf_exempt
from pithos.lib.user import get_user
from pithos.lib.filter import parse_filters
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
......@@ -52,6 +53,7 @@ from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header
copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method)
from pithos.backends.base import NotAllowedError, QuotaError
from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS
import logging
import hashlib
......@@ -62,6 +64,7 @@ logger = logging.getLogger(__name__)
@csrf_exempt
def top_demux(request):
get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
if request.method == 'GET':
if getattr(request, 'user', None) is not None:
return account_list(request)
......@@ -71,6 +74,7 @@ def top_demux(request):
@csrf_exempt
def account_demux(request, v_account):
get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
if request.method == 'HEAD':
return account_meta(request, v_account)
elif request.method == 'POST':
......@@ -82,6 +86,7 @@ def account_demux(request, v_account):
@csrf_exempt
def container_demux(request, v_account, v_container):
get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
if request.method == 'HEAD':
return container_meta(request, v_account, v_container)
elif request.method == 'PUT':
......@@ -97,6 +102,7 @@ def container_demux(request, v_account, v_container):
@csrf_exempt
def object_demux(request, v_account, v_container, v_object):
get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
if request.method == 'HEAD':
return object_meta(request, v_account, v_container, v_object)
elif request.method == 'GET':
......
......@@ -36,11 +36,14 @@ import logging
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from pithos.lib.user import get_user
from pithos.api.faults import (Fault, BadRequest, ItemNotFound)
from pithos.api.util import (put_object_headers, update_manifest_meta,
validate_modification_preconditions, validate_matching_preconditions,
object_data_response, api_method)
from pithos.api.short_url import decode_url
from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS
logger = logging.getLogger(__name__)
......@@ -48,6 +51,7 @@ logger = logging.getLogger(__name__)
@csrf_exempt
def public_demux(request, v_public):
get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
if request.method == 'HEAD':
return public_meta(request, v_public)
elif request.method == 'GET':
......
#coding=utf8
from django.conf import settings
from os.path import abspath, dirname, join
PROJECT_PATH = getattr(settings, 'PROJECT_PATH', dirname(dirname(abspath(__file__))))
# Set local users, or a remote host. To disable local users set them to None.
sample_users = {
'0000': 'test',
'0001': 'verigak',
'0002': 'chazapis',
'0003': 'gtsouk',
'0004': 'papagian',
'0005': 'louridas',
'0006': 'chstath',
'0007': 'pkanavos',
'0008': 'mvasilak',
'0009': 'διογένης'
}
AUTHENTICATION_URL = getattr(settings, 'PITHOS_AUTHENTICATION_URL', 'http://127.0.0.1:8000/im/authenticate')
AUTHENTICATION_USERS = getattr(settings, 'PITHOS_AUTHENTICATION_USERS', sample_users)
# SQLAlchemy (choose SQLite/MySQL/PostgreSQL).
BACKEND_DB_MODULE = getattr(settings, 'PITHOS_BACKEND_DB_MODULE', 'pithos.backends.lib.sqlalchemy')
BACKEND_DB_CONNECTION = getattr(settings, 'PITHOS_BACKEND_DB_CONNECTION', 'sqlite:///' + join(PROJECT_PATH, 'backend.db'))
......
......@@ -119,8 +119,18 @@ def get_header_prefix(request, prefix):
# TODO: Document or remove '~' replacing.
return dict([(format_header_key(k[5:]), v.replace('~', '')) for k, v in request.META.iteritems() if k.startswith(prefix) and len(k) > len(prefix)])
def check_meta_headers(meta):
if len(meta) > 90:
raise BadRequest('Too many headers.')
for k, v in meta.iteritems():
if len(k) > 128:
raise BadRequest('Header name too large.')
if len(v) > 256:
raise BadRequest('Header value too large.')
def get_account_headers(request):
meta = get_header_prefix(request, 'X-Account-Meta-')
check_meta_headers(meta)
groups = {}
for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
n = k[16:].lower()
......@@ -151,6 +161,7 @@ def put_account_headers(response, meta, groups, policy):
def get_container_headers(request):
meta = get_header_prefix(request, 'X-Container-Meta-')
check_meta_headers(meta)
policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in get_header_prefix(request, 'X-Container-Policy-').iteritems()])
return meta, policy
......@@ -174,6 +185,7 @@ def put_container_headers(request, response, meta, policy):
def get_object_headers(request):
content_type = request.META.get('CONTENT_TYPE', None)
meta = get_header_prefix(request, 'X-Object-Meta-')
check_meta_headers(meta)
if request.META.get('HTTP_CONTENT_ENCODING'):
meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
if request.META.get('HTTP_CONTENT_DISPOSITION'):
......@@ -782,16 +794,9 @@ def get_backend():
return backend
def update_request_headers(request):
# Handle URL-encoded keys and values.
# Handle URL-encoded keys and values.
meta = dict([(k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
if len(meta) > 90:
raise BadRequest('Too many headers.')
for k, v in meta.iteritems():
if len(k) > 128:
raise BadRequest('Header name too large.')
if len(v) > 256:
raise BadRequest('Header value too large.')
try:
k.decode('ascii')
v.decode('ascii')
......
......@@ -44,9 +44,9 @@ class Groups(DBWorker):
DBWorker.__init__(self, **params)
metadata = MetaData()
columns=[]
columns.append(Column('owner', String(255), primary_key=True))
columns.append(Column('name', String(255), primary_key=True))
columns.append(Column('member', String(255), primary_key=True))
columns.append(Column('owner', String(256), primary_key=True))
columns.append(Column('name', String(256), primary_key=True))
columns.append(Column('member', String(256), primary_key=True))
self.groups = Table('groups', metadata, *columns, mysql_engine='InnoDB')
# place an index on member
......
......@@ -122,8 +122,7 @@ class Node(DBWorker):
ondelete='CASCADE',
onupdate='CASCADE'),
autoincrement=False))
path_length = 2048
columns.append(Column('path', String(path_length), default='', nullable=False))
columns.append(Column('path', String(2048), default='', nullable=False))
self.nodes = Table('nodes', metadata, *columns, mysql_engine='InnoDB')
Index('idx_nodes_path', self.nodes.c.path, unique=True)
......@@ -134,8 +133,8 @@ class Node(DBWorker):
ondelete='CASCADE',
onupdate='CASCADE'),
primary_key=True))
columns.append(Column('key', String(255), primary_key=True))
columns.append(Column('value', String(255)))
columns.append(Column('key', String(128), primary_key=True))
columns.append(Column('value', String(256)))
self.policies = Table('policy', metadata, *columns, mysql_engine='InnoDB')
#create statistics table
......@@ -159,14 +158,14 @@ class Node(DBWorker):
ForeignKey('nodes.node',
ondelete='CASCADE',
onupdate='CASCADE')))
columns.append(Column('hash', String(255)))
columns.append(Column('hash', String(256)))
columns.append(Column('size', BigInteger, nullable=False, default=0))
columns.append(Column('type', String(255), nullable=False, default=''))
columns.append(Column('type', String(256), nullable=False, default=''))
columns.append(Column('source', Integer))
columns.append(Column('mtime', DECIMAL(precision=16, scale=6)))
columns.append(Column('muser', String(255), nullable=False, default=''))
columns.append(Column('muser', String(256), nullable=False, default=''))
columns.append(Column('uuid', String(64), nullable=False, default=''))
columns.append(Column('checksum', String(255), nullable=False, default=''))
columns.append(Column('checksum', String(256), nullable=False, default=''))
columns.append(Column('cluster', Integer, nullable=False, default=0))
self.versions = Table('versions', metadata, *columns, mysql_engine='InnoDB')
Index('idx_versions_node_mtime', self.versions.c.node, self.versions.c.mtime)
......@@ -179,9 +178,9 @@ class Node(DBWorker):
ondelete='CASCADE',
onupdate='CASCADE'),
primary_key=True))
columns.append(Column('domain', String(255), primary_key=True))
columns.append(Column('key', String(255), primary_key=True))
columns.append(Column('value', String(255)))
columns.append(Column('domain', String(256), primary_key=True))
columns.append(Column('key', String(128), primary_key=True))
columns.append(Column('value', String(256)))
self.attributes = Table('attributes', metadata, *columns, mysql_engine='InnoDB')
metadata.create_all(self.engine)
......
......@@ -62,7 +62,7 @@ class XFeatures(DBWorker):
primary_key=True))
columns.append(Column('key', Integer, primary_key=True,
autoincrement=False))
columns.append(Column('value', String(255), primary_key=True))
columns.append(Column('value', String(256), primary_key=True))
self.xfeaturevals = Table('xfeaturevals', metadata, *columns, mysql_engine='InnoDB')
metadata.create_all(self.engine)
......
......@@ -428,19 +428,22 @@ class ModularBackend(BaseBackend):
if user != account and until:
raise NotAllowedError
allowed = self._list_object_permissions(user, account, container, prefix, shared)
if shared and not allowed:
return []
path, node = self._lookup_container(account, container)
allowed = self._get_formatted_paths(allowed)
return self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, allowed, all_props)
def _list_object_permissions(self, user, account, container, prefix, shared):
allowed = []
path = '/'.join((account, container, prefix)).rstrip('/')
if user != account:
allowed = self.permissions.access_list_paths(user, '/'.join((account, container, prefix)))
allowed = self.permissions.access_list_paths(user, path)
if not allowed:
raise NotAllowedError
else:
if shared:
allowed = self.permissions.access_list_shared('/'.join((account, container, prefix)))
allowed = self.permissions.access_list_shared(path)
if not allowed:
return []
return allowed
......
......@@ -32,23 +32,30 @@
# or implied, of GRNET S.A.
from time import time, mktime
from httplib import HTTPConnection
from urlparse import urlparse
from httplib import HTTPConnection, HTTPSConnection
from urllib import quote, unquote
from django.conf import settings
from django.utils import simplejson as json
def authenticate(authentication_host, token):
con = HTTPConnection(authentication_host)
def authenticate(token, authentication_url='http://127.0.0.1:8000/im/authenticate'):
p = urlparse(authentication_url)
if p.scheme == 'http':
conn = HTTPConnection(p.netloc)
elif p.scheme == 'https':
conn = HTTPSConnection(p.netloc)
else:
raise Exception('Unknown URL scheme')
kwargs = {}
kwargs['headers'] = {}
kwargs['headers']['X-Auth-Token'] = token
kwargs['headers']['Content-Length'] = 0
path = '/im/authenticate'
con.request('GET', path, **kwargs)
response = con.getresponse()
conn.request('GET', p.path, **kwargs)
response = conn.getresponse()
headers = response.getheaders()
headers = dict((unquote(h), unquote(v)) for h,v in headers)
......@@ -61,34 +68,31 @@ def authenticate(authentication_host, token):
return json.loads(data)
def get_user_from_token(token):
def user_for_token(token, authentication_url, override_users):
if not token:
return None
users = settings.AUTHENTICATION_USERS
if users is not None:
if override_users:
try:
return {'id': 0, 'uniq': users[token].decode('utf8')}
return {'uniq': override_users[token].decode('utf8')}
except:
return None
host = settings.AUTHENTICATION_HOST
try:
return authenticate(host, token)
return authenticate(token, authentication_url)
except:
return None
class UserMiddleware(object):
def process_request(self, request):
request.user = None
request.user_uniq = None
# Try to find token in a parameter, in a request header, or in a cookie.
user = get_user_from_token(request.GET.get('X-Auth-Token'))
if not user:
user = get_user_from_token(request.META.get('HTTP_X_AUTH_TOKEN'))
if not user:
return
request.user = user
request.user_uniq = user['uniq']
def get_user(request, authentication_url='http://127.0.0.1:8000/im/authenticate', override_users={}):
request.user = None
request.user_uniq = None
# Try to find token in a parameter or in a request header.
user = user_for_token(request.GET.get('X-Auth-Token'), authentication_url, override_users)
if not user:
user = user_for_token(request.META.get('HTTP_X_AUTH_TOKEN'), authentication_url, override_users)
if not user:
return
request.user = user
request.user_uniq = user['uniq']
from log import LoggingConfigMiddleware
from secure import SecureMiddleware
from user import UserMiddleware
......@@ -72,8 +72,7 @@ MIDDLEWARE_CLASSES = (
#'django.contrib.auth.middleware.AuthenticationMiddleware',
#'django.contrib.messages.middleware.MessageMiddleware',
'pithos.middleware.LoggingConfigMiddleware',
'pithos.middleware.SecureMiddleware',
'pithos.middleware.UserMiddleware'
'pithos.middleware.SecureMiddleware'
)
ROOT_URLCONF = 'pithos.urls'
......@@ -123,24 +122,6 @@ USE_X_FORWARDED_HOST = False
# Set umask (needed for gunicorn setup).
#umask(0077)
# Either set local users here, or a remote host.
# To disable local users set to None.
AUTHENTICATION_USERS = {
'0000': 'test',
'0001': 'verigak',
'0002': 'chazapis',
'0003': 'gtsouk',
'0004': 'papagian',
'0005': 'louridas',
'0006': 'chstath',
'0007': 'pkanavos',
'0008': 'mvasilak',
'0009': 'διογένης'
}
# Where astakos is hosted.
AUTHENTICATION_HOST = '127.0.0.1:10000'
conf = join(PROJECT_PATH, 'settings.local')
if exists(conf):
execfile(conf)
......
......@@ -1416,7 +1416,7 @@ class ObjectPost(BaseTestCase):
self.containers[0],
self.obj[0]['name']):
#perform update metadata
more = {'foo':'foo', 'bar':'bar'}
more = {'foo': 'foo', 'bar': 'bar', 'f' * 114: 'b' * 256}
status = self.client.update_object_metadata(self.containers[0],
self.obj[0]['name'],
**more)[0]
......@@ -1431,6 +1431,13 @@ class ObjectPost(BaseTestCase):
for k,v in more.items():
self.assertTrue(k in headers.keys())
self.assertTrue(headers[k], v)
#out of limits
more = {'f' * 114: 'b' * 257}
self.assert_raises_fault(400, self.client.update_object_metadata,
self.containers[0],
self.obj[0]['name'],
**more)
def test_update_object(self,
first_byte_pos=0,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment