Commit b7f3b202 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

Change pithos public implementation

Public URLs should consist of a random part.
A public URL should be unique and should not be
equal to any deleted public URL.
Finally publishing a public object has no effect.
Therefore, to generate a new public URL for an
object, the object should be unpublished and 
published again.
parent ab6dabca
......@@ -166,7 +166,11 @@ class ObjectMigration(Migration):
self.create_tags(headerid, nodeid, vserials)
#set object's publicity
if public:
self.backend.permissions.public_set(object)
self.backend.permissions.public_set(
object,
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
#set object's permissions
self.create_permissions(headerid, object, username, is_folder=False)
......
......@@ -27,37 +27,39 @@ Settings
Configure in ``settings.py`` or a ``.conf`` file in ``/etc/synnefo`` if using snf-webproject.
=============================== ================================================= ============================================================
Name Default value Description
=============================== ================================================= ============================================================
PITHOS_AUTHENTICATION_URL \https://<astakos.host>/im/authenticate/ Astakos Authentication URL
PITHOS_AUTHENTICATION_USERS A dictionary of sample users (token to username) Set to empty or None to disable
PITHOS_ASTAKOS_COOKIE_NAME _pithos2_a Cookie name to retrieve fallback token
=============================== ================================================================ ============================================================
Name Default value Description
=============================== ================================================================ ============================================================
PITHOS_AUTHENTICATION_URL \https://<astakos.host>/im/authenticate/ Astakos Authentication URL
PITHOS_AUTHENTICATION_USERS A dictionary of sample users (token to username) Set to empty or None to disable
PITHOS_ASTAKOS_COOKIE_NAME _pithos2_a Cookie name to retrieve fallback token
PITHOS_BACKEND_DB_MODULE pithos.backends.lib.sqlalchemy
PITHOS_BACKEND_DB_CONNECTION sqlite:////tmp/pithos-backend.db SQLAlchemy database connection string
PITHOS_BACKEND_BLOCK_MODULE pithos.backends.lib.hashfiler
PITHOS_BACKEND_BLOCK_PATH /tmp/pithos-data/ Map and block storage path
PITHOS_BACKEND_BLOCK_UMASK 0o022 Map and block storage umask
PITHOS_BACKEND_QUEUE_MODULE None Use ``pithos.backends.lib.rabbitmq`` to enable
PITHOS_BACKEND_QUEUE_HOSTS None Format like [``amqp://guest:guest@localhost:5672``
PITHOS_BACKEND_DB_CONNECTION sqlite:////tmp/pithos-backend.db SQLAlchemy database connection string
PITHOS_BACKEND_BLOCK_MODULE pithos.backends.lib.hashfile
PITHOS_BACKEND_BLOCK_PATH /tmp/pithos-data/ Map and block storage path
PITHOS_BACKEND_BLOCK_UMASK 0o022 Map and block storage umask
PITHOS_BACKEND_QUEUE_MODULE None Use ``pithos.backends.lib.rabbitmq`` to enable
PITHOS_BACKEND_QUEUE_HOSTS None Format like [``amqp://guest:guest@localhost:5672``
PITHOS_BACKEND_QUEUE_EXCHANGE pithos
PITHOS_BACKEND_QUOTA 50 GB (50 * 1024 ** 3) Default user quota
PITHOS_BACKEND_VERSIONING auto Default versioning policy for containers
PITHOS_BACKEND_FREE_VERSIONING True Default versioning debit policy (default free)
PITHOS_UPDATE_MD5 True Update object checksums when using hashmaps
PITHOS_SERVICE_TOKEN '' Service token acquired by the identity provider (astakos)
PITHOS_RADOS_STORAGE False Enables or disables secondary Pithos storage on RADOS
PITHOS_RADOS_POOL_BLOCKS None RADOS pool to be used for block storage
PITHOS_RADOS_POOL_MAPS None RADOS pool to be used for maps storage
PITHOS_TRANSLATE_UUIDS False Enables a ui compatibility layer for the introduction of UUIDs in identity management.
PITHOS_PROXY_USER_SERVICES True Whether to proxy user feedback and catalog services
PITHOS_USER_CATALOG_URL \https://<astakos.host>/user_catalogs/ Astakos User Catalog URL
PITHOS_USER_FEEDBACK_URL \https://<astakos.host>/feedback/ Astakos User Feedback URL
PITHOS_USER_LOGIN_URL \https://<astakos.host>/login/ Astakos User Login URL
PITHOS_USE_QUOTAHOLDER True Enable quotaholder
PITHOS_QUOTAHOLDER_URL '' Quotaholder URL
PITHOS_QUOTAHOLDER_TOKEN '' Quotaholder token
=============================== ================================================= ============================================================
PITHOS_BACKEND_QUOTA 50 GB (50 * 1024 ** 3) Default user quota
PITHOS_BACKEND_VERSIONING auto Default versioning policy for containers
PITHOS_BACKEND_FREE_VERSIONING True Default versioning debit policy (default free)
PITHOS_UPDATE_MD5 True Update object checksums when using hashmaps
PITHOS_SERVICE_TOKEN '' Service token acquired by the identity provider (astakos)
PITHOS_RADOS_STORAGE False Enables or disables secondary Pithos storage on RADOS
PITHOS_RADOS_POOL_BLOCKS None RADOS pool to be used for block storage
PITHOS_RADOS_POOL_MAPS None RADOS pool to be used for maps storage
PITHOS_TRANSLATE_UUIDS False Enables a ui compatibility layer for the introduction of UUIDs in identity management.
PITHOS_PROXY_USER_SERVICES True Whether to proxy user feedback and catalog services
PITHOS_USER_CATALOG_URL \https://<astakos.host>/user_catalogs/ Astakos User Catalog URL
PITHOS_USER_FEEDBACK_URL \https://<astakos.host>/feedback/ Astakos User Feedback URL
PITHOS_USER_LOGIN_URL \https://<astakos.host>/login/ Astakos User Login URL
PITHOS_USE_QUOTAHOLDER True Enable quotaholder
PITHOS_QUOTAHOLDER_URL '' Quotaholder URL
PITHOS_QUOTAHOLDER_TOKEN '' Quotaholder token
PITHOS_PUBLIC_URL_MIN_LENGTH 8 Public URL minimun length
PITHOS_PUBLIC_URL_ALPHABET '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' Public URL alphabet
=============================== ================================================================ ============================================================
To update checksums asynchronously, enable the queue, install snf-pithos-tools and use ``pithos-dispatcher``::
......
......@@ -51,7 +51,10 @@
#
# Tune the size of the http pool for the quotaholder client.
# It limits the maximum number of quota changing requests
# that pithos can serve. Extra requests will be blocked
# that pithos can serve. Extra requests will be blocked
# until another has completed.
#
#PITHOS_QUOTAHOLDER_POOLSIZE = 200
#
# Set public url length
#PITHOS_PUBLIC_URL_MIN_LENGTH = 8
......@@ -44,7 +44,6 @@ from pithos.api.util import (put_object_headers, update_manifest_meta,
validate_matching_preconditions,
object_data_response, api_method,
split_container_object_string)
from pithos.api.short_url import decode_url
from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS
......@@ -72,7 +71,7 @@ def public_meta(request, v_public):
try:
v_account, v_container, v_object = request.backend.get_public(
request.user_uniq,
decode_url(v_public))
v_public)
meta = request.backend.get_object_meta(request.user_uniq, v_account,
v_container, v_object, 'pithos')
public = request.backend.get_object_public(
......@@ -99,10 +98,11 @@ def public_read(request, v_public):
# itemNotFound (404),
# badRequest (400),
# notModified (304)
try:
v_account, v_container, v_object = request.backend.get_public(
request.user_uniq,
decode_url(v_public))
v_public)
meta = request.backend.get_object_meta(request.user_uniq, v_account,
v_container, v_object, 'pithos')
public = request.backend.get_object_public(
......
......@@ -65,3 +65,11 @@ USE_QUOTAHOLDER = getattr(settings, 'PITHOS_USE_QUOTAHOLDER', False)
QUOTAHOLDER_URL = getattr(settings, 'PITHOS_QUOTAHOLDER_URL', '')
QUOTAHOLDER_TOKEN = getattr(settings, 'PITHOS_QUOTAHOLDER_TOKEN', '')
QUOTAHOLDER_POOLSIZE = getattr(settings, 'PITHOS_QUOTAHOLDER_POOLSIZE', 200)
# Set public url length and alphabet
PUBLIC_URL_MIN_LENGTH = getattr(settings, 'PITHOS_PUBLIC_URL_MIN_LENGTH', 8)
PUBLIC_URL_ALPHABET = getattr(
settings,
'PITHOS_PUBLIC_URL_ALPHABET',
'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
)
......@@ -194,7 +194,11 @@ class SwissArmy():
if public:
# set destination object public
fullpath = '/'.join([dest_account, dest_container, dest_name])
self.backend.permissions.public_set(fullpath)
self.backend.permissions.public_set(
fullpath,
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
def _merge_account(self, src_account, dest_account, delete_src=False):
# TODO: handle exceptions
......
# Copyright 2011 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import unittest
import random
import string
import datetime
import time as _time
import pithos.api.settings as settings
from pithos.api.swiss_army import SwissArmy
def get_random_data(length=500):
char_set = string.ascii_uppercase + string.digits
return ''.join(random.choice(char_set) for x in xrange(length))
class TestPublic(unittest.TestCase):
def setUp(self):
self.utils = SwissArmy()
self.backend = self.utils.backend
self.utils.create_account('account')
def tearDown(self):
self.utils._delete_account('account')
self.utils.cleanup()
def assert_not_public_object(self, account, container, object):
public = self.backend.get_object_public(
account, account, container, object
)
self.assertTrue(public == None)
self.assertRaises(
NameError,
self.backend.get_public,
'$$account$$',
public
)
self.assertRaises(
Exception, self.backend._can_read,
'$$account$$', account, container, object
)
return public
def assert_public_object(self, account, container, object):
public = self.backend.get_object_public(
account, account, container, object
)
self.assertTrue(public != None)
self.assertTrue(len(public) >= settings.PUBLIC_URL_MIN_LENGTH)
self.assertTrue(set(public) <= set(settings.PUBLIC_URL_ALPHABET))
self.assertEqual(
self.backend.get_public('$$account$$', public),
(account, container, object)
)
try:
self.backend._can_read('$$account$$', account, container, object)
except Exception:
self.fail('Public object should be readable.')
return public
def test_set_object_public(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
self.assert_public_object('account', 'container', 'object')
def test_set_twice(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
public2 = self.assert_public_object('account', 'container', 'object')
self.assertEqual(public, public2)
def test_set_unset_set(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_unset('account/container/object')
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
public3 = self.assert_public_object('account', 'container', 'object')
self.assertTrue(public != public3)
def test_update_object_public(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=False
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=True
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=False
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=True
)
new_public = self.assert_public_object('account', 'container', 'object')
self.assertTrue(public != new_public)
def test_delete_not_public_object(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.delete_object('account', 'account', 'container', 'object')
self.assert_not_public_object('account', 'container', 'object')
def test_delete_public_object(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
self.assert_public_object('account', 'container', 'object')
self.backend.delete_object('account', 'account', 'container', 'object')
self.assert_not_public_object('account', 'container', 'object')
def test_delete_public_object_history(self):
self.utils.backend.put_container('account', 'account', 'container')
for i in range(random.randint(1, 10)):
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
_time.sleep(1)
versions = self.backend.list_versions(
'account', 'account', 'container', 'object'
)
mtime = [int(i[1]) for i in versions]
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
i = random.randrange(len(mtime))
self.backend.delete_object(
'account', 'account', 'container', 'object', until=mtime[i]
)
self.assert_public_object('account', 'container', 'object')
public = self.assert_public_object('account', 'container', 'object')
_time.sleep(1)
t = datetime.datetime.utcnow()
now = int(_time.mktime(t.timetuple()))
self.backend.delete_object(
'account', 'account', 'container', 'object', until=now
)
self.assertRaises(
NameError,
self.backend.get_public,
'$$account$$',
public
)
if __name__ == '__main__':
unittest.main()
......@@ -55,7 +55,6 @@ from pithos.api.faults import (
Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound,
Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge,
RangeNotSatisfiable, InternalServerError, NotImplemented)
from pithos.api.short_url import encode_url
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
BACKEND_BLOCK_UMASK,
......@@ -68,7 +67,9 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
AUTHENTICATION_URL, AUTHENTICATION_USERS,
COOKIE_NAME, USER_CATALOG_URL,
RADOS_STORAGE, RADOS_POOL_BLOCKS,
RADOS_POOL_MAPS, TRANSLATE_UUIDS)
RADOS_POOL_MAPS, TRANSLATE_UUIDS,
PUBLIC_URL_MIN_LENGTH,
PUBLIC_URL_ALPHABET)
from pithos.backends import connect_backend
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
VersionNotExists)
......@@ -387,7 +388,7 @@ def update_sharing_meta(request, permissions, v_account, v_container, v_object,
def update_public_meta(public, meta):
if not public:
return
meta['X-Object-Public'] = '/public/' + encode_url(public)
meta['X-Object-Public'] = '/public/' + public
def validate_modification_preconditions(request, meta):
......@@ -983,7 +984,9 @@ _pithos_backend_pool = PithosBackendPool(
quotaholder_token=QUOTAHOLDER_TOKEN,
quotaholder_client_poolsize=QUOTAHOLDER_POOLSIZE,
free_versioning=BACKEND_FREE_VERSIONING,
block_params=BLOCK_PARAMS)
block_params=BLOCK_PARAMS,
public_url_min_length=PUBLIC_URL_MIN_LENGTH,
public_url_alphabet=PUBLIC_URL_ALPHABET)
def get_backend():
backend = _pithos_backend_pool.pool_get()
......
"""alter public add column url
Revision ID: 27381099d477
Revises: 2a309a9a3438
Create Date: 2013-03-20 16:14:20.058077
"""
# revision identifiers, used by Alembic.
revision = '27381099d477'
down_revision = '2a309a9a3438'
from alembic import op
import sqlalchemy as sa
from pithos.backends.modular import ULTIMATE_ANSWER
from pithos.api.short_url import encode_url
def upgrade():
op.add_column('public', sa.Column('url', sa.String(2048)))
op.create_unique_constraint('idx_public_url', 'public', ['url'])
# migrate old rows
p = sa.sql.table(
'public',
sa.sql.column('public_id', sa.Integer),
sa.sql.column('url', sa.String),
)
get_url = lambda x: encode_url(x + ULTIMATE_ANSWER)
conn = op.get_bind()
s = sa.select([p.c.public_id])
rows = conn.execute(s).fetchall()
for r in rows:
s = p.update().values(url=get_url(r[0])).where(p.c.public_id==r[0])
op.execute(s)
def downgrade():
op.drop_constraint('idx_public_url', 'public')
op.drop_column('public', 'url')
......@@ -37,6 +37,7 @@ from sqlalchemy.sql import and_, select
from sqlalchemy.schema import Index
from sqlalchemy.exc import NoSuchTableError
from pithos.backends.random_word import get_word
def create_tables(engine):
metadata = MetaData()
......@@ -44,10 +45,13 @@ def create_tables(engine):
columns.append(Column('public_id', Integer, primary_key=True))
columns.append(Column('path', String(2048), nullable=False))
columns.append(Column('active', Boolean, nullable=False, default=True))
columns.append(Column('url', String(2048), nullable=True))
public = Table('public', metadata, *columns, mysql_engine='InnoDB',
sqlite_autoincrement=True)
# place an index on path
Index('idx_public_path', public.c.path, unique=True)
# place an index on url
Index('idx_public_url', public.c.url, unique=True)
metadata.create_all(engine)
return metadata.sorted_tables
......@@ -64,39 +68,49 @@ class Public(DBWorker):
tables = create_tables(self.engine)
map(lambda t: self.__setattr__(t.name, t), tables)
def public_set(self, path):
def get_unique_url(self, serial, public_url_min_length, public_url_alphabet):
l = public_url_min_length
while 1:
candidate = get_word(serial, length=l, alphabet=public_url_alphabet)
if self.public_path(candidate) is None:
return candidate
l +=1
def public_set(self, path, public_url_min_length, public_url_alphabet):
s = select([self.public.c.public_id])
s = s.where(self.public.c.path == path)
r = self.conn.execute(s)
row = r.fetchone()
r.close()
if row:
s = self.public.update().where(self.public.c.public_id == row[0])
s = s.values(active=True)
else:
if not row:
s = self.public.insert()
s = s.values(path=path, active=True)
r = self.conn.execute(s)
r.close()
r = self.conn.execute(s)
serial = r.inserted_primary_key[0]
r.close()
url = self.get_unique_url(
serial, public_url_min_length, public_url_alphabet
)
s = self.public.update().where(self.public.c.public_id == serial)
s = s.values(url=url)
self.conn.execute(s).close()
def public_unset(self, path):
s = self.public.update()
s = self.public.delete()
s = s.where(self.public.c.path == path)
s = s.values(active=False)
r = self.conn.execute(s)
r.close()
self.conn.execute(s).close()
def public_unset_bulk(self, paths):
if not paths:
return
s = self.public.update()
s = self.public.delete()
s = s.where(self.public.c.path.in_(paths))
s = s.values(active=False)
r = self.conn.execute(s)
r.close()
self.conn.execute(s).close()
def public_get(self, path):
s = select([self.public.c.public_id])
s = select([