Commit 3bd50d9c authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

Change pithos public URL implementation

Refs: commit: 56f3c759

Do not include encoded serial in the public URL.

Change PITHOS_PUBLIC_URL_MIN_LENGTH setting to 
PITHOS_PUBLIC_URL_SECURITY.

Log public set/unset functions.
parent 814b4b6d
......@@ -168,7 +168,7 @@ class ObjectMigration(Migration):
if public:
self.backend.permissions.public_set(
object,
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
#set object's permissions
......
......@@ -57,8 +57,8 @@ PITHOS_USER_LOGIN_URL \https://<astakos.host>/login/
PITHOS_USE_QUOTAHOLDER True Enable quotaholder
PITHOS_QUOTAHOLDER_URL '' Quotaholder URL
PITHOS_QUOTAHOLDER_TOKEN '' Quotaholder token
PITHOS_PUBLIC_URL_MIN_LENGTH 8 Public URL minimun length
PITHOS_PUBLIC_URL_ALPHABET '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' Public URL alphabet
PITHOS_PUBLIC_URL_SECURITY 16 How many random bytes to use for constructing the URL of Pithos public files
PITHOS_PUBLIC_URL_ALPHABET '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' The alphabet to use for constructing the URL of Pithos public files
=============================== ================================================================ ============================================================
To update checksums asynchronously, enable the queue, install snf-pithos-tools and use ``pithos-dispatcher``::
......
......@@ -56,5 +56,10 @@
#
#PITHOS_QUOTAHOLDER_POOLSIZE = 200
#
# Set public url length
#PITHOS_PUBLIC_URL_MIN_LENGTH = 8
# How many random bytes to use for constructing the URL of Pithos public files.
# Lower values mean accidental reuse of (discarded) URLs is more probable.
# Note: the active public URLs will always be unique.
#       Only the old and discarded URLs can ever be reused.
# Higher values mean more safety and longer URLs
#
#PITHOS_PUBLIC_URL_SECURITY = 16
......@@ -66,8 +66,9 @@ QUOTAHOLDER_URL = getattr(settings, 'PITHOS_QUOTAHOLDER_URL', '')
QUOTAHOLDER_TOKEN = getattr(settings, 'PITHOS_QUOTAHOLDER_TOKEN', '')
QUOTAHOLDER_POOLSIZE = getattr(settings, 'PITHOS_QUOTAHOLDER_POOLSIZE', 200)
# Set public url length and alphabet
PUBLIC_URL_MIN_LENGTH = getattr(settings, 'PITHOS_PUBLIC_URL_MIN_LENGTH', 8)
# Set how many random bytes to use for constructing the URL of Pithos public files
PUBLIC_URL_SECURITY = getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16)
# Set the alphabet to use for constructing the URL of Pithos public files
PUBLIC_URL_ALPHABET = getattr(
settings,
'PITHOS_PUBLIC_URL_ALPHABET',
......
......@@ -196,7 +196,7 @@ class SwissArmy():
fullpath = '/'.join([dest_account, dest_container, dest_name])
self.backend.permissions.public_set(
fullpath,
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
......
......@@ -77,7 +77,7 @@ class TestPublic(unittest.TestCase):
account, account, container, object
)
self.assertTrue(public != None)
self.assertTrue(len(public) >= settings.PUBLIC_URL_MIN_LENGTH)
self.assertTrue(len(public) >= settings.PUBLIC_URL_SECURITY)
self.assertTrue(set(public) <= set(settings.PUBLIC_URL_ALPHABET))
self.assertEqual(
self.backend.get_public('$$account$$', public),
......@@ -103,7 +103,7 @@ class TestPublic(unittest.TestCase):
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
self.assert_public_object('account', 'container', 'object')
......@@ -120,14 +120,14 @@ class TestPublic(unittest.TestCase):
)
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public2 = self.assert_public_object('account', 'container', 'object')
......@@ -146,7 +146,7 @@ class TestPublic(unittest.TestCase):
)
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
......@@ -156,7 +156,7 @@ class TestPublic(unittest.TestCase):
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public3 = self.assert_public_object('account', 'container', 'object')
......@@ -225,7 +225,7 @@ class TestPublic(unittest.TestCase):
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
self.assert_public_object('account', 'container', 'object')
......@@ -253,7 +253,7 @@ class TestPublic(unittest.TestCase):
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_min_length,
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
......
......@@ -68,7 +68,7 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
COOKIE_NAME, USER_CATALOG_URL,
RADOS_STORAGE, RADOS_POOL_BLOCKS,
RADOS_POOL_MAPS, TRANSLATE_UUIDS,
PUBLIC_URL_MIN_LENGTH,
PUBLIC_URL_SECURITY,
PUBLIC_URL_ALPHABET)
from pithos.backends import connect_backend
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
......@@ -985,7 +985,7 @@ _pithos_backend_pool = PithosBackendPool(
quotaholder_client_poolsize=QUOTAHOLDER_POOLSIZE,
free_versioning=BACKEND_FREE_VERSIONING,
block_params=BLOCK_PARAMS,
public_url_min_length=PUBLIC_URL_MIN_LENGTH,
public_url_security=PUBLIC_URL_SECURITY,
public_url_alphabet=PUBLIC_URL_ALPHABET)
def get_backend():
......
......@@ -37,7 +37,11 @@ from sqlalchemy.sql import and_, select
from sqlalchemy.schema import Index
from sqlalchemy.exc import NoSuchTableError
from pithos.backends.random_word import get_word
from pithos.backends.random_word import get_random_word
import logging
logger = logging.getLogger(__name__)
def create_tables(engine):
metadata = MetaData()
......@@ -68,15 +72,15 @@ class Public(DBWorker):
tables = create_tables(self.engine)
map(lambda t: self.__setattr__(t.name, t), tables)
def get_unique_url(self, serial, public_url_min_length, public_url_alphabet):
l = public_url_min_length
def get_unique_url(self, public_security, public_url_alphabet):
l = public_security
while 1:
candidate = get_word(serial, length=l, alphabet=public_url_alphabet)
candidate = get_random_word(length=l, alphabet=public_url_alphabet)
if self.public_path(candidate) is None:
return candidate
l +=1
def public_set(self, path, public_url_min_length, public_url_alphabet):
def public_set(self, path, public_security, public_url_alphabet):
s = select([self.public.c.public_id])
s = s.where(self.public.c.path == path)
r = self.conn.execute(s)
......@@ -84,23 +88,20 @@ class Public(DBWorker):
r.close()
if not row:
url = self.get_unique_url(
public_security, public_url_alphabet
)
s = self.public.insert()
s = s.values(path=path, active=True)
s = s.values(path=path, active=True, url=url)
r = self.conn.execute(s)
serial = r.inserted_primary_key[0]
r.close()
url = self.get_unique_url(
serial, public_url_min_length, public_url_alphabet
)
s = self.public.update().where(self.public.c.public_id == serial)
s = s.values(url=url)
self.conn.execute(s).close()
logger.info('Public url: %s set for path: %s' % (url, path))
def public_unset(self, path):
s = self.public.delete()
s = s.where(self.public.c.path == path)
self.conn.execute(s).close()
logger.info('Public url unset for path: %s' % (path))
def public_unset_bulk(self, paths):
if not paths:
......
......@@ -33,7 +33,11 @@
from dbworker import DBWorker
from pithos.backends.random_word import get_word
from pithos.backends.random_word import get_random_word
import logging
logger = logging.getLogger(__name__)
class Public(DBWorker):
"""Paths can be marked as public."""
......@@ -52,35 +56,35 @@ class Public(DBWorker):
execute(""" create unique index if not exists idx_public_url
on public(url) """)
def get_unique_url(self, serial, public_url_min_length, public_url_alphabet):
l = public_url_min_length
def get_unique_url(self, public_url_security, public_url_alphabet):
l = public_url_security
while 1:
candidate = get_word(serial, length=l, alphabet=public_url_alphabet)
candidate = get_random_word(length=l, alphabet=public_url_alphabet)
if self.public_path(candidate) is None:
return candidate
l +=1
def public_set(self, path, public_url_min_length, public_url_alphabet):
def public_set(self, path, public_url_security, public_url_alphabet):
q = "select public_id from public where path = ?"
self.execute(q, (path,))
row = self.fetchone()
if not row:
q = "insert into public(path, active) values(?, ?)"
serial = self.execute(q, (path, active)).lastrowid
url = self.get_unique_url(
serial, public_url_min_length, public_url_alphabet
public_url_security, public_url_alphabet
)
q = "update public set url=url where public_id = ?"
self.execute(q, (serial,))
q = "insert into public(path, active, url) values(?, 1, ?)"
self.execute(q, (path, url))
logger.info('Public url: %s set for path: %s' % (url, path))
def public_unset(self, path):
q = "delete from public where path = ?"
self.execute(q, (path,))
logger.info('Public url unset for path: %s' % (path))
def public_unset_bulk(self, paths):
placeholders = ','.join('?' for path in paths)
q = "delete from public where path in (%s)"
q = "delete from public where path in (%s)" % placeholders
self.execute(q, paths)
def public_get(self, path):
......
......@@ -84,10 +84,10 @@ DEFAULT_BLOCK_UMASK = 0o022
DEFAULT_BLOCK_PARAMS = { 'mappool': None, 'blockpool': None }
#DEFAULT_QUEUE_HOSTS = '[amqp://guest:guest@localhost:5672]'
#DEFAULT_QUEUE_EXCHANGE = 'pithos'
DEFAULT_ALPHABET = ('0123456789'
'abcdefghijklmnopqrstuvwxyz'
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
DEFAULT_MIN_LENGTH = 8
DEFAULT_PUBLIC_URL_ALPHABET = ('0123456789'
'abcdefghijklmnopqrstuvwxyz'
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
DEFAULT_PUBLIC_URL_SECURITY = 8
QUEUE_MESSAGE_KEY_PREFIX = 'pithos.%s'
QUEUE_CLIENT_ID = 'pithos'
......@@ -153,7 +153,7 @@ class ModularBackend(BaseBackend):
quotaholder_url=None, quotaholder_token=None,
quotaholder_client_poolsize=None,
free_versioning=True, block_params=None,
public_url_min_length=None,
public_url_security=None,
public_url_alphabet=None):
db_module = db_module or DEFAULT_DB_MODULE
db_connection = db_connection or DEFAULT_DB_CONNECTION
......@@ -167,8 +167,8 @@ class ModularBackend(BaseBackend):
#queue_hosts = queue_hosts or DEFAULT_QUEUE_HOSTS
#queue_exchange = queue_exchange or DEFAULT_QUEUE_EXCHANGE
self.public_url_min_length = public_url_min_length or DEFAULT_MIN_LENGTH
self.public_url_alphabet = public_url_alphabet or DEFAULT_ALPHABET
self.public_url_security = public_url_security or DEFAULT_PUBLIC_URL_SECURITY
self.public_url_alphabet = public_url_alphabet or DEFAULT_PUBLIC_URL_ALPHABET
self.hash_algorithm = 'sha256'
self.block_size = 4 * 1024 * 1024 # 4MB
......@@ -838,7 +838,7 @@ class ModularBackend(BaseBackend):
self.permissions.public_unset(path)
else:
self.permissions.public_set(
path, self.public_url_min_length, self.public_url_alphabet
path, self.public_url_security, self.public_url_alphabet
)
@backend_method
......
......@@ -31,33 +31,27 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
DEFAULT_ALPHABET = ('0123456789'
'abcdefghijklmnopqrstuvwxyz'
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
DEFAULT_MIN_LENGTH = 8
import random
from random import randrange
from math import log
getrandbits = random.SystemRandom().getrandbits
def get_random_word(length=DEFAULT_MIN_LENGTH, alphabet=DEFAULT_ALPHABET):
alphabet_length = len(alphabet)
word = ''.join(alphabet[randrange(alphabet_length)]
for _ in xrange(length))
return word
DEFAULT_ALPHABET = ("0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz")
def encode_serial(serial, alphabet=DEFAULT_ALPHABET):
i = 0
def get_random_word(length, alphabet=DEFAULT_ALPHABET):
remainder = getrandbits(length * 8)
return encode_word(remainder, alphabet=alphabet)
def encode_word(number, alphabet=DEFAULT_ALPHABET):
base = len(alphabet)
quotient = int(serial)
digits = []
append = digits.append
while quotient > 0:
quotient = number
while True:
quotient, remainder = divmod(quotient, base)
append(alphabet[remainder])
word = ''.join(reversed(digits))
return word
if quotient <= 0:
break
def get_word(serial, length=DEFAULT_MIN_LENGTH, alphabet=DEFAULT_ALPHABET):
word = encode_serial(serial, alphabet)
word += get_random_word(length, alphabet)
return word
return ''.join(digits)
......@@ -49,7 +49,7 @@ class PithosBackendPool(ObjectPool):
quotaholder_url=None, quotaholder_token=None,
quotaholder_client_poolsize=None,
block_params=None,
public_url_min_length=None,
public_url_security=None,
public_url_alphabet=None):
super(PithosBackendPool, self).__init__(size=size)
self.db_module = db_module
......@@ -66,7 +66,7 @@ class PithosBackendPool(ObjectPool):
self.quotaholder_token = quotaholder_token
self.quotaholder_client_poolsize = quotaholder_client_poolsize
self.free_versioning = free_versioning
self.public_url_min_length=public_url_min_length
self.public_url_security=public_url_security
self.public_url_alphabet=public_url_alphabet
def _pool_create(self):
......@@ -85,7 +85,7 @@ class PithosBackendPool(ObjectPool):
quotaholder_token=self.quotaholder_token,
quotaholder_client_poolsize=self.quotaholder_client_poolsize,
free_versioning=self.free_versioning,
public_url_min_length=self.public_url_min_length,
public_url_security=self.public_url_security,
public_url_alphabet=self.public_url_alphabet)
backend._real_close = backend.close
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment