Commit 1639759e authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

pithos: pep8 fixes

parent f49fa98a
......@@ -691,9 +691,9 @@ def object_list(request, v_account, v_container):
try:
objects = request.backend.list_object_meta(
request.user_uniq, v_account,
v_container, prefix, delimiter, marker,
limit, virtual, 'pithos', keys, shared, until, None, public_granted)
request.user_uniq, v_account, v_container, prefix, delimiter,
marker, limit, virtual, 'pithos', keys, shared, until, None,
public_granted)
object_permissions = {}
object_public = {}
if until is None:
......@@ -920,9 +920,9 @@ def _object_read(request, v_account, v_container, v_object):
try:
for x in objects:
s, h = \
request.backend.get_object_hashmap(request.user_uniq,
v_account, src_container,
x[0], x[1])
request.backend.get_object_hashmap(
request.user_uniq, v_account, src_container, x[0],
x[1])
sizes.append(s)
hashmaps.append(h)
except NotAllowedError:
......@@ -1009,8 +1009,8 @@ def object_write(request, v_account, v_container, v_object):
else:
if TRANSLATE_UUIDS:
try:
src_account = retrieve_uuid(getattr(request, 'token', None),
src_account)
src_account = retrieve_uuid(
getattr(request, 'token', None), src_account)
except ItemNotExists:
faults.ItemNotFound('Invalid source account')
......@@ -1139,7 +1139,8 @@ def object_write(request, v_account, v_container, v_object):
return response
@api_method('POST', user_required=True, logger=logger, lock_container_path=True)
@api_method('POST', user_required=True, logger=logger,
lock_container_path=True)
def object_write_form(request, v_account, v_container, v_object):
# Normal Response Codes: 201
# Error Response Codes: internalServerError (500),
......@@ -1301,10 +1302,9 @@ def object_update(request, v_account, v_container, v_object):
# Do permissions first, as it may fail easier.
if permissions is not None:
try:
request.backend.update_object_permissions(request.user_uniq,
v_account,
v_container, v_object,
permissions)
request.backend.update_object_permissions(
request.user_uniq, v_account, v_container, v_object,
permissions)
except NotAllowedError:
raise faults.Forbidden('Not allowed')
except ItemNotExists:
......@@ -1345,8 +1345,8 @@ def object_update(request, v_account, v_container, v_object):
try:
size, hashmap = \
request.backend.get_object_hashmap(request.user_uniq,
v_account, v_container, v_object)
request.backend.get_object_hashmap(
request.user_uniq, v_account, v_container, v_object)
except NotAllowedError:
raise faults.Forbidden('Not allowed')
except ItemNotExists:
......@@ -1417,8 +1417,8 @@ def object_update(request, v_account, v_container, v_object):
hashmap[bi] = src_hashmap[sbi]
else:
data = request.backend.get_block(src_hashmap[sbi])
hashmap[bi] = request.backend.update_block(hashmap[bi],
data[:bl], 0)
hashmap[bi] = request.backend.update_block(
hashmap[bi], data[:bl], 0)
else:
hashmap.append(src_hashmap[sbi])
offset += bl
......
......@@ -344,8 +344,8 @@ class ManageAccountsTests(unittest.TestCase):
'application/octet-stream', data, meta, permissions))
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account1', container, object,
'application/octet-stream', data, meta, permissions)
'account1', container, object, 'application/octet-stream',
data, meta, permissions)
self.utils.merge_account('Account1', 'account1', only_stats=False,
dry=False, silent=True)
......
......@@ -27,8 +27,8 @@ UI_PREFIX = get_path(pithos_services, 'pithos_ui.prefix')
astakos_services = deepcopy(vanilla_astakos_services)
fill_endpoints(astakos_services, ASTAKOS_BASE_URL)
CUSTOMIZE_ASTAKOS_SERVICES = \
getattr(settings, 'PITHOS_CUSTOMIZE_ASTAKOS_SERVICES', ())
CUSTOMIZE_ASTAKOS_SERVICES = getattr(settings,
'PITHOS_CUSTOMIZE_ASTAKOS_SERVICES', ())
for path, value in CUSTOMIZE_ASTAKOS_SERVICES:
set_path(astakos_services, path, value, createpath=True)
......@@ -42,7 +42,8 @@ BASE_ASTAKOS_PROXY_PATH = join_urls(BASE_PATH, BASE_ASTAKOS_PROXY_PATH)
BASE_ASTAKOS_PROXY_PATH = BASE_ASTAKOS_PROXY_PATH.strip('/')
ASTAKOSCLIENT_POOLSIZE = getattr(settings, 'PITHOS_ASTAKOSCLIENT_POOLSIZE', 200)
ASTAKOSCLIENT_POOLSIZE = getattr(settings, 'PITHOS_ASTAKOSCLIENT_POOLSIZE',
200)
COOKIE_NAME = getattr(settings, 'PITHOS_ASTAKOS_COOKIE_NAME', '_pithos2_a')
......@@ -60,10 +61,14 @@ BACKEND_BLOCK_PATH = getattr(
BACKEND_BLOCK_UMASK = getattr(settings, 'PITHOS_BACKEND_BLOCK_UMASK', 0o022)
# Queue for billing.
BACKEND_QUEUE_MODULE = getattr(settings, 'PITHOS_BACKEND_QUEUE_MODULE',
None) # Example: 'pithos.backends.lib.rabbitmq'
BACKEND_QUEUE_HOSTS = getattr(settings, 'PITHOS_BACKEND_QUEUE_HOSTS', None) # Example: "['amqp://guest:guest@localhost:5672']"
BACKEND_QUEUE_EXCHANGE = getattr(settings, 'PITHOS_BACKEND_QUEUE_EXCHANGE', 'pithos')
BACKEND_QUEUE_MODULE = getattr(settings, 'PITHOS_BACKEND_QUEUE_MODULE', None)
# Example: 'pithos.backends.lib.rabbitmq'
BACKEND_QUEUE_HOSTS = getattr(settings, 'PITHOS_BACKEND_QUEUE_HOSTS', None)
# Example: "['amqp://guest:guest@localhost:5672']"
BACKEND_QUEUE_EXCHANGE = getattr(settings, 'PITHOS_BACKEND_QUEUE_EXCHANGE',
'pithos')
# Default setting for new accounts.
BACKEND_ACCOUNT_QUOTA = getattr(
......@@ -71,7 +76,8 @@ BACKEND_ACCOUNT_QUOTA = getattr(
BACKEND_CONTAINER_QUOTA = getattr(
settings, 'PITHOS_BACKEND_CONTAINER_QUOTA', 0)
BACKEND_VERSIONING = getattr(settings, 'PITHOS_BACKEND_VERSIONING', 'auto')
BACKEND_FREE_VERSIONING = getattr(settings, 'PITHOS_BACKEND_FREE_VERSIONING', True)
BACKEND_FREE_VERSIONING = getattr(settings, 'PITHOS_BACKEND_FREE_VERSIONING',
True)
# Enable backend pooling
BACKEND_POOL_ENABLED = getattr(settings, 'PITHOS_BACKEND_POOL_ENABLED', True)
......@@ -86,7 +92,7 @@ UPDATE_MD5 = getattr(settings, 'PITHOS_UPDATE_MD5', False)
SERVICE_TOKEN = getattr(settings, 'PITHOS_SERVICE_TOKEN', '')
RADOS_STORAGE = getattr(settings, 'PITHOS_RADOS_STORAGE', False)
RADOS_POOL_BLOCKS= getattr(settings, 'PITHOS_RADOS_POOL_BLOCKS', 'blocks')
RADOS_POOL_BLOCKS = getattr(settings, 'PITHOS_RADOS_POOL_BLOCKS', 'blocks')
RADOS_POOL_MAPS = getattr(settings, 'PITHOS_RADOS_POOL_MAPS', 'maps')
# This enables a ui compatibility layer for the introduction of UUIDs in
......@@ -100,14 +106,14 @@ TRANSLATE_UUIDS = getattr(settings, 'PITHOS_TRANSLATE_UUIDS', False)
# the requests on its own.
PROXY_USER_SERVICES = getattr(settings, 'PITHOS_PROXY_USER_SERVICES', True)
# Set how many random bytes to use for constructing the URL of Pithos public files
PUBLIC_URL_SECURITY = getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16)
# Set how many random bytes to use for constructing the URL
# of Pithos public files
PUBLIC_URL_SECURITY = getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16)
# Set the alphabet to use for constructing the URL of Pithos public files
PUBLIC_URL_ALPHABET = getattr(
PUBLIC_URL_ALPHABET = getattr(
settings,
'PITHOS_PUBLIC_URL_ALPHABET',
'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
)
'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')
# The maximum number or items returned by the listing api methods
API_LIST_LIMIT = getattr(settings, 'PITHOS_API_LIST_LIMIT', 10000)
......
......@@ -73,7 +73,8 @@ MIN_LENGTH = 5
class UrlEncoder(object):
def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE):
def __init__(self, alphabet=DEFAULT_ALPHABET,
block_size=DEFAULT_BLOCK_SIZE):
self.alphabet = alphabet
self.block_size = block_size
self.mask = (1 << block_size) - 1
......
......@@ -56,9 +56,9 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
ASTAKOSCLIENT_POOLSIZE,
SERVICE_TOKEN,
ASTAKOS_BASE_URL,
BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA,
BACKEND_VERSIONING,
BACKEND_FREE_VERSIONING,
BACKEND_ACCOUNT_QUOTA,
BACKEND_CONTAINER_QUOTA,
BACKEND_VERSIONING, BACKEND_FREE_VERSIONING,
BACKEND_POOL_ENABLED, BACKEND_POOL_SIZE,
BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,
RADOS_STORAGE, RADOS_POOL_BLOCKS,
......@@ -200,7 +200,8 @@ def put_container_headers(request, response, meta, policy):
int(meta['until_timestamp']))
for k, v in policy.iteritems():
response[smart_str(format_header_key('X-Container-Policy-' + k),
strings_only=True)] = smart_str(v, strings_only=True)
strings_only=True)] = smart_str(v,
strings_only=True)
def get_object_headers(request):
......@@ -402,7 +403,7 @@ def update_public_meta(public, meta):
def validate_modification_preconditions(request, meta):
"""Check that the modified timestamp conforms with the preconditions set."""
"""Check the modified timestamp conforms with the preconditions set."""
if 'modified' not in meta:
return # TODO: Always return?
......@@ -441,8 +442,8 @@ def validate_matching_preconditions(request, meta):
if if_none_match is not None:
# TODO: If this passes, must ignore If-Modified-Since header.
if etag is not None:
if (if_none_match == '*'
or etag in [x.lower() for x in parse_etags(if_none_match)]):
if (if_none_match == '*' or etag in [x.lower() for x in
parse_etags(if_none_match)]):
# TODO: Continue if an If-Modified-Since header is present.
if request.method in ('HEAD', 'GET'):
raise faults.NotModified('Resource ETag matches')
......@@ -697,7 +698,7 @@ MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024) # 5GB
def socket_read_iterator(request, length=0, blocksize=4096):
"""Return a maximum of blocksize data read from the socket in each iteration
"""Return maximum of blocksize data read from the socket in each iteration
Read up to 'length'. If 'length' is negative, will attempt a chunked read.
The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
......
......@@ -37,7 +37,6 @@ DEFAULT_CONTAINER_QUOTA = 0 # No quota.
DEFAULT_CONTAINER_VERSIONING = 'auto'
class NotAllowedError(Exception):
pass
......@@ -71,22 +70,30 @@ class VersionNotExists(IndexError):
class BaseBackend(object):
"""Abstract backend class that serves as a reference for actual implementations.
"""Abstract backend class.
This class serves as a reference for actual implementations.
The purpose of the backend is to provide the necessary functions for handling data
and metadata. It is responsible for the actual storage and retrieval of information.
The purpose of the backend is to provide the necessary functions
for handling data and metadata.
Note that the account level is always valid as it is checked from another subsystem.
It is responsible for the actual storage and retrieval of information.
When not replacing metadata/groups/policy, keys with empty values should be deleted.
Note that the account level is always valid as it is checked
from another subsystem.
When not replacing metadata/groups/policy, keys with empty values
should be deleted.
The following variables should be available:
'hash_algorithm': Suggested is 'sha256'
'block_size': Suggested is 4MB
'default_account_policy': A dictionary with default account policy settings
'default_container_policy': A dictionary with default container policy settings
'default_account_policy': A dictionary with default account policy
settings
'default_container_policy': A dictionary with default container policy
settings
"""
def close(self):
......@@ -96,14 +103,15 @@ class BaseBackend(object):
def list_accounts(self, user, marker=None, limit=10000):
"""Return a list of accounts the user can access.
Parameters:
Keyword arguments:
'marker': Start list from the next item after 'marker'
'limit': Number of containers to return
"""
return []
def get_account_meta(self, user, account, domain, until=None, include_user_defined=True):
def get_account_meta(self, user, account, domain, until=None,
include_user_defined=True, external_quota=None):
"""Return a dictionary with the account metadata for the domain.
The keys returned are all user-defined, except:
......@@ -117,6 +125,9 @@ class BaseBackend(object):
'until_timestamp': Last modification until the timestamp provided
'external_quota': The quota computed from external quota holder
mechanism
Raises:
NotAllowedError: Operation not permitted
"""
......@@ -128,6 +139,7 @@ class BaseBackend(object):
Parameters:
'domain': Metadata domain
Keyword arguments:
'meta': Dictionary with metadata to update
'replace': Replace instead of update
......@@ -138,7 +150,7 @@ class BaseBackend(object):
return
def get_account_groups(self, user, account):
"""Return a dictionary with the user groups defined for this account.
"""Return a dictionary with the user groups defined for the account.
Raises:
NotAllowedError: Operation not permitted
......@@ -161,7 +173,7 @@ class BaseBackend(object):
The keys returned are:
'quota': The maximum bytes allowed (default is 0 - unlimited)
'versioning': Can be 'auto', 'manual' or 'none' (default is 'manual')
'versioning': Can be 'auto', 'manual' or 'none' (default: 'manual')
Raises:
NotAllowedError: Operation not permitted
......@@ -198,10 +210,11 @@ class BaseBackend(object):
"""
return
def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None, public=False):
def list_containers(self, user, account, marker=None, limit=10000,
shared=False, until=None, public=False):
"""Return a list of container names existing under an account.
Parameters:
Keyword arguments:
'marker': Start list from the next item after 'marker'
'limit': Number of containers to return
......@@ -216,8 +229,9 @@ class BaseBackend(object):
"""
return []
def list_container_meta(self, user, account, container, domain, until=None):
"""Return a list with all the container's object meta keys for the domain.
def list_container_meta(self, user, account, container, domain,
until=None):
"""Return a list of the container's object meta keys for a domain.
Raises:
NotAllowedError: Operation not permitted
......@@ -226,7 +240,8 @@ class BaseBackend(object):
"""
return []
def get_container_meta(self, user, account, container, domain, until=None, include_user_defined=True):
def get_container_meta(self, user, account, container, domain, until=None,
include_user_defined=True):
"""Return a dictionary with the container metadata for the domain.
The keys returned are all user-defined, except:
......@@ -247,12 +262,14 @@ class BaseBackend(object):
"""
return {}
def update_container_meta(self, user, account, container, domain, meta, replace=False):
def update_container_meta(self, user, account, container, domain, meta,
replace=False):
"""Update the metadata associated with the container for the domain.
Parameters:
'domain': Metadata domain
Keyword arguments:
'meta': Dictionary with metadata to update
'replace': Replace instead of update
......@@ -270,7 +287,7 @@ class BaseBackend(object):
The keys returned are:
'quota': The maximum bytes allowed (default is 0 - unlimited)
'versioning': Can be 'auto', 'manual' or 'none' (default is 'manual')
'versioning': Can be 'auto', 'manual' or 'none' (default: 'manual')
Raises:
NotAllowedError: Operation not permitted
......@@ -279,7 +296,8 @@ class BaseBackend(object):
"""
return {}
def update_container_policy(self, user, account, container, policy, replace=False):
def update_container_policy(self, user, account, container, policy,
replace=False):
"""Update the policy associated with the container.
Raises:
......@@ -291,12 +309,9 @@ class BaseBackend(object):
"""
return
def put_container(self, user, account, container, policy=None, delimiter=None):
def put_container(self, user, account, container, policy=None):
"""Create a new container with the given name.
Parameters:
'delimiter': If present deletes container contents instead of the container
Raises:
NotAllowedError: Operation not permitted
......@@ -306,9 +321,14 @@ class BaseBackend(object):
"""
return
def delete_container(self, user, account, container, until=None):
def delete_container(self, user, account, container, until=None,
delimiter=None):
"""Delete/purge the container with the given name.
Keyword arguments:
'delimiter': If not None, deletes the container contents starting
with the delimiter
Raises:
NotAllowedError: Operation not permitted
......@@ -318,13 +338,17 @@ class BaseBackend(object):
"""
return
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None, public=False):
"""Return a list of object (name, version_id) tuples existing under a container.
def list_objects(self, user, account, container, prefix='', delimiter=None,
marker=None, limit=10000, virtual=True, domain=None,
keys=None, shared=False, until=None, size_range=None,
public=False):
"""List (object name, object version_id) under a container.
Parameters:
Keyword arguments:
'prefix': List objects starting with 'prefix'
'delimiter': Return unique names before 'delimiter' and after 'prefix'
'delimiter': Return unique names before 'delimiter' and
after 'prefix'
'marker': Start list from the next item after 'marker'
......@@ -333,7 +357,7 @@ class BaseBackend(object):
'virtual': If not set, the result will only include names starting
with 'prefix' and ending without a 'delimiter' or with
the first occurance of the 'delimiter' after 'prefix'.
If set, the result will include all names after 'prefix',
If set, the result will include all names after 'prefix'
up to and including the 'delimiter' if it is found
'domain': Metadata domain for keys
......@@ -357,8 +381,11 @@ class BaseBackend(object):
"""
return []
def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None):
"""Return a list of object metadata dicts existing under a container.
def list_object_meta(self, user, account, container, prefix='',
delimiter=None, marker=None, limit=10000,
virtual=True, domain=None, keys=None, shared=False,
until=None, size_range=None, public=False):
"""Return a list of metadata dicts of objects under a container.
Same parameters with list_objects. Returned dicts have no user-defined
metadata and, if until is not None, a None 'modified' timestamp.
......@@ -371,7 +398,7 @@ class BaseBackend(object):
return []
def list_object_permissions(self, user, account, container, prefix=''):
"""Return a list of paths that enforce permissions under a container.
"""Return a list of paths enforce permissions under a container.
Raises:
NotAllowedError: Operation not permitted
......@@ -379,10 +406,11 @@ class BaseBackend(object):
return []
def list_object_public(self, user, account, container, prefix=''):
"""Return a dict mapping paths to public ids for objects that are public under a container."""
"""Return a mapping of object paths to public ids under a container."""
return {}
def get_object_meta(self, user, account, container, name, domain, version=None, include_user_defined=True):
def get_object_meta(self, user, account, container, name, domain,
version=None, include_user_defined=True):
"""Return a dictionary with the object metadata for the domain.
The keys returned are all user-defined, except:
......@@ -396,13 +424,15 @@ class BaseBackend(object):
'modified': Last modification timestamp (overall)
'modified_by': The user that committed the object (version requested)
'modified_by': The user that committed the object
(version requested)
'version': The version identifier
'version_timestamp': The version's modification timestamp
'uuid': A unique identifier that persists data or metadata updates and renames
'uuid': A unique identifier that persists data or metadata updates
and renames
'checksum': The MD5 sum of the object (may be empty)
......@@ -415,8 +445,9 @@ class BaseBackend(object):
"""
return {}
def update_object_meta(self, user, account, container, name, domain, meta, replace=False):
"""Update the metadata associated with the object for the domain and return the new version.
def update_object_meta(self, user, account, container, name, domain, meta,
replace=False):
"""Update object metadata for a domain and return the new version.
Parameters:
'domain': Metadata domain
......@@ -449,7 +480,8 @@ class BaseBackend(object):
"""
return {}
def update_object_permissions(self, user, account, container, name, permissions):
def update_object_permissions(self, user, account, container, name,
permissions):
"""Update (set) the permissions associated with the object.
Parameters:
......@@ -499,8 +531,10 @@ class BaseBackend(object):
"""
return 0, []
def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta=None, replace_meta=False, permissions=None):
"""Create/update an object with the specified size and partial hashes and return the new version.
def update_object_hashmap(self, user, account, container, name, size, type,
hashmap, checksum, domain, meta=None,
replace_meta=False, permissions=None):
"""Create/update an object's hashmap and return the new version.
Parameters:
'domain': Metadata domain
......@@ -522,17 +556,22 @@ class BaseBackend(object):
"""
return ''
def update_object_checksum(self, user, account, container, name, version, checksum):
def update_object_checksum(self, user, account, container, name, version,
checksum):
"""Update an object's checksum."""
return
def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, src_version=None, delimiter=None):
def copy_object(self, user, src_account, src_container, src_name,
dest_account, dest_container, dest_name, type, domain,
meta=None, replace_meta=False, permissions=None,
src_version=None, delimiter=None):
"""Copy an object's data and metadata and return the new version.
Parameters:
'domain': Metadata domain
'meta': Dictionary with metadata to change from source to destination
'meta': Dictionary with metadata to change from source
to destination
'replace_meta': Replace metadata instead of update
......@@ -540,7 +579,8 @@ class BaseBackend(object):
'src_version': Copy from the version provided
'delimiter': Copy objects whose path starts with src_name + delimiter
'delimiter': Copy objects whose path starts with
src_name + delimiter
Raises:
NotAllowedError: Operation not permitted
......@@ -555,19 +595,24 @@ class BaseBackend(object):
"""
return ''
def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, delimiter=None):
def move_object(self, user, src_account, src_container, src_name,
dest_account, dest_container, dest_name, type, domain,
meta=None, replace_meta=False, permissions=None,
delimiter=None):
"""Move an object's data and metadata and return the new version.
Parameters:
'domain': Metadata domain
'meta': Dictionary with metadata to change from source to destination
'meta': Dictionary with metadata to change from source
to destination
'replace_meta': Replace metadata instead of update
'permissions': New object permissions
'delimiter': Move objects whose path starts with src_name + delimiter
'delimiter': Move objects whose path starts with
src_name + delimiter
Raises:
NotAllowedError: Operation not permitted
......@@ -580,11 +625,13 @@ class BaseBackend(object):
"""
return ''
def delete_object(self, user, account, container, name, until=None, delimiter=None):
def delete_object(self, user, account, container, name, until=None,
delimiter=None):