Skip to content

GitLab

S
synnefo
Commit 7ac5bef1 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files
Options

Fix code formatting to conform to the PEP 8 style guide

parent 65f9b8fd
Show whitespace changes
Inline Side-by-side
Showing with 2987 additions and 2519 deletions
other/migrate.py
View file @ 7ac5bef1
......@@ -41,6 +41,7 @@ from django.conf import settings
from pithos.backends.modular import ModularBackend
class Migration(object):
def __init__(self, db):
self.engine = create_engine(db)
......@@ -54,12 +55,13 @@ class Migration(object):
def execute(self):
pass
class Cache():
def __init__(self, db):
self.engine = create_engine(db)
metadata = MetaData(self.engine)
columns=[]
columns = []
columns.append(Column('path', String(2048), primary_key=True))
columns.append(Column('hash', String(255)))
self.files = Table('files', metadata, *columns)
......@@ -69,7 +71,7 @@ class Cache():
def put(self, path, hash):
# Insert or replace.
s = self.files.delete().where(self.files.c.path==path)
s = self.files.delete().where(self.files.c.path == path)
r = self.conn.execute(s)
r.close()
s = self.files.insert()
......
snf-pithos-app/distribute_setup.py
View file @ 7ac5bef1
......@@ -144,7 +144,7 @@ def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
except ImportError:
return _do_download(version, download_base, to_dir, download_delay)
try:
pkg_resources.require("distribute>="+version)
pkg_resources.require("distribute>=" + version)
return
except pkg_resources.VersionConflict:
e = sys.exc_info()[1]
......@@ -167,6 +167,7 @@ def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
if not no_fake:
_create_fake_setuptools_pkg_info(to_dir)
def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
to_dir=os.curdir, delay=15):
"""Download distribute from a specified location and return its filename
......@@ -203,6 +204,7 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
dst.close()
return os.path.realpath(saveto)
def _no_sandbox(function):
def __no_sandbox(*args, **kw):
try:
......@@ -227,6 +229,7 @@ def _no_sandbox(function):
return __no_sandbox
def _patch_file(path, content):
"""Will backup the file then patch it"""
existing_content = open(path).read()
......@@ -245,15 +248,18 @@ def _patch_file(path, content):
_patch_file = _no_sandbox(_patch_file)
def _same_content(path, content):
return open(path).read() == content
def _rename_path(path):
new_name = path + '.OLD.%s' % time.time()
log.warn('Renaming %s into %s', path, new_name)
os.rename(path, new_name)
return new_name
def _remove_flat_installation(placeholder):
if not os.path.isdir(placeholder):
log.warn('Unkown installation at %s', placeholder)
......@@ -289,11 +295,13 @@ def _remove_flat_installation(placeholder):
_remove_flat_installation = _no_sandbox(_remove_flat_installation)
def _after_install(dist):
log.warn('After install bootstrap.')
placeholder = dist.get_command_obj('install').install_purelib
_create_fake_setuptools_pkg_info(placeholder)
def _create_fake_setuptools_pkg_info(placeholder):
if not placeholder or not os.path.exists(placeholder):
log.warn('Could not find the install location')
......@@ -321,7 +329,9 @@ def _create_fake_setuptools_pkg_info(placeholder):
finally:
f.close()
_create_fake_setuptools_pkg_info = _no_sandbox(_create_fake_setuptools_pkg_info)
_create_fake_setuptools_pkg_info = _no_sandbox(
_create_fake_setuptools_pkg_info)
def _patch_egg_dir(path):
# let's check if it's already patched
......@@ -343,6 +353,7 @@ def _patch_egg_dir(path):
_patch_egg_dir = _no_sandbox(_patch_egg_dir)
def _before_install():
log.warn('Before install bootstrap.')
_fake_setuptools()
......@@ -351,7 +362,7 @@ def _before_install():
def _under_prefix(location):
if 'install' not in sys.argv:
return True
args = sys.argv[sys.argv.index('install')+1:]
args = sys.argv[sys.argv.index('install') + 1:]
for index, arg in enumerate(args):
for option in ('--root', '--prefix'):
if arg.startswith('%s=' % option):
......@@ -359,7 +370,7 @@ def _under_prefix(location):
return location.startswith(top_dir)
elif arg == option:
if len(args) > index:
top_dir = args[index+1]
top_dir = args[index + 1]
return location.startswith(top_dir)
if arg == '--user' and USER_SITE is not None:
return location.startswith(USER_SITE)
......@@ -380,7 +391,8 @@ def _fake_setuptools():
replacement=False))
except TypeError:
# old distribute API
setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools'))
setuptools_dist = ws.find(
pkg_resources.Requirement.parse('setuptools'))
if setuptools_dist is None:
log.warn('No setuptools distribution found')
......
snf-pithos-app/pithos/api/delegate.py
View file @ 7ac5bef1
......@@ -46,6 +46,7 @@ from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS, SERVIC
logger = logging.getLogger(__name__)
def delegate_to_login_service(request):
url = AUTHENTICATION_URL
users = AUTHENTICATION_USERS
......@@ -61,6 +62,7 @@ def delegate_to_login_service(request):
uri = proto + p.netloc + '/login?' + urlencode(params)
return HttpResponseRedirect(uri)
@csrf_exempt
def delegate_to_feedback_service(request):
url = AUTHENTICATION_URL
......@@ -75,7 +77,7 @@ def delegate_to_feedback_service(request):
proto = 'http://'
uri = proto + p.netloc + '/im/service/api/v2.0/feedback'
headers = { 'X-Auth-Token' : SERVICE_TOKEN }
headers = {'X-Auth-Token': SERVICE_TOKEN}
values = dict([(k, v) for k, v in request.POST.items()])
data = urllib.urlencode(values)
req = urllib2.Request(uri, data, headers)
......
snf-pithos-app/pithos/api/dispatch.py
View file @ 7ac5bef1
......@@ -14,6 +14,7 @@ from astakos.im.settings import DEFAULT_FROM_EMAIL
import socket
from smtplib import SMTPException
def update_md5(m):
if m['resource'] != 'object' or m['details']['action'] != 'object update':
return
......@@ -33,17 +34,21 @@ def update_md5(m):
version = m['details']['version']
meta = None
try:
meta = backend.get_object_meta(account, account, container, name, 'pithos', version)
meta = backend.get_object_meta(
account, account, container, name, 'pithos', version)
if meta['checksum'] == '':
size, hashmap = backend.get_object_hashmap(account, account, container, name, version)
size, hashmap = backend.get_object_hashmap(
account, account, container, name, version)
checksum = hashmap_md5(backend, hashmap, size)
backend.update_object_checksum(account, account, container, name, version, checksum)
backend.update_object_checksum(
account, account, container, name, version, checksum)
print 'INFO: Updated checksum for path "%s"' % (path,)
except Exception, e:
print 'WARNING: Can not update checksum for path "%s" (%s)' % (path, e)
backend.close()
def send_sharing_notification(m):
if m['resource'] != 'sharing':
return
......@@ -56,9 +61,11 @@ def send_sharing_notification(m):
subject = 'Invitation to a Pithos+ shared object'
from_email = DEFAULT_FROM_EMAIL
recipient_list = members
message = 'User %s has invited you to a Pithos+ shared object. You can view it under "Shared to me" at "%s".' %(user, path)
message = 'User %s has invited you to a Pithos+ shared object. You can view it under "Shared to me" at "%s".' % (user, path)
try:
send_mail(subject, message, from_email, recipient_list)
print 'INFO: Sharing notification sent for path "%s" to %s' % (path, ','.join(recipient_list))
print 'INFO: Sharing notification sent for path "%s" to %s' % (
path, ','.join(recipient_list))
except (SMTPException, socket.error) as e:
print 'WARNING: Can not update send email for sharing "%s" (%s)' % (path, e)
print 'WARNING: Can not update send email for sharing "%s" (%s)' % (
path, e)
snf-pithos-app/pithos/api/faults.py
View file @ 7ac5bef1
......@@ -31,6 +31,7 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
def camelCase(s):
return s[0].lower() + s[1:]
......@@ -42,41 +43,54 @@ class Fault(Exception):
self.details = details
self.name = name or camelCase(self.__class__.__name__)
class NotModified(Fault):
code = 304
class BadRequest(Fault):
code = 400
class Unauthorized(Fault):
code = 401
class Forbidden(Fault):
code = 403
class ItemNotFound(Fault):
code = 404
class Conflict(Fault):
code = 409
class LengthRequired(Fault):
code = 411
class PreconditionFailed(Fault):
code = 412
class RequestEntityTooLarge(Fault):
code = 413
class RangeNotSatisfiable(Fault):
code = 416
class UnprocessableEntity(Fault):
code = 422
class InternalServerError(Fault):
code = 500
class NotImplemented(Fault):
code = 501
snf-pithos-app/pithos/api/functions.py
View file @ 7ac5bef1
......@@ -44,9 +44,11 @@ from django.views.decorators.csrf import csrf_exempt
from synnefo.lib.astakos import get_user
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
from pithos.api.faults import (
Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
from pithos.api.util import (
json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
......@@ -79,6 +81,7 @@ def top_demux(request):
else:
return method_not_allowed(request)
@csrf_exempt
def account_demux(request, v_account):
if request.method == 'HEAD':
......@@ -90,6 +93,7 @@ def account_demux(request, v_account):
else:
return method_not_allowed(request)
@csrf_exempt
def container_demux(request, v_account, v_container):
if request.method == 'HEAD':
......@@ -105,6 +109,7 @@ def container_demux(request, v_account, v_container):
else:
return method_not_allowed(request)
@csrf_exempt
def object_demux(request, v_account, v_container, v_object):
# Helper to avoid placing the token in the URL when loading objects from a browser.
......@@ -127,6 +132,7 @@ def object_demux(request, v_account, v_container, v_object):
else:
return method_not_allowed(request)
@api_method('GET', user_required=False)
def authenticate(request):
# Normal Response Codes: 204
......@@ -145,9 +151,11 @@ def authenticate(request):
uri = uri[:uri.find('?')]
response['X-Auth-Token'] = x_auth_key
response['X-Storage-Url'] = uri + ('' if uri.endswith('/') else '/') + x_auth_user
response['X-Storage-Url'] = uri + ('' if uri.endswith('/')
else '/') + x_auth_user
return response
@api_method('GET', format_allowed=True)
def account_list(request):
# Normal Response Codes: 200, 204
......@@ -176,15 +184,18 @@ def account_list(request):
if x == request.user_uniq:
continue
try:
meta = request.backend.get_account_meta(request.user_uniq, x, 'pithos', include_user_defined=False)
meta = request.backend.get_account_meta(
request.user_uniq, x, 'pithos', include_user_defined=False)
groups = request.backend.get_account_groups(request.user_uniq, x)
except NotAllowedError:
raise Forbidden('Not allowed')
else:
rename_meta_key(meta, 'modified', 'last_modified')
rename_meta_key(meta, 'until_timestamp', 'x_account_until_timestamp')
rename_meta_key(
meta, 'until_timestamp', 'x_account_until_timestamp')
if groups:
meta['X-Account-Group'] = printable_header_dict(dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
meta['X-Account-Group'] = printable_header_dict(
dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
account_meta.append(printable_header_dict(meta))
if request.serialization == 'xml':
data = render_to_string('accounts.xml', {'accounts': account_meta})
......@@ -194,6 +205,7 @@ def account_list(request):
response.content = data
return response
@api_method('HEAD')
def account_meta(request, v_account):
# Normal Response Codes: 204
......@@ -203,9 +215,12 @@ def account_meta(request, v_account):
until = get_int_parameter(request.GET.get('until'))
try:
meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
groups = request.backend.get_account_groups(request.user_uniq, v_account)
policy = request.backend.get_account_policy(request.user_uniq, v_account)
meta = request.backend.get_account_meta(
request.user_uniq, v_account, 'pithos', until)
groups = request.backend.get_account_groups(
request.user_uniq, v_account)
policy = request.backend.get_account_policy(
request.user_uniq, v_account)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -215,6 +230,7 @@ def account_meta(request, v_account):
put_account_headers(response, meta, groups, policy)
return response
@api_method('POST')
def account_update(request, v_account):
# Normal Response Codes: 202
......@@ -242,6 +258,7 @@ def account_update(request, v_account):
raise Forbidden('Not allowed')
return HttpResponse(status=202)
@api_method('GET', format_allowed=True)
def container_list(request, v_account):
# Normal Response Codes: 200, 204
......@@ -252,9 +269,12 @@ def container_list(request, v_account):
until = get_int_parameter(request.GET.get('until'))
try:
meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
groups = request.backend.get_account_groups(request.user_uniq, v_account)
policy = request.backend.get_account_policy(request.user_uniq, v_account)
meta = request.backend.get_account_meta(
request.user_uniq, v_account, 'pithos', until)
groups = request.backend.get_account_groups(
request.user_uniq, v_account)
policy = request.backend.get_account_policy(
request.user_uniq, v_account)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -276,7 +296,8 @@ def container_list(request, v_account):
public = True
try:
containers = request.backend.list_containers(request.user_uniq, v_account,
containers = request.backend.list_containers(
request.user_uniq, v_account,
marker, limit, shared, until, public)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -295,7 +316,8 @@ def container_list(request, v_account):
container_meta = []
for x in containers:
try:
meta = request.backend.get_container_meta(request.user_uniq, v_account,
meta = request.backend.get_container_meta(
request.user_uniq, v_account,
x, 'pithos', until, include_user_defined=False)
policy = request.backend.get_container_policy(request.user_uniq,
v_account, x)
......@@ -305,18 +327,22 @@ def container_list(request, v_account):
pass
else:
rename_meta_key(meta, 'modified', 'last_modified')
rename_meta_key(meta, 'until_timestamp', 'x_container_until_timestamp')
rename_meta_key(
meta, 'until_timestamp', 'x_container_until_timestamp')
if policy:
meta['X-Container-Policy'] = printable_header_dict(dict([(k, v) for k, v in policy.iteritems()]))
meta['X-Container-Policy'] = printable_header_dict(
dict([(k, v) for k, v in policy.iteritems()]))
container_meta.append(printable_header_dict(meta))
if request.serialization == 'xml':
data = render_to_string('containers.xml', {'account': v_account, 'containers': container_meta})
data = render_to_string('containers.xml', {'account':
v_account, 'containers': container_meta})
elif request.serialization == 'json':
data = json.dumps(container_meta)
response.status_code = 200
response.content = data
return response
@api_method('HEAD')
def container_meta(request, v_account, v_container):
# Normal Response Codes: 204
......@@ -331,7 +357,8 @@ def container_meta(request, v_account, v_container):
v_container, 'pithos', until)
meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
v_account, v_container, 'pithos', until)
policy = request.backend.get_container_policy(request.user_uniq, v_account,
policy = request.backend.get_container_policy(
request.user_uniq, v_account,
v_container)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -344,6 +371,7 @@ def container_meta(request, v_account, v_container):
put_container_headers(request, response, meta, policy)
return response
@api_method('PUT')
def container_create(request, v_account, v_container):
# Normal Response Codes: 201, 202
......@@ -355,7 +383,8 @@ def container_create(request, v_account, v_container):
meta, policy = get_container_headers(request)
try:
request.backend.put_container(request.user_uniq, v_account, v_container, policy)
request.backend.put_container(
request.user_uniq, v_account, v_container, policy)
ret = 201
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -366,7 +395,8 @@ def container_create(request, v_account, v_container):
if ret == 202 and policy:
try:
request.backend.update_container_policy(request.user_uniq, v_account,
request.backend.update_container_policy(
request.user_uniq, v_account,
v_container, policy, replace=False)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -385,6 +415,7 @@ def container_create(request, v_account, v_container):
return HttpResponse(status=ret)
@api_method('POST', format_allowed=True)
def container_update(request, v_account, v_container):
# Normal Response Codes: 202
......@@ -399,7 +430,8 @@ def container_update(request, v_account, v_container):
replace = False
if policy:
try:
request.backend.update_container_policy(request.user_uniq, v_account,
request.backend.update_container_policy(
request.user_uniq, v_account,
v_container, policy, replace)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -418,7 +450,8 @@ def container_update(request, v_account, v_container):
content_length = -1
if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
content_length = get_int_parameter(request.META.get('CONTENT_LENGTH', 0))
content_length = get_int_parameter(
request.META.get('CONTENT_LENGTH', 0))
content_type = request.META.get('CONTENT_TYPE')
hashmap = []
if content_type and content_type == 'application/octet-stream' and content_length != 0:
......@@ -433,6 +466,7 @@ def container_update(request, v_account, v_container):
response.content = simple_list_response(request, hashmap)
return response
@api_method('DELETE')
def container_delete(request, v_account, v_container):
# Normal Response Codes: 204
......@@ -447,7 +481,8 @@ def container_delete(request, v_account, v_container):
delimiter = request.GET.get('delimiter')
try:
request.backend.delete_container(request.user_uniq, v_account, v_container,
request.backend.delete_container(
request.user_uniq, v_account, v_container,
until, delimiter=delimiter)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -457,6 +492,7 @@ def container_delete(request, v_account, v_container):
raise Conflict('Container is not empty')
return HttpResponse(status=204)
@api_method('GET', format_allowed=True)
def object_list(request, v_account, v_container):
# Normal Response Codes: 200, 204
......@@ -471,7 +507,8 @@ def object_list(request, v_account, v_container):
v_container, 'pithos', until)
meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
v_account, v_container, 'pithos', until)
policy = request.backend.get_container_policy(request.user_uniq, v_account,
policy = request.backend.get_container_policy(
request.user_uniq, v_account,
v_container)
except NotAllowedError:
raise Forbidden('Not allowed')
......@@ -508,12 +545,15 @@ def object_list(request, v_account, v_container):
keys = request.GET.get('meta')
if keys:
keys = [smart_str(x.strip()) for x in keys.split(',') if x.strip() != '']
keys = [smart_str(x.strip()) for x in keys.split(',')
if x.strip() != '']
included, excluded, opers = parse_filters(keys)
keys = []
keys += [format_header_key('X-Object-Meta-' + x) for x in included]
keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]
keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
keys += ['!' + format_header_key('X-Object-Meta-' + x)
for x in excluded]
keys += ['%s%s%s' % (format_header_key(
'X-Object-Meta-' + k), o, v) for k, o, v in opers]
else:
keys = []
......@@ -526,7 +566,8 @@ def object_list(request, v_account, v_container):
if request.serialization == 'text':
try:
objects = request.backend.list_objects(request.user_uniq, v_account,
objects = request.backend.list_objects(
request.user_uniq, v_account,
v_container, prefix, delimiter, marker,
limit, virtual, 'pithos', keys, shared,
until, None, public)
......@@ -544,7 +585,8 @@ def object_list(request, v_account, v_container):
return response
try:
objects = request.backend.list_object_meta(request.user_uniq, v_account,
objects = request.backend.list_object_meta(
request.user_uniq, v_account,
v_container, prefix, delimiter, marker,
limit, virtual, 'pithos', keys, shared, until, None, public)
object_permissions = {}
......@@ -570,7 +612,8 @@ def object_list(request, v_account, v_container):
# Virtual objects/directories.
object_meta.append(meta)
else:
rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
rename_meta_key(
meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
rename_meta_key(meta, 'checksum', 'hash')
rename_meta_key(meta, 'type', 'content_type')
rename_meta_key(meta, 'uuid', 'x_object_uuid')
......@@ -580,22 +623,26 @@ def object_list(request, v_account, v_container):
rename_meta_key(meta, 'modified', 'last_modified')
rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
rename_meta_key(meta, 'version', 'x_object_version')
rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
rename_meta_key(
meta, 'version_timestamp', 'x_object_version_timestamp')
permissions = object_permissions.get(meta['name'], None)
if permissions:
update_sharing_meta(request, permissions, v_account, v_container, meta['name'], meta)
update_sharing_meta(request, permissions, v_account,
v_container, meta['name'], meta)
public = object_public.get(meta['name'], None)
if public:
update_public_meta(public, meta)
object_meta.append(printable_header_dict(meta))
if request.serialization == 'xml':
data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
data = render_to_string(