Skip to content

GitLab

S
synnefo
Commit 7ac5bef1 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files
Options

Fix code formatting to conform to the PEP 8 style guide

parent 65f9b8fd
Hide whitespace changes
Inline Side-by-side
Showing with 2987 additions and 2519 deletions
other/migrate.py
View file @ 7ac5bef1
#!/usr/bin/env python
# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
......@@ -27,7 +27,7 @@
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
......@@ -41,41 +41,43 @@ from django.conf import settings
from pithos.backends.modular import ModularBackend
class Migration(object):
def __init__(self, db):
self.engine = create_engine(db)
self.metadata = MetaData(self.engine)
#self.engine.echo = True
self.conn = self.engine.connect()
options = getattr(settings, 'BACKEND', None)[1]
self.backend = ModularBackend(*options)
def execute(self):
pass
class Cache():
def __init__(self, db):
self.engine = create_engine(db)
metadata = MetaData(self.engine)
columns=[]
columns = []
columns.append(Column('path', String(2048), primary_key=True))
columns.append(Column('hash', String(255)))
self.files = Table('files', metadata, *columns)
self.conn = self.engine.connect()
self.engine.echo = True
metadata.create_all(self.engine)
def put(self, path, hash):
# Insert or replace.
s = self.files.delete().where(self.files.c.path==path)
s = self.files.delete().where(self.files.c.path == path)
r = self.conn.execute(s)
r.close()
s = self.files.insert()
r = self.conn.execute(s, {'path': path, 'hash': hash})
r.close()
def get(self, path):
s = select([self.files.c.hash], self.files.c.path == path)
r = self.conn.execute(s)
......
snf-pithos-app/distribute_setup.py
View file @ 7ac5bef1
......@@ -144,17 +144,17 @@ def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
except ImportError:
return _do_download(version, download_base, to_dir, download_delay)
try:
pkg_resources.require("distribute>="+version)
pkg_resources.require("distribute>=" + version)
return
except pkg_resources.VersionConflict:
e = sys.exc_info()[1]
if was_imported:
sys.stderr.write(
"The required version of distribute (>=%s) is not available,\n"
"and can't be installed while this script is running. Please\n"
"install a more recent version first, using\n"
"'easy_install -U distribute'."
"\n\n(Currently using %r)\n" % (version, e.args[0]))
"The required version of distribute (>=%s) is not available,\n"
"and can't be installed while this script is running. Please\n"
"install a more recent version first, using\n"
"'easy_install -U distribute'."
"\n\n(Currently using %r)\n" % (version, e.args[0]))
sys.exit(2)
else:
del pkg_resources, sys.modules['pkg_resources'] # reload ok
......@@ -167,6 +167,7 @@ def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
if not no_fake:
_create_fake_setuptools_pkg_info(to_dir)
def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
to_dir=os.curdir, delay=15):
"""Download distribute from a specified location and return its filename
......@@ -203,6 +204,7 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
dst.close()
return os.path.realpath(saveto)
def _no_sandbox(function):
def __no_sandbox(*args, **kw):
try:
......@@ -227,6 +229,7 @@ def _no_sandbox(function):
return __no_sandbox
def _patch_file(path, content):
"""Will backup the file then patch it"""
existing_content = open(path).read()
......@@ -245,15 +248,18 @@ def _patch_file(path, content):
_patch_file = _no_sandbox(_patch_file)
def _same_content(path, content):
return open(path).read() == content
def _rename_path(path):
new_name = path + '.OLD.%s' % time.time()
log.warn('Renaming %s into %s', path, new_name)
os.rename(path, new_name)
return new_name
def _remove_flat_installation(placeholder):
if not os.path.isdir(placeholder):
log.warn('Unkown installation at %s', placeholder)
......@@ -289,18 +295,20 @@ def _remove_flat_installation(placeholder):
_remove_flat_installation = _no_sandbox(_remove_flat_installation)
def _after_install(dist):
log.warn('After install bootstrap.')
placeholder = dist.get_command_obj('install').install_purelib
_create_fake_setuptools_pkg_info(placeholder)
def _create_fake_setuptools_pkg_info(placeholder):
if not placeholder or not os.path.exists(placeholder):
log.warn('Could not find the install location')
return
pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1])
setuptools_file = 'setuptools-%s-py%s.egg-info' % \
(SETUPTOOLS_FAKED_VERSION, pyver)
(SETUPTOOLS_FAKED_VERSION, pyver)
pkg_info = os.path.join(placeholder, setuptools_file)
if os.path.exists(pkg_info):
log.warn('%s already exists', pkg_info)
......@@ -321,7 +329,9 @@ def _create_fake_setuptools_pkg_info(placeholder):
finally:
f.close()
_create_fake_setuptools_pkg_info = _no_sandbox(_create_fake_setuptools_pkg_info)
_create_fake_setuptools_pkg_info = _no_sandbox(
_create_fake_setuptools_pkg_info)
def _patch_egg_dir(path):
# let's check if it's already patched
......@@ -343,6 +353,7 @@ def _patch_egg_dir(path):
_patch_egg_dir = _no_sandbox(_patch_egg_dir)
def _before_install():
log.warn('Before install bootstrap.')
_fake_setuptools()
......@@ -351,7 +362,7 @@ def _before_install():
def _under_prefix(location):
if 'install' not in sys.argv:
return True
args = sys.argv[sys.argv.index('install')+1:]
args = sys.argv[sys.argv.index('install') + 1:]
for index, arg in enumerate(args):
for option in ('--root', '--prefix'):
if arg.startswith('%s=' % option):
......@@ -359,7 +370,7 @@ def _under_prefix(location):
return location.startswith(top_dir)
elif arg == option:
if len(args) > index:
top_dir = args[index+1]
top_dir = args[index + 1]
return location.startswith(top_dir)
if arg == '--user' and USER_SITE is not None:
return location.startswith(USER_SITE)
......@@ -380,7 +391,8 @@ def _fake_setuptools():
replacement=False))
except TypeError:
# old distribute API
setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools'))
setuptools_dist = ws.find(
pkg_resources.Requirement.parse('setuptools'))
if setuptools_dist is None:
log.warn('No setuptools distribution found')
......@@ -406,7 +418,7 @@ def _fake_setuptools():
log.warn('Egg installation')
pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO')
if (os.path.exists(pkg_info) and
_same_content(pkg_info, SETUPTOOLS_PKG_INFO)):
_same_content(pkg_info, SETUPTOOLS_PKG_INFO)):
log.warn('Already patched.')
return
log.warn('Patching...')
......@@ -448,7 +460,7 @@ def _extractall(self, path=".", members=None):
# Extract directories with a safe mode.
directories.append(tarinfo)
tarinfo = copy.copy(tarinfo)
tarinfo.mode = 448 # decimal for oct 0700
tarinfo.mode = 448 # decimal for oct 0700
self.extract(tarinfo, path)
# Reverse sort directories.
......
snf-pithos-app/pithos/api/delegate.py
View file @ 7ac5bef1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
......@@ -25,7 +25,7 @@
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
......@@ -46,12 +46,13 @@ from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS, SERVIC
logger = logging.getLogger(__name__)
def delegate_to_login_service(request):
url = AUTHENTICATION_URL
users = AUTHENTICATION_USERS
if users or not url:
return HttpResponseNotFound()
p = urlparse(url)
if request.is_secure():
proto = 'https://'
......@@ -61,21 +62,22 @@ def delegate_to_login_service(request):
uri = proto + p.netloc + '/login?' + urlencode(params)
return HttpResponseRedirect(uri)
@csrf_exempt
def delegate_to_feedback_service(request):
url = AUTHENTICATION_URL
users = AUTHENTICATION_USERS
if users or not url:
return HttpResponseNotFound()
p = urlparse(url)
if request.is_secure():
proto = 'https://'
else:
proto = 'http://'
uri = proto + p.netloc + '/im/service/api/v2.0/feedback'
headers = { 'X-Auth-Token' : SERVICE_TOKEN }
headers = {'X-Auth-Token': SERVICE_TOKEN}
values = dict([(k, v) for k, v in request.POST.items()])
data = urllib.urlencode(values)
req = urllib2.Request(uri, data, headers)
......@@ -87,4 +89,4 @@ def delegate_to_feedback_service(request):
except urllib2.URLError, e:
logger.exception(e)
return HttpResponse(status=e.reason)
return HttpResponse()
\ No newline at end of file
return HttpResponse()
snf-pithos-app/pithos/api/dispatch.py
View file @ 7ac5bef1
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
BACKEND_BLOCK_UMASK,
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_CONNECTION,
BACKEND_QUOTA, BACKEND_VERSIONING)
BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
BACKEND_BLOCK_UMASK,
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_CONNECTION,
BACKEND_QUOTA, BACKEND_VERSIONING)
from pithos.backends import connect_backend
from pithos.api.util import hashmap_md5
......@@ -14,10 +14,11 @@ from astakos.im.settings import DEFAULT_FROM_EMAIL
import socket
from smtplib import SMTPException
def update_md5(m):
if m['resource'] != 'object' or m['details']['action'] != 'object update':
return
backend = connect_backend(db_module=BACKEND_DB_MODULE,
db_connection=BACKEND_DB_CONNECTION,
block_module=BACKEND_BLOCK_MODULE,
......@@ -27,38 +28,44 @@ def update_md5(m):
queue_connection=BACKEND_QUEUE_CONNECTION)
backend.default_policy['quota'] = BACKEND_QUOTA
backend.default_policy['versioning'] = BACKEND_VERSIONING
path = m['value']
account, container, name = path.split('/', 2)
version = m['details']['version']
meta = None
try:
meta = backend.get_object_meta(account, account, container, name, 'pithos', version)
meta = backend.get_object_meta(
account, account, container, name, 'pithos', version)
if meta['checksum'] == '':
size, hashmap = backend.get_object_hashmap(account, account, container, name, version)
size, hashmap = backend.get_object_hashmap(
account, account, container, name, version)
checksum = hashmap_md5(backend, hashmap, size)
backend.update_object_checksum(account, account, container, name, version, checksum)
backend.update_object_checksum(
account, account, container, name, version, checksum)
print 'INFO: Updated checksum for path "%s"' % (path,)
except Exception, e:
print 'WARNING: Can not update checksum for path "%s" (%s)' % (path, e)
backend.close()
def send_sharing_notification(m):
if m['resource'] != 'sharing':
return
members = m['details']['members']
user = m['details']['user']
path = m['value']
account, container, name = path.split('/', 2)
subject = 'Invitation to a Pithos+ shared object'
from_email = DEFAULT_FROM_EMAIL
recipient_list = members
message = 'User %s has invited you to a Pithos+ shared object. You can view it under "Shared to me" at "%s".' %(user, path)
message = 'User %s has invited you to a Pithos+ shared object. You can view it under "Shared to me" at "%s".' % (user, path)
try:
send_mail(subject, message, from_email, recipient_list)
print 'INFO: Sharing notification sent for path "%s" to %s' % (path, ','.join(recipient_list))
print 'INFO: Sharing notification sent for path "%s" to %s' % (
path, ','.join(recipient_list))
except (SMTPException, socket.error) as e:
print 'WARNING: Can not update send email for sharing "%s" (%s)' % (path, e)
print 'WARNING: Can not update send email for sharing "%s" (%s)' % (
path, e)
snf-pithos-app/pithos/api/faults.py
View file @ 7ac5bef1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
......@@ -25,12 +25,13 @@
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
def camelCase(s):
return s[0].lower() + s[1:]
......@@ -42,41 +43,54 @@ class Fault(Exception):
self.details = details
self.name = name or camelCase(self.__class__.__name__)
class NotModified(Fault):
code = 304
class BadRequest(Fault):
code = 400
class Unauthorized(Fault):
code = 401
class Forbidden(Fault):
code = 403
class ItemNotFound(Fault):
code = 404
class Conflict(Fault):
code = 409
class LengthRequired(Fault):
code = 411
class PreconditionFailed(Fault):
code = 412
class RequestEntityTooLarge(Fault):
code = 413
class RangeNotSatisfiable(Fault):
code = 416
class UnprocessableEntity(Fault):
code = 422
class InternalServerError(Fault):
code = 500
class NotImplemented(Fault):
code = 501
snf-pithos-app/pithos/api/functions.py
View file @ 7ac5bef1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
......@@ -25,7 +25,7 @@
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
......@@ -44,9 +44,11 @@ from django.views.decorators.csrf import csrf_exempt
from synnefo.lib.astakos import get_user
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
from pithos.api.faults import (
Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
from pithos.api.util import (
json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
......@@ -68,17 +70,18 @@ logger = logging.getLogger(__name__)
@csrf_exempt
def top_demux(request):
if request.method == 'GET':
try:
request.GET['X-Auth-Token']
except KeyError:
try:
request.META['HTTP_X_AUTH_TOKEN']
except KeyError:
return authenticate(request)
return account_list(request)
try:
request.GET['X-Auth-Token']
except KeyError:
try:
request.META['HTTP_X_AUTH_TOKEN']
except KeyError:
return authenticate(request)
return account_list(request)
else:
return method_not_allowed(request)
@csrf_exempt
def account_demux(request, v_account):
if request.method == 'HEAD':
......@@ -90,6 +93,7 @@ def account_demux(request, v_account):
else:
return method_not_allowed(request)
@csrf_exempt
def container_demux(request, v_account, v_container):
if request.method == 'HEAD':
......@@ -105,6 +109,7 @@ def container_demux(request, v_account, v_container):
else:
return method_not_allowed(request)
@csrf_exempt
def object_demux(request, v_account, v_container, v_object):
# Helper to avoid placing the token in the URL when loading objects from a browser.
......@@ -127,41 +132,44 @@ def object_demux(request, v_account, v_container, v_object):
else:
return method_not_allowed(request)
@api_method('GET', user_required=False)
def authenticate(request):
# Normal Response Codes: 204
# Error Response Codes: internalServerError (500),
# forbidden (403),
# badRequest (400)
x_auth_user = request.META.get('HTTP_X_AUTH_USER')
x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
if not x_auth_user or not x_auth_key:
raise BadRequest('Missing X-Auth-User or X-Auth-Key header')
response = HttpResponse(status=204)
uri = request.build_absolute_uri()
if '?' in uri:
uri = uri[:uri.find('?')]
response['X-Auth-Token'] = x_auth_key
response['X-Storage-Url'] = uri + ('' if uri.endswith('/') else '/') + x_auth_user
response['X-Storage-Url'] = uri + ('' if uri.endswith('/')
else '/') + x_auth_user
return response
@api_method('GET', format_allowed=True)
def account_list(request):
# Normal Response Codes: 200, 204
# Error Response Codes: internalServerError (500),
# badRequest (400)
response = HttpResponse()
marker = request.GET.get('marker')
limit = get_int_parameter(request.GET.get('limit'))
if not limit:
limit = 10000
accounts = request.backend.list_accounts(request.user_uniq, marker, limit)
if request.serialization == 'text':
if len(accounts) == 0:
# The cloudfiles python bindings expect 200 if json/xml.
......@@ -170,58 +178,66 @@ def account_list(request):
response.status_code = 200
response.content = '\n'.join(accounts) + '\n'
return response
account_meta = []
for x in accounts:
if x == request.user_uniq: