Commit 96a3cbf6 authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

Improve performance in listing images

Use '_get_domain_objects' function of Pithos backend, to list all
objects in a 'plankton' domain, instead of looping over all accessible
accounts, containers and objects. This function also returns metadata
and permissions for each object in order to avoid two extra calls for
each returned image.

This should provide a significant performance boost in listing available
images.

Note: Currently plankton API filters in 'list_images' are not
implemented.

Refs #3510
parent 73d42021
......@@ -32,6 +32,7 @@
# or implied, of GRNET S.A.
from logging import getLogger
from itertools import ifilter
from dateutil.parser import parse as date_parse
......@@ -120,16 +121,12 @@ def list_images(request, detail=False):
log.debug('list_images detail=%s', detail)
since = utils.isoparse(request.GET.get('changes-since'))
with image_backend(request.user_uniq) as backend:
images = backend.list_images()
if since:
images = []
for image in backend.iter():
updated = date_parse(image['updated_at'])
if updated >= since:
images.append(image)
updated_since = lambda img: date_parse(img["updated_at"]) >= since
images = ifilter(updated_since, images)
if not images:
return HttpResponse(status=304)
else:
images = backend.list()
images = sorted(images, key=lambda x: x['id'])
reply = [image_to_dict(image, detail) for image in images]
......
......@@ -66,7 +66,7 @@ class ImageAPITest(BaseAPITest):
images = [{'id': 1, 'name': 'image-1'},
{'id': 2, 'name': 'image-2'},
{'id': 3, 'name': 'image-3'}]
mimage().list.return_value = images
mimage().list_images.return_value = images
response = self.get('/api/v1.1/images/', 'user')
self.assertSuccess(response)
api_images = json.loads(response.content)['images']['values']
......@@ -115,7 +115,7 @@ class ImageAPITest(BaseAPITest):
'progress': 100,
'created': '2012-11-26T11:52:54+00:00',
'updated': '2012-12-26T11:52:54+00:00'}]
mimage().list.return_value = images
mimage().list_images.return_value = images
response = self.get('/api/v1.1/images/detail', 'user')
self.assertSuccess(response)
api_images = json.loads(response.content)['images']['values']
......@@ -146,7 +146,7 @@ class ImageAPITest(BaseAPITest):
'updated_at': new_time.isoformat(),
'deleted_at': new_time.isoformat(),
'properties': ''}]
mimage().iter.return_value = images
mimage().list_images.return_value = images
response =\
self.get('/api/v1.1/images/detail?changes-since=%sUTC' % new_time)
self.assertSuccess(response)
......@@ -235,7 +235,7 @@ class ImageMetadataAPITest(BaseAPITest):
backend.return_value.get_image.return_value = self.image
response = self.delete('/api/v1.1/images/42/meta/foo', 'user')
self.assertEqual(response.status_code, 204)
backend.return_value.update.assert_called_once_with('42', {'properties': {'foo2':
backend.return_value.update_metadata.assert_called_once_with('42', {'properties': {'foo2':
'bar2'}})
@assert_backend_closed
......@@ -245,7 +245,7 @@ class ImageMetadataAPITest(BaseAPITest):
response = self.put('/api/v1.1/images/42/meta/foo3', 'user',
json.dumps(request), 'json')
self.assertEqual(response.status_code, 201)
backend.return_value.update.assert_called_once_with('42',
backend.return_value.update_metadata.assert_called_once_with('42',
{'properties':
{'foo': 'bar', 'foo2': 'bar2', 'foo3': 'bar3'}})
......@@ -288,7 +288,7 @@ class ImageMetadataAPITest(BaseAPITest):
response = self.post('/api/v1.1/images/42/meta', 'user',
json.dumps(request), 'json')
self.assertEqual(response.status_code, 201)
backend.return_value.update.assert_called_once_with('42',
backend.return_value.update_metadata.assert_called_once_with('42',
{'properties':
{'foo': 'bar_new', 'foo2': 'bar2', 'foo4': 'bar4'}
})
......
......@@ -58,10 +58,8 @@ from time import gmtime, strftime
from functools import wraps
from operator import itemgetter
from django.conf import settings
from pithos.backends.base import NotAllowedError, VersionNotExists
import snf_django.lib.astakos as lib_astakos
logger = logging.getLogger(__name__)
......@@ -73,34 +71,6 @@ PROPERTY_PREFIX = 'property:'
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
'status')
TRANSLATE_UUIDS = getattr(settings, 'TRANSLATE_UUIDS', False)
def get_displaynames(names):
try:
auth_url = settings.ASTAKOS_URL
url = auth_url.replace('im/authenticate', 'service/api/user_catalogs')
token = settings.CYCLADES_ASTAKOS_SERVICE_TOKEN
uuids = lib_astakos.get_displaynames(token, names, url=url)
except Exception:
return {}
return uuids
def get_location(account, container, object):
assert '/' not in account, "Invalid account"
assert '/' not in container, "Invalid container"
return 'pithos://%s/%s/%s' % (account, container, object)
def split_location(location):
"""Returns (accout, container, object) from a location string"""
t = location.split('/', 4)
assert len(t) == 5, "Invalid location"
return t[2:5]
from pithos.backends.util import PithosBackendPool
POOL_SIZE = 8
_pithos_backend_pool = \
......@@ -183,58 +153,24 @@ class ImageBackend(object):
account, container, name = split_url(image_url)
versions = self.backend.list_versions(self.user, account, container,
name)
if not versions:
raise Exception("Image without versions %s" % image_url)
image = {}
try:
meta = self._get_meta(image_url)
image["deleted_at"] = ""
meta["deleted"] = ""
except NameError:
# Object was deleted, use the latest version
version, timestamp = versions[-1]
meta = self._get_meta(image_url, version)
image["deleted_at"] = format_timestamp(timestamp)
meta["deleted"] = timestamp
meta["created"] = versions[0][1]
if PLANKTON_PREFIX + 'name' not in meta:
raise ImageNotFound("'%s' is not a Plankton image" % image_url)
image["id"] = meta["uuid"]
image["location"] = image_url
image["checksum"] = meta["hash"]
image["created_at"] = format_timestamp(versions[0][1])
image["updated_at"] = format_timestamp(meta["modified"])
image["size"] = meta["bytes"]
image["store"] = "pithos"
if TRANSLATE_UUIDS:
displaynames = get_displaynames([account])
if account in displaynames:
display_account = displaynames[account]
else:
display_account = 'unknown'
image['owner'] = display_account
else:
image['owner'] = account
# Permissions
permissions = self._get_permissions(image_url)
image["is_public"] = "*" in permissions.get('read', [])
for key, val in meta.items():
# Get plankton properties
if key.startswith(PLANKTON_PREFIX):
# Remove plankton prefix
key = key.replace(PLANKTON_PREFIX, "")
# Keep only those in plankton meta
if key in PLANKTON_META:
if key == "properties":
val = json.loads(val)
image[key] = val
return image
return image_to_dict(image_url, meta, permissions)
def _get_meta(self, image_url, version=None):
"""Get object's metadata."""
......@@ -430,75 +366,51 @@ class ImageBackend(object):
self._update_permissions(image_url, permissions)
return self._get_image(image_url)
# TODO: Fix all these
def _iter(self, public=False, filters=None, shared_from=None):
def _list_images(self, user=None, filters=None, params=None):
filters = filters or {}
# Fix keys
keys = [PLANKTON_PREFIX + 'name']
size_range = (None, None)
for key, val in filters.items():
if key == 'size_min':
size_range = (val, size_range[1])
elif key == 'size_max':
size_range = (size_range[0], val)
else:
keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
backend = self.backend
if shared_from:
# To get shared images, we connect as shared_from member and
# get the list shared by us
user = shared_from
accounts = [self.user]
else:
user = None if public else self.user
accounts = backend.list_accounts(user)
for account in accounts:
for container in backend.list_containers(user, account,
shared=True):
for path, _ in backend.list_objects(user, account, container,
domain=PLANKTON_DOMAIN,
keys=keys, shared=True,
size_range=size_range):
location = get_location(account, container, path)
image = self._get_image(location)
if image:
yield image
@handle_backend_exceptions
def iter(self, filters=None):
"""Iter over all images available to the user"""
return self._iter(filters=filters)
@handle_backend_exceptions
def iter_public(self, filters=None):
"""Iter over public images"""
return self._iter(public=True, filters=filters)
@handle_backend_exceptions
def iter_shared(self, filters=None, member=None):
"""Iter over images shared to member"""
return self._iter(filters=filters, shared_from=member)
@handle_backend_exceptions
def list(self, filters=None, params={}):
"""Return all images available to the user"""
images = list(self.iter(filters))
# TODO: Use filters
# # Fix keys
# keys = [PLANKTON_PREFIX + 'name']
# size_range = (None, None)
# for key, val in filters.items():
# if key == 'size_min':
# size_range = (val, size_range[1])
# elif key == 'size_max':
# size_range = (size_range[0], val)
# else:
# keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
_images = self.backend.get_domain_objects(domain=PLANKTON_DOMAIN,
user=user)
images = []
for (location, meta, permissions) in _images:
image_url = "pithos://" + location
meta["modified"] = meta["version_timestamp"]
# TODO: Create metadata when registering an Image
meta["created"] = meta["version_timestamp"]
images.append(image_to_dict(image_url, meta, permissions))
if params is None:
params = {}
key = itemgetter(params.get('sort_key', 'created_at'))
reverse = params.get('sort_dir', 'desc') == 'desc'
images.sort(key=key, reverse=reverse)
return images
@handle_backend_exceptions
def list_public(self, filters, params={}):
"""Return public images"""
images = list(self.iter_public(filters))
key = itemgetter(params.get('sort_key', 'created_at'))
reverse = params.get('sort_dir', 'desc') == 'desc'
images.sort(key=key, reverse=reverse)
return images
def list_images(self, filters=None, params=None):
return self._list_images(user=self.user, filters=filters,
params=params)
def list_shared_images(self, member, filters=None, params=None):
images = self._list_images(user=self.user, filters=filters,
params=params)
is_shared = lambda img: not img["is_public"] and img["owner"] == member
return filter(is_shared, images)
def list_public_images(self, filters=None, params=None):
images = self._list_images(user=None, filters=filters, params=params)
return filter(lambda img: img["is_public"], images)
class ImageBackendError(Exception):
......@@ -511,3 +423,39 @@ class ImageNotFound(ImageBackendError):
class Forbidden(ImageBackendError):
pass
def image_to_dict(image_url, meta, permissions):
"""Render an image to a dictionary"""
account, container, name = split_url(image_url)
image = {}
if PLANKTON_PREFIX + 'name' not in meta:
raise ImageNotFound("'%s' is not a Plankton image" % image_url)
image["id"] = meta["uuid"]
image["location"] = image_url
image["checksum"] = meta["hash"]
image["created_at"] = format_timestamp(meta["created"])
deleted = meta.get("deleted", None)
image["deleted_at"] = format_timestamp(deleted) if deleted else ""
image["updated_at"] = format_timestamp(meta["modified"])
image["size"] = meta["bytes"]
image["store"] = "pithos"
image['owner'] = account
# Permissions
image["is_public"] = "*" in permissions.get('read', [])
for key, val in meta.items():
# Get plankton properties
if key.startswith(PLANKTON_PREFIX):
# Remove plankton prefix
key = key.replace(PLANKTON_PREFIX, "")
# Keep only those in plankton meta
if key in PLANKTON_META:
if key == "properties":
val = json.loads(val)
image[key] = val
return image
......@@ -131,7 +131,7 @@ def assert_backend_closed(func):
class PlanktonTest(BaseAPITest):
@assert_backend_closed
def test_list_images(self, backend):
backend.return_value.list.return_value =\
backend.return_value.list_images.return_value =\
deepcopy(DummyImages).values()
response = self.get("/plankton/images/")
self.assertSuccess(response)
......@@ -143,12 +143,12 @@ class PlanktonTest(BaseAPITest):
if key in LIST_FIELDS])
self.assertEqual(api_image, pithos_image)
backend.return_value\
.list.assert_called_once_with({}, {'sort_key': 'created_at',
.list_images.assert_called_once_with({}, {'sort_key': 'created_at',
'sort_dir': 'desc'})
@assert_backend_closed
def test_list_images_detail(self, backend):
backend.return_value.list.return_value =\
backend.return_value.list_images.return_value =\
deepcopy(DummyImages).values()
response = self.get("/plankton/images/detail")
self.assertSuccess(response)
......@@ -160,19 +160,19 @@ class PlanktonTest(BaseAPITest):
if key in DETAIL_FIELDS])
self.assertEqual(api_image, pithos_image)
backend.return_value\
.list.assert_called_once_with({}, {'sort_key': 'created_at',
.list_images.assert_called_once_with({}, {'sort_key': 'created_at',
'sort_dir': 'desc'})
@assert_backend_closed
def test_list_images_filters(self, backend):
backend.return_value.list.return_value =\
backend.return_value.list_images.return_value =\
deepcopy(DummyImages).values()
response = self.get("/plankton/images/?size_max=1000")
self.assertSuccess(response)
backend.return_value\
.list.assert_called_once_with({'size_max': 1000},
{'sort_key': 'created_at',
'sort_dir': 'desc'})
.list_images.assert_called_once_with({'size_max': 1000},
{'sort_key': 'created_at',
'sort_dir': 'desc'})
@assert_backend_closed
def test_list_images_filters_error_1(self, backend):
......@@ -191,8 +191,8 @@ class PlanktonTest(BaseAPITest):
json.dumps({}),
'json', HTTP_X_IMAGE_META_OWNER='user2')
self.assertSuccess(response)
backend.return_value.update.assert_called_once_with(db_image['id'],
{"owner": "user2"})
backend.return_value.update_metadata.assert_called_once_with(db_image['id'],
{"owner": "user2"})
@assert_backend_closed
def test_add_image_member(self, backend):
......
......@@ -291,7 +291,7 @@ def list_images(request, detail=False):
raise faults.BadRequest("Malformed request.")
with image_backend(request.user_uniq) as backend:
images = backend.list(filters, params)
images = backend.list_images(filters, params)
# Remove keys that should not be returned
fields = DETAIL_FIELDS if detail else LIST_FIELDS
......@@ -320,7 +320,7 @@ def list_shared_images(request, member):
images = []
with image_backend(request.user_uniq) as backend:
for image in backend.iter_shared(member=member):
for image in backend.list_shared_images(member=member):
image_id = image['id']
images.append({'image_id': image_id, 'can_share': False})
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment