Skip to content

GitLab

Commit bf44e0c6 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

pithos: Fix TypeError when listing over-populated containers 

Update tests

Refs: #5119
parent c93fb732
Hide whitespace changes
Inline Side-by-side
Showing with 132 additions and 46 deletions
snf-pithos-app/pithos/api/functions.py
View file @ bf44e0c6
......@@ -590,7 +590,7 @@ def container_delete(request, v_account, v_container):
try:
request.backend.delete_container(
request.user_uniq, v_account, v_container,
until, delimiter=delimiter)
until, delimiter=delimiter, listing_limit=settings.API_LIST_LIMIT)
except NotAllowedError:
raise faults.Forbidden('Not allowed')
except ItemNotExists:
......@@ -1037,7 +1037,8 @@ def object_write(request, v_account, v_container, v_object):
version_id = copy_or_move_object(
request, src_account, src_container, src_name,
v_account, v_container, v_object,
move=True, delimiter=delimiter)
move=True, delimiter=delimiter,
listing_limit=settings.API_LIST_LIMIT)
else:
try:
src_container, src_name = split_container_object_string(
......@@ -1047,7 +1048,8 @@ def object_write(request, v_account, v_container, v_object):
version_id = copy_or_move_object(
request, src_account, src_container, src_name,
v_account, v_container, v_object,
move=False, delimiter=delimiter)
move=False, delimiter=delimiter,
listing_limit=settings.API_LIST_LIMIT)
response = HttpResponse(status=201)
response['X-Object-Version'] = version_id
return response
......@@ -1229,7 +1231,8 @@ def object_copy(request, v_account, v_container, v_object):
version_id = copy_or_move_object(request, v_account, v_container, v_object,
dest_account, dest_container, dest_name,
move=False, delimiter=delimiter)
move=False, delimiter=delimiter,
listing_limit=settings.API_LIST_LIMIT)
response = HttpResponse(status=201)
response['X-Object-Version'] = version_id
return response
......@@ -1524,7 +1527,8 @@ def object_delete(request, v_account, v_container, v_object):
try:
request.backend.delete_object(
request.user_uniq, v_account, v_container,
v_object, until, delimiter=delimiter)
v_object, until, delimiter=delimiter,
listing_limit=settings.API_LIST_LIMIT)
except NotAllowedError:
raise faults.Forbidden('Not allowed')
except ItemNotExists:
......
snf-pithos-app/pithos/api/test/containers.py
View file @ bf44e0c6
......@@ -933,13 +933,16 @@ class ContainerDelete(PithosAPITest):
r = self.delete(url)
self.assertEqual(r.status_code, 404)
@pithos_test_settings(API_LIST_LIMIT=10)
def test_delete_contents(self):
folder = self.create_folder('c1')[0]
descendant = strnextling(folder)
self.upload_object('c1', descendant)
for i in range(11):
descendant = '%s_%d' % (strnextling(folder), i)
self.upload_object('c1', descendant)
self.create_folder('c1', '%s/%s' % (folder, get_random_data(5)))[0]
self.delete('%s?delimiter=/' % join_urls(
r = self.delete('%s?delimiter=/' % join_urls(
self.pithos_path, self.user, 'c1'))
self.assertEqual(r.status_code, 204)
self.assertEqual([], self.list_objects('c1'))
self.assertTrue('c1' in self.list_containers(format=None))
snf-pithos-app/pithos/api/test/objects.py
View file @ bf44e0c6
......@@ -41,7 +41,7 @@ from functools import partial
from pithos.api.test import (PithosAPITest, pithos_settings,
AssertMappingInvariant, AssertUUidInvariant,
TEST_BLOCK_SIZE, TEST_HASH_ALGORITHM,
DATE_FORMATS)
DATE_FORMATS, pithos_test_settings)
from pithos.api.test.util import (md5_hash, merkle, strnextling,
get_random_data, get_random_name)
......@@ -924,18 +924,17 @@ class ObjectPutCopy(PithosAPITest):
get_random_name(), self.object))
self.assertEqual(r.status_code, 404)
@pithos_test_settings(API_LIST_LIMIT=10)
def test_copy_dir(self):
folder = self.create_folder(self.container)[0]
subfolder = self.create_folder(
self.container, oname='%s/%s' % (folder, get_random_name()))[0]
objects = [subfolder]
append = objects.append
append(self.upload_object(self.container,
'%s/%s' % (folder, get_random_name()),
depth='1')[0])
append(self.upload_object(self.container,
'%s/%s' % (subfolder, get_random_name()),
depth='2')[0])
for i in range(11):
append(self.upload_object(self.container,
'%s/%d' % (folder, i),
depth='1')[0])
other = self.upload_object(self.container, strnextling(folder))[0]
# copy dir
......@@ -1006,18 +1005,17 @@ class ObjectPutMove(PithosAPITest):
r = self.head(url)
self.assertEqual(r.status_code, 404)
@pithos_test_settings(API_LIST_LIMIT=10)
def test_move_dir(self):
folder = self.create_folder(self.container)[0]
subfolder = self.create_folder(
self.container, oname='%s/%s' % (folder, get_random_name()))[0]
objects = [subfolder]
append = objects.append
append(self.upload_object(self.container,
'%s/%s' % (folder, get_random_name()),
depth='1')[0])
append(self.upload_object(self.container,
'%s/%s' % (subfolder, get_random_name()),
depth='1')[0])
for i in range(11):
append(self.upload_object(self.container,
'%s/%d' % (folder, i),
depth='1')[0])
other = self.upload_object(self.container, strnextling(folder))[0]
# move dir
......@@ -1212,7 +1210,7 @@ class ObjectCopy(PithosAPITest):
self.assertTrue('X-Object-Hash' in r)
self.assertEqual(r['X-Object-Hash'], self.etag)
# assert source object still exists
# assert source object still exists
url = join_urls(self.pithos_path, self.user, self.container,
self.object)
r = self.head(url)
......@@ -1229,6 +1227,43 @@ class ObjectCopy(PithosAPITest):
self.assertTrue('X-Object-Hash' in r)
self.assertEqual(r['X-Object-Hash'], self.etag)
@pithos_test_settings(API_LIST_LIMIT=10)
def test_copy_dir_contents(self):
folder = self.create_folder(self.container)[0]
subfolder = self.create_folder(
self.container, oname='%s/%s' % (folder, get_random_name()))[0]
objects = [subfolder]
append = objects.append
for i in range(11):
append(self.upload_object(self.container,
'%s/%d' % (folder, i),
depth='1')[0])
other = self.upload_object(self.container, strnextling(folder))[0]
# copy dir
url = join_urls(self.pithos_path, self.user, self.container,
folder)
copy_folder = self.create_folder(self.container)[0]
r = self.copy('%s?delimiter=/' % url,
HTTP_X_OBJECT_META_TEST='testcopy',
HTTP_DESTINATION='/%s/%s' % (self.container,
copy_folder))
self.assertEqual(r.status_code, 201)
for obj in objects:
# assert object exists
url = join_urls(self.pithos_path, self.user, self.container,
obj.replace(folder, copy_folder))
r = self.head(url)
self.assertEqual(r.status_code, 200)
# assert other has not been created under copy folder
url = join_urls(self.pithos_path, self.user, self.container,
'%s/%s' % (copy_folder,
other.replace(folder, copy_folder)))
r = self.head(url)
self.assertEqual(r.status_code, 404)
def test_copy_to_other_account(self):
# create a container under alice account
cname = self.create_container(user='alice')[0]
......@@ -1347,6 +1382,42 @@ class ObjectMove(PithosAPITest):
r = self.head(url)
self.assertEqual(r.status_code, 404)
@pithos_test_settings(API_LIST_LIMIT=10)
def test_move_dir_contents(self):
folder = self.create_folder(self.container)[0]
subfolder = self.create_folder(
self.container, oname='%s/%s' % (folder, get_random_name()))[0]
objects = [subfolder]
append = objects.append
for i in range(11):
append(self.upload_object(self.container,
'%s/%d' % (folder, i),
depth='1')[0])
other = self.upload_object(self.container, strnextling(folder))[0]
# copy dir
url = join_urls(self.pithos_path, self.user, self.container, folder)
copy_folder = self.create_folder(self.container)[0]
r = self.move('%s?delimiter=/' % url,
HTTP_X_OBJECT_META_TEST='testcopy',
HTTP_DESTINATION='/%s/%s' % (self.container,
copy_folder))
self.assertEqual(r.status_code, 201)
for obj in objects:
# assert object exists
url = join_urls(self.pithos_path, self.user, self.container,
obj.replace(folder, copy_folder))
r = self.head(url)
self.assertEqual(r.status_code, 200)
# assert other has not been created under copy folder
url = join_urls(self.pithos_path, self.user, self.container,
'%s/%s' % (copy_folder,
other.replace(folder, copy_folder)))
r = self.head(url)
self.assertEqual(r.status_code, 404)
def test_move_to_other_container(self):
# move object to other container (not existing)
cname = get_random_name()
......@@ -1920,18 +1991,17 @@ class ObjectDelete(PithosAPITest):
r = self.delete(url)
self.assertEqual(r.status_code, 404)
@pithos_test_settings(API_LIST_LIMIT=10)
def test_delete_dir(self):
folder = self.create_folder(self.container)[0]
subfolder = self.create_folder(
self.container, oname='%s/%s' % (folder, get_random_name()))[0]
objects = [subfolder]
append = objects.append
append(self.upload_object(self.container,
'%s/%s' % (folder, get_random_name()),
depth='1')[0])
append(self.upload_object(self.container,
'%s/%s' % (subfolder, get_random_name()),
depth='2')[0])
for i in range(11):
append(self.upload_object(self.container,
'%s/%d' % (folder, i),
depth='1')[0])
other = self.upload_object(self.container, strnextling(folder))[0]
# move dir
......
snf-pithos-app/pithos/api/util.py
View file @ bf44e0c6
......@@ -482,7 +482,7 @@ def split_container_object_string(s):
def copy_or_move_object(request, src_account, src_container, src_name,
dest_account, dest_container, dest_name,
move=False, delimiter=None):
move=False, delimiter=None, listing_limit=None):
"""Copy or move an object."""
if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
......@@ -494,13 +494,14 @@ def copy_or_move_object(request, src_account, src_container, src_name,
version_id = request.backend.move_object(
request.user_uniq, src_account, src_container, src_name,
dest_account, dest_container, dest_name,
content_type, 'pithos', meta, False, permissions, delimiter)
content_type, 'pithos', meta, False, permissions, delimiter,
listing_limit=listing_limit)
else:
version_id = request.backend.copy_object(
request.user_uniq, src_account, src_container, src_name,
dest_account, dest_container, dest_name,
content_type, 'pithos', meta, False, permissions,
src_version, delimiter)
src_version, delimiter, listing_limit=listing_limit)
except NotAllowedError:
raise faults.Forbidden('Not allowed')
except (ItemNotExists, VersionNotExists):
......
snf-pithos-backend/pithos/backends/modular.py
View file @ bf44e0c6
......@@ -683,7 +683,7 @@ class ModularBackend(BaseBackend):
@debug_method
@backend_method
def delete_container(self, user, account, container, until=None, prefix='',
delimiter=None):
delimiter=None, listing_limit=None):
"""Delete/purge the container with the given name."""
self._can_write_container(user, account, container)
......@@ -731,7 +731,8 @@ class ModularBackend(BaseBackend):
src_names = self._list_objects_no_limit(
user, account, container, prefix='', delimiter=None,
virtual=False, domain=None, keys=[], shared=False, until=None,
size_range=None, all_props=True, public=False)
size_range=None, all_props=True, public=False,
listing_limit=listing_limit)
paths = []
for t in src_names:
path = '/'.join((account, container, t[0]))
......@@ -820,11 +821,12 @@ class ModularBackend(BaseBackend):
def _list_objects_no_limit(self, user, account, container, prefix,
delimiter, virtual, domain, keys, shared, until,
size_range, all_props, public):
size_range, all_props, public,
listing_limit=10000):
objects = []
while True:
marker = objects[-1] if objects else None
limit = 10000
marker = objects[-1][0] if objects else None
limit = listing_limit
l = self._list_objects(
user, account, container, prefix, delimiter, marker, limit,
virtual, domain, keys, shared, until, size_range, all_props,
......@@ -1207,7 +1209,7 @@ class ModularBackend(BaseBackend):
dest_account, dest_container, dest_name, type,
dest_domain=None, dest_meta=None, replace_meta=False,
permissions=None, src_version=None, is_move=False,
delimiter=None):
delimiter=None, listing_limit=10000):
report_size_change = not is_move
dest_meta = dest_meta or {}
......@@ -1249,7 +1251,8 @@ class ModularBackend(BaseBackend):
src_names = self._list_objects_no_limit(
user, src_account, src_container, prefix, delimiter=None,
virtual=False, domain=None, keys=[], shared=False, until=None,
size_range=None, all_props=True, public=False)
size_range=None, all_props=True, public=False,
listing_limit=listing_limit)
src_names.sort(key=lambda x: x[2]) # order by nodes
paths = [elem[0] for elem in src_names]
nodes = [elem[2] for elem in src_names]
......@@ -1284,14 +1287,15 @@ class ModularBackend(BaseBackend):
def copy_object(self, user, src_account, src_container, src_name,
dest_account, dest_container, dest_name, type, domain,
meta=None, replace_meta=False, permissions=None,
src_version=None, delimiter=None):
src_version=None, delimiter=None, listing_limit=None):
"""Copy an object's data and metadata."""
meta = meta or {}
dest_version_id = self._copy_object(
user, src_account, src_container, src_name, dest_account,
dest_container, dest_name, type, domain, meta, replace_meta,
permissions, src_version, False, delimiter)
permissions, src_version, False, delimiter,
listing_limit=listing_limit)
return dest_version_id
@debug_method
......@@ -1299,7 +1303,7 @@ class ModularBackend(BaseBackend):
def move_object(self, user, src_account, src_container, src_name,
dest_account, dest_container, dest_name, type, domain,
meta=None, replace_meta=False, permissions=None,
delimiter=None):
delimiter=None, listing_limit=None):
"""Move an object's data and metadata."""
meta = meta or {}
......@@ -1308,11 +1312,13 @@ class ModularBackend(BaseBackend):
dest_version_id = self._move_object(
user, src_account, src_container, src_name, dest_account,
dest_container, dest_name, type, domain, meta, replace_meta,
permissions, None, delimiter=delimiter)
permissions, None, delimiter=delimiter,
listing_limit=listing_limit)
return dest_version_id
def _delete_object(self, user, account, container, name, until=None,
delimiter=None, report_size_change=True):
delimiter=None, report_size_change=True,
listing_limit=None):
if user != account:
raise NotAllowedError
......@@ -1377,7 +1383,8 @@ class ModularBackend(BaseBackend):
src_names = self._list_objects_no_limit(
user, account, container, prefix, delimiter=None,
virtual=False, domain=None, keys=[], shared=False, until=None,
size_range=None, all_props=True, public=False)
size_range=None, all_props=True, public=False,
listing_limit=listing_limit)
paths = []
for t in src_names:
path = '/'.join((account, container, t[0]))
......@@ -1409,10 +1416,11 @@ class ModularBackend(BaseBackend):
@debug_method
@backend_method
def delete_object(self, user, account, container, name, until=None,
prefix='', delimiter=None):
prefix='', delimiter=None, listing_limit=None):
"""Delete/purge an object."""
self._delete_object(user, account, container, name, until, delimiter)
self._delete_object(user, account, container, name, until, delimiter,
listing_limit=listing_limit)
@debug_method
@backend_method
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment