Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
itminedu
synnefo
Commits
bf44e0c6
Commit
bf44e0c6
authored
Feb 20, 2014
by
Sofia Papagiannaki
Browse files
pithos: Fix TypeError when listing over-populated containers
Update tests Refs: #5119
parent
c93fb732
Changes
5
Hide whitespace changes
Inline
Side-by-side
snf-pithos-app/pithos/api/functions.py
View file @
bf44e0c6
...
...
@@ -590,7 +590,7 @@ def container_delete(request, v_account, v_container):
try
:
request
.
backend
.
delete_container
(
request
.
user_uniq
,
v_account
,
v_container
,
until
,
delimiter
=
delimiter
)
until
,
delimiter
=
delimiter
,
listing_limit
=
settings
.
API_LIST_LIMIT
)
except
NotAllowedError
:
raise
faults
.
Forbidden
(
'Not allowed'
)
except
ItemNotExists
:
...
...
@@ -1037,7 +1037,8 @@ def object_write(request, v_account, v_container, v_object):
version_id
=
copy_or_move_object
(
request
,
src_account
,
src_container
,
src_name
,
v_account
,
v_container
,
v_object
,
move
=
True
,
delimiter
=
delimiter
)
move
=
True
,
delimiter
=
delimiter
,
listing_limit
=
settings
.
API_LIST_LIMIT
)
else
:
try
:
src_container
,
src_name
=
split_container_object_string
(
...
...
@@ -1047,7 +1048,8 @@ def object_write(request, v_account, v_container, v_object):
version_id
=
copy_or_move_object
(
request
,
src_account
,
src_container
,
src_name
,
v_account
,
v_container
,
v_object
,
move
=
False
,
delimiter
=
delimiter
)
move
=
False
,
delimiter
=
delimiter
,
listing_limit
=
settings
.
API_LIST_LIMIT
)
response
=
HttpResponse
(
status
=
201
)
response
[
'X-Object-Version'
]
=
version_id
return
response
...
...
@@ -1229,7 +1231,8 @@ def object_copy(request, v_account, v_container, v_object):
version_id
=
copy_or_move_object
(
request
,
v_account
,
v_container
,
v_object
,
dest_account
,
dest_container
,
dest_name
,
move
=
False
,
delimiter
=
delimiter
)
move
=
False
,
delimiter
=
delimiter
,
listing_limit
=
settings
.
API_LIST_LIMIT
)
response
=
HttpResponse
(
status
=
201
)
response
[
'X-Object-Version'
]
=
version_id
return
response
...
...
@@ -1524,7 +1527,8 @@ def object_delete(request, v_account, v_container, v_object):
try
:
request
.
backend
.
delete_object
(
request
.
user_uniq
,
v_account
,
v_container
,
v_object
,
until
,
delimiter
=
delimiter
)
v_object
,
until
,
delimiter
=
delimiter
,
listing_limit
=
settings
.
API_LIST_LIMIT
)
except
NotAllowedError
:
raise
faults
.
Forbidden
(
'Not allowed'
)
except
ItemNotExists
:
...
...
snf-pithos-app/pithos/api/test/containers.py
View file @
bf44e0c6
...
...
@@ -933,13 +933,16 @@ class ContainerDelete(PithosAPITest):
r
=
self
.
delete
(
url
)
self
.
assertEqual
(
r
.
status_code
,
404
)
@
pithos_test_settings
(
API_LIST_LIMIT
=
10
)
def
test_delete_contents
(
self
):
folder
=
self
.
create_folder
(
'c1'
)[
0
]
descendant
=
strnextling
(
folder
)
self
.
upload_object
(
'c1'
,
descendant
)
for
i
in
range
(
11
):
descendant
=
'%s_%d'
%
(
strnextling
(
folder
),
i
)
self
.
upload_object
(
'c1'
,
descendant
)
self
.
create_folder
(
'c1'
,
'%s/%s'
%
(
folder
,
get_random_data
(
5
)))[
0
]
self
.
delete
(
'%s?delimiter=/'
%
join_urls
(
r
=
self
.
delete
(
'%s?delimiter=/'
%
join_urls
(
self
.
pithos_path
,
self
.
user
,
'c1'
))
self
.
assertEqual
(
r
.
status_code
,
204
)
self
.
assertEqual
([],
self
.
list_objects
(
'c1'
))
self
.
assertTrue
(
'c1'
in
self
.
list_containers
(
format
=
None
))
snf-pithos-app/pithos/api/test/objects.py
View file @
bf44e0c6
...
...
@@ -41,7 +41,7 @@ from functools import partial
from
pithos.api.test
import
(
PithosAPITest
,
pithos_settings
,
AssertMappingInvariant
,
AssertUUidInvariant
,
TEST_BLOCK_SIZE
,
TEST_HASH_ALGORITHM
,
DATE_FORMATS
)
DATE_FORMATS
,
pithos_test_settings
)
from
pithos.api.test.util
import
(
md5_hash
,
merkle
,
strnextling
,
get_random_data
,
get_random_name
)
...
...
@@ -924,18 +924,17 @@ class ObjectPutCopy(PithosAPITest):
get_random_name
(),
self
.
object
))
self
.
assertEqual
(
r
.
status_code
,
404
)
@
pithos_test_settings
(
API_LIST_LIMIT
=
10
)
def
test_copy_dir
(
self
):
folder
=
self
.
create_folder
(
self
.
container
)[
0
]
subfolder
=
self
.
create_folder
(
self
.
container
,
oname
=
'%s/%s'
%
(
folder
,
get_random_name
()))[
0
]
objects
=
[
subfolder
]
append
=
objects
.
append
append
(
self
.
upload_object
(
self
.
container
,
'%s/%s'
%
(
folder
,
get_random_name
()),
depth
=
'1'
)[
0
])
append
(
self
.
upload_object
(
self
.
container
,
'%s/%s'
%
(
subfolder
,
get_random_name
()),
depth
=
'2'
)[
0
])
for
i
in
range
(
11
):
append
(
self
.
upload_object
(
self
.
container
,
'%s/%d'
%
(
folder
,
i
),
depth
=
'1'
)[
0
])
other
=
self
.
upload_object
(
self
.
container
,
strnextling
(
folder
))[
0
]
# copy dir
...
...
@@ -1006,18 +1005,17 @@ class ObjectPutMove(PithosAPITest):
r
=
self
.
head
(
url
)
self
.
assertEqual
(
r
.
status_code
,
404
)
@
pithos_test_settings
(
API_LIST_LIMIT
=
10
)
def
test_move_dir
(
self
):
folder
=
self
.
create_folder
(
self
.
container
)[
0
]
subfolder
=
self
.
create_folder
(
self
.
container
,
oname
=
'%s/%s'
%
(
folder
,
get_random_name
()))[
0
]
objects
=
[
subfolder
]
append
=
objects
.
append
append
(
self
.
upload_object
(
self
.
container
,
'%s/%s'
%
(
folder
,
get_random_name
()),
depth
=
'1'
)[
0
])
append
(
self
.
upload_object
(
self
.
container
,
'%s/%s'
%
(
subfolder
,
get_random_name
()),
depth
=
'1'
)[
0
])
for
i
in
range
(
11
):
append
(
self
.
upload_object
(
self
.
container
,
'%s/%d'
%
(
folder
,
i
),
depth
=
'1'
)[
0
])
other
=
self
.
upload_object
(
self
.
container
,
strnextling
(
folder
))[
0
]
# move dir
...
...
@@ -1212,7 +1210,7 @@ class ObjectCopy(PithosAPITest):
self
.
assertTrue
(
'X-Object-Hash'
in
r
)
self
.
assertEqual
(
r
[
'X-Object-Hash'
],
self
.
etag
)
# assert source object still exists
# assert source object still exists
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
self
.
object
)
r
=
self
.
head
(
url
)
...
...
@@ -1229,6 +1227,43 @@ class ObjectCopy(PithosAPITest):
self
.
assertTrue
(
'X-Object-Hash'
in
r
)
self
.
assertEqual
(
r
[
'X-Object-Hash'
],
self
.
etag
)
@
pithos_test_settings
(
API_LIST_LIMIT
=
10
)
def
test_copy_dir_contents
(
self
):
folder
=
self
.
create_folder
(
self
.
container
)[
0
]
subfolder
=
self
.
create_folder
(
self
.
container
,
oname
=
'%s/%s'
%
(
folder
,
get_random_name
()))[
0
]
objects
=
[
subfolder
]
append
=
objects
.
append
for
i
in
range
(
11
):
append
(
self
.
upload_object
(
self
.
container
,
'%s/%d'
%
(
folder
,
i
),
depth
=
'1'
)[
0
])
other
=
self
.
upload_object
(
self
.
container
,
strnextling
(
folder
))[
0
]
# copy dir
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
folder
)
copy_folder
=
self
.
create_folder
(
self
.
container
)[
0
]
r
=
self
.
copy
(
'%s?delimiter=/'
%
url
,
HTTP_X_OBJECT_META_TEST
=
'testcopy'
,
HTTP_DESTINATION
=
'/%s/%s'
%
(
self
.
container
,
copy_folder
))
self
.
assertEqual
(
r
.
status_code
,
201
)
for
obj
in
objects
:
# assert object exists
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
obj
.
replace
(
folder
,
copy_folder
))
r
=
self
.
head
(
url
)
self
.
assertEqual
(
r
.
status_code
,
200
)
# assert other has not been created under copy folder
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
'%s/%s'
%
(
copy_folder
,
other
.
replace
(
folder
,
copy_folder
)))
r
=
self
.
head
(
url
)
self
.
assertEqual
(
r
.
status_code
,
404
)
def
test_copy_to_other_account
(
self
):
# create a container under alice account
cname
=
self
.
create_container
(
user
=
'alice'
)[
0
]
...
...
@@ -1347,6 +1382,42 @@ class ObjectMove(PithosAPITest):
r
=
self
.
head
(
url
)
self
.
assertEqual
(
r
.
status_code
,
404
)
@
pithos_test_settings
(
API_LIST_LIMIT
=
10
)
def
test_move_dir_contents
(
self
):
folder
=
self
.
create_folder
(
self
.
container
)[
0
]
subfolder
=
self
.
create_folder
(
self
.
container
,
oname
=
'%s/%s'
%
(
folder
,
get_random_name
()))[
0
]
objects
=
[
subfolder
]
append
=
objects
.
append
for
i
in
range
(
11
):
append
(
self
.
upload_object
(
self
.
container
,
'%s/%d'
%
(
folder
,
i
),
depth
=
'1'
)[
0
])
other
=
self
.
upload_object
(
self
.
container
,
strnextling
(
folder
))[
0
]
# copy dir
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
folder
)
copy_folder
=
self
.
create_folder
(
self
.
container
)[
0
]
r
=
self
.
move
(
'%s?delimiter=/'
%
url
,
HTTP_X_OBJECT_META_TEST
=
'testcopy'
,
HTTP_DESTINATION
=
'/%s/%s'
%
(
self
.
container
,
copy_folder
))
self
.
assertEqual
(
r
.
status_code
,
201
)
for
obj
in
objects
:
# assert object exists
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
obj
.
replace
(
folder
,
copy_folder
))
r
=
self
.
head
(
url
)
self
.
assertEqual
(
r
.
status_code
,
200
)
# assert other has not been created under copy folder
url
=
join_urls
(
self
.
pithos_path
,
self
.
user
,
self
.
container
,
'%s/%s'
%
(
copy_folder
,
other
.
replace
(
folder
,
copy_folder
)))
r
=
self
.
head
(
url
)
self
.
assertEqual
(
r
.
status_code
,
404
)
def
test_move_to_other_container
(
self
):
# move object to other container (not existing)
cname
=
get_random_name
()
...
...
@@ -1920,18 +1991,17 @@ class ObjectDelete(PithosAPITest):
r
=
self
.
delete
(
url
)
self
.
assertEqual
(
r
.
status_code
,
404
)
@
pithos_test_settings
(
API_LIST_LIMIT
=
10
)
def
test_delete_dir
(
self
):
folder
=
self
.
create_folder
(
self
.
container
)[
0
]
subfolder
=
self
.
create_folder
(
self
.
container
,
oname
=
'%s/%s'
%
(
folder
,
get_random_name
()))[
0
]
objects
=
[
subfolder
]
append
=
objects
.
append
append
(
self
.
upload_object
(
self
.
container
,
'%s/%s'
%
(
folder
,
get_random_name
()),
depth
=
'1'
)[
0
])
append
(
self
.
upload_object
(
self
.
container
,
'%s/%s'
%
(
subfolder
,
get_random_name
()),
depth
=
'2'
)[
0
])
for
i
in
range
(
11
):
append
(
self
.
upload_object
(
self
.
container
,
'%s/%d'
%
(
folder
,
i
),
depth
=
'1'
)[
0
])
other
=
self
.
upload_object
(
self
.
container
,
strnextling
(
folder
))[
0
]
# move dir
...
...
snf-pithos-app/pithos/api/util.py
View file @
bf44e0c6
...
...
@@ -482,7 +482,7 @@ def split_container_object_string(s):
def
copy_or_move_object
(
request
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
move
=
False
,
delimiter
=
None
):
move
=
False
,
delimiter
=
None
,
listing_limit
=
None
):
"""Copy or move an object."""
if
'ignore_content_type'
in
request
.
GET
and
'CONTENT_TYPE'
in
request
.
META
:
...
...
@@ -494,13 +494,14 @@ def copy_or_move_object(request, src_account, src_container, src_name,
version_id
=
request
.
backend
.
move_object
(
request
.
user_uniq
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
content_type
,
'pithos'
,
meta
,
False
,
permissions
,
delimiter
)
content_type
,
'pithos'
,
meta
,
False
,
permissions
,
delimiter
,
listing_limit
=
listing_limit
)
else
:
version_id
=
request
.
backend
.
copy_object
(
request
.
user_uniq
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
content_type
,
'pithos'
,
meta
,
False
,
permissions
,
src_version
,
delimiter
)
src_version
,
delimiter
,
listing_limit
=
listing_limit
)
except
NotAllowedError
:
raise
faults
.
Forbidden
(
'Not allowed'
)
except
(
ItemNotExists
,
VersionNotExists
):
...
...
snf-pithos-backend/pithos/backends/modular.py
View file @
bf44e0c6
...
...
@@ -683,7 +683,7 @@ class ModularBackend(BaseBackend):
@
debug_method
@
backend_method
def
delete_container
(
self
,
user
,
account
,
container
,
until
=
None
,
prefix
=
''
,
delimiter
=
None
):
delimiter
=
None
,
listing_limit
=
None
):
"""Delete/purge the container with the given name."""
self
.
_can_write_container
(
user
,
account
,
container
)
...
...
@@ -731,7 +731,8 @@ class ModularBackend(BaseBackend):
src_names
=
self
.
_list_objects_no_limit
(
user
,
account
,
container
,
prefix
=
''
,
delimiter
=
None
,
virtual
=
False
,
domain
=
None
,
keys
=
[],
shared
=
False
,
until
=
None
,
size_range
=
None
,
all_props
=
True
,
public
=
False
)
size_range
=
None
,
all_props
=
True
,
public
=
False
,
listing_limit
=
listing_limit
)
paths
=
[]
for
t
in
src_names
:
path
=
'/'
.
join
((
account
,
container
,
t
[
0
]))
...
...
@@ -820,11 +821,12 @@ class ModularBackend(BaseBackend):
def
_list_objects_no_limit
(
self
,
user
,
account
,
container
,
prefix
,
delimiter
,
virtual
,
domain
,
keys
,
shared
,
until
,
size_range
,
all_props
,
public
):
size_range
,
all_props
,
public
,
listing_limit
=
10000
):
objects
=
[]
while
True
:
marker
=
objects
[
-
1
]
if
objects
else
None
limit
=
10000
marker
=
objects
[
-
1
]
[
0
]
if
objects
else
None
limit
=
listing_limit
l
=
self
.
_list_objects
(
user
,
account
,
container
,
prefix
,
delimiter
,
marker
,
limit
,
virtual
,
domain
,
keys
,
shared
,
until
,
size_range
,
all_props
,
...
...
@@ -1207,7 +1209,7 @@ class ModularBackend(BaseBackend):
dest_account
,
dest_container
,
dest_name
,
type
,
dest_domain
=
None
,
dest_meta
=
None
,
replace_meta
=
False
,
permissions
=
None
,
src_version
=
None
,
is_move
=
False
,
delimiter
=
None
):
delimiter
=
None
,
listing_limit
=
10000
):
report_size_change
=
not
is_move
dest_meta
=
dest_meta
or
{}
...
...
@@ -1249,7 +1251,8 @@ class ModularBackend(BaseBackend):
src_names
=
self
.
_list_objects_no_limit
(
user
,
src_account
,
src_container
,
prefix
,
delimiter
=
None
,
virtual
=
False
,
domain
=
None
,
keys
=
[],
shared
=
False
,
until
=
None
,
size_range
=
None
,
all_props
=
True
,
public
=
False
)
size_range
=
None
,
all_props
=
True
,
public
=
False
,
listing_limit
=
listing_limit
)
src_names
.
sort
(
key
=
lambda
x
:
x
[
2
])
# order by nodes
paths
=
[
elem
[
0
]
for
elem
in
src_names
]
nodes
=
[
elem
[
2
]
for
elem
in
src_names
]
...
...
@@ -1284,14 +1287,15 @@ class ModularBackend(BaseBackend):
def
copy_object
(
self
,
user
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
type
,
domain
,
meta
=
None
,
replace_meta
=
False
,
permissions
=
None
,
src_version
=
None
,
delimiter
=
None
):
src_version
=
None
,
delimiter
=
None
,
listing_limit
=
None
):
"""Copy an object's data and metadata."""
meta
=
meta
or
{}
dest_version_id
=
self
.
_copy_object
(
user
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
type
,
domain
,
meta
,
replace_meta
,
permissions
,
src_version
,
False
,
delimiter
)
permissions
,
src_version
,
False
,
delimiter
,
listing_limit
=
listing_limit
)
return
dest_version_id
@
debug_method
...
...
@@ -1299,7 +1303,7 @@ class ModularBackend(BaseBackend):
def
move_object
(
self
,
user
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
type
,
domain
,
meta
=
None
,
replace_meta
=
False
,
permissions
=
None
,
delimiter
=
None
):
delimiter
=
None
,
listing_limit
=
None
):
"""Move an object's data and metadata."""
meta
=
meta
or
{}
...
...
@@ -1308,11 +1312,13 @@ class ModularBackend(BaseBackend):
dest_version_id
=
self
.
_move_object
(
user
,
src_account
,
src_container
,
src_name
,
dest_account
,
dest_container
,
dest_name
,
type
,
domain
,
meta
,
replace_meta
,
permissions
,
None
,
delimiter
=
delimiter
)
permissions
,
None
,
delimiter
=
delimiter
,
listing_limit
=
listing_limit
)
return
dest_version_id
def
_delete_object
(
self
,
user
,
account
,
container
,
name
,
until
=
None
,
delimiter
=
None
,
report_size_change
=
True
):
delimiter
=
None
,
report_size_change
=
True
,
listing_limit
=
None
):
if
user
!=
account
:
raise
NotAllowedError
...
...
@@ -1377,7 +1383,8 @@ class ModularBackend(BaseBackend):
src_names
=
self
.
_list_objects_no_limit
(
user
,
account
,
container
,
prefix
,
delimiter
=
None
,
virtual
=
False
,
domain
=
None
,
keys
=
[],
shared
=
False
,
until
=
None
,
size_range
=
None
,
all_props
=
True
,
public
=
False
)
size_range
=
None
,
all_props
=
True
,
public
=
False
,
listing_limit
=
listing_limit
)
paths
=
[]
for
t
in
src_names
:
path
=
'/'
.
join
((
account
,
container
,
t
[
0
]))
...
...
@@ -1409,10 +1416,11 @@ class ModularBackend(BaseBackend):
@
debug_method
@
backend_method
def
delete_object
(
self
,
user
,
account
,
container
,
name
,
until
=
None
,
prefix
=
''
,
delimiter
=
None
):
prefix
=
''
,
delimiter
=
None
,
listing_limit
=
None
):
"""Delete/purge an object."""
self
.
_delete_object
(
user
,
account
,
container
,
name
,
until
,
delimiter
)
self
.
_delete_object
(
user
,
account
,
container
,
name
,
until
,
delimiter
,
listing_limit
=
listing_limit
)
@
debug_method
@
backend_method
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment