Commit 776b275c authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Use multiformed ranges in kamaki pithos lib

Refs: #4059

Also, adjust corresponding unit and functional tests
parent 1e1d6f27
......@@ -131,18 +131,23 @@ class RangeArgument(ValueArgument):
return getattr(self, '_value', self.default)
@value.setter
def value(self, newvalue):
if newvalue is None:
def value(self, newvalues):
if not newvalues:
self._value = self.default
return
start, sep, end = newvalue.partition('-')
if sep:
if start:
self._value = '%s-%s' % (int(start), int(end))
self._value = ''
for newvalue in newvalues.split(','):
self._value = ('%s,' % self._value) if self._value else ''
start, sep, end = newvalue.partition('-')
if sep:
if start:
start, end = (int(start), int(end))
assert start <= end, 'Invalid range value %s' % newvalue
self._value += '%s-%s' % (int(start), int(end))
else:
self._value += '-%s' % int(end)
else:
self._value = '-%s' % int(end)
else:
self._value = '%s' % int(start)
self._value += '%s' % int(start)
# Command specs
......
......@@ -322,7 +322,7 @@ class Client(object):
MAX_THREADS = 7
DATE_FORMATS = ['%a %b %d %H:%M:%S %Y', ]
LOG_TOKEN = False
LOG_DATA = True
LOG_DATA = False
CONNECTION_RETRY_LIMIT = 0
def __init__(self, base_url, token):
......
......@@ -556,9 +556,12 @@ class Pithos(livetest.Generic):
r = self.client.object_head(obj)
self.assertEqual(r.status_code, 200)
etag = r.headers['etag']
real_version = r.headers['x-object-version']
r = self.client.object_head(obj, version=40)
self.assertEqual(r.headers['x-object-version'], '40')
self.assertRaises(
ClientError, self.client.object_head, obj, version=-10)
r = self.client.object_head(obj, version=real_version)
self.assertEqual(r.headers['x-object-version'], real_version)
r = self.client.object_head(obj, if_etag_match=etag)
self.assertEqual(r.status_code, 200)
......@@ -568,8 +571,8 @@ class Pithos(livetest.Generic):
self.assertNotEqual(r.status_code, 200)
r = self.client.object_head(
obj, version=40, if_etag_match=etag, success=412)
self.assertEqual(r.status_code, 412)
obj, version=real_version, if_etag_match=etag, success=200)
self.assertEqual(r.status_code, 200)
"""Check and if(un)modified_since"""
for format in self.client.DATE_FORMATS:
......@@ -1254,6 +1257,7 @@ class Pithos(livetest.Generic):
def create_large_file(self, size):
"""Create a large file at fs"""
print
self.files.append(NamedTemporaryFile())
f = self.files[-1]
Ki = size / 8
......
......@@ -52,25 +52,49 @@ def _pithos_hash(block, blockhash):
return h.hexdigest()
def _range_up(start, end, a_range):
if a_range:
rstart, sep, rend = a_range.partition('-')
if rstart:
def _range_up(start, end, max_value, a_range):
"""
:param start: (int) the window bottom
:param end: (int) the window top
:param max_value: (int) maximum accepted value
:param a_range: (str) a range string in the form X[,X'[,X''[...]]]
where X: x|x-y|-x where x < y and x, y natural numbers
:returns: (str) a range string cut-off for the start-end range
an empty response means this window is out of range
"""
assert start >= 0, '_range_up was called with start < 0'
assert end >= start, '_range_up was called with end < start'
assert end <= max_value, '_range_up was called with max_value < end'
if not a_range:
return '%s-%s' % (start, end)
selected = []
for some_range in a_range.split(','):
v0, sep, v1 = some_range.partition('-')
if v0:
v0 = int(v0)
if sep:
rstart, rend = int(rstart), int(rend)
v1 = int(v1)
if v1 < start or v0 > end or v1 < v0:
continue
v0 = v0 if v0 > start else start
v1 = v1 if v1 < end else end
selected.append('%s-%s' % (v0, v1))
elif v0 < start:
continue
else:
rstart, rend = 0, int(rstart)
elif sep:
return (0, - int(rend))
v1 = v0 if v0 <= end else end
selected.append('%s-%s' % (start, v1))
else:
return (0, 0)
if rstart > end or rend < start:
return (0, 0)
if rstart > start:
start = rstart
if rend < end:
end = rend
return (start, end)
v1 = int(v1)
if max_value - v1 > end:
continue
v0 = (max_value - v1) if max_value - v1 > start else start
selected.append('%s-%s' % (v0, end))
return ','.join(selected)
class PithosClient(PithosRestClient):
......@@ -641,9 +665,11 @@ class PithosClient(PithosRestClient):
start = blocksize * blockid
is_last = start + blocksize > total_size
end = (total_size - 1) if is_last else (start + blocksize - 1)
(start, end) = _range_up(start, end, crange)
args['data_range'] = 'bytes=%s-%s' % (
(start, end) if end >= 0 else ('', - end))
data_range = _range_up(start, end, total_size, crange)
if not data_range:
self._cb_next()
continue
args['data_range'] = 'bytes=%s' % data_range
r = self.object_get(obj, success=(200, 206), **args)
self._cb_next()
dst.write(r.content)
......@@ -689,9 +715,6 @@ class PithosClient(PithosRestClient):
flying = dict()
blockid_dict = dict()
offset = 0
if filerange is not None:
rstart = int(filerange.split('-')[0])
offset = rstart if blocksize > rstart else rstart % blocksize
self._init_thread_limit()
for block_hash, blockids in remote_hashes.items():
......@@ -708,13 +731,11 @@ class PithosClient(PithosRestClient):
**restargs)
end = total_size - 1 if (
key + blocksize > total_size) else key + blocksize - 1
start, end = _range_up(key, end, filerange)
if start == end:
data_range = _range_up(key, end, total_size, filerange)
if not data_range:
self._cb_next()
continue
restargs['async_headers'] = {
'Range': 'bytes=%s-%s' % (
(start, end) if end >= 0 else ('', - end))}
restargs['async_headers'] = {'Range': 'bytes=%s' % data_range}
flying[key] = self._get_block_async(obj, **restargs)
blockid_dict[key] = unsaved
......@@ -857,9 +878,10 @@ class PithosClient(PithosRestClient):
start = blocksize * blockid
is_last = start + blocksize > total_size
end = (total_size - 1) if is_last else (start + blocksize - 1)
(start, end) = _range_up(start, end, range_str)
if start < end or end < 0:
data_range_str = _range_up(start, end, end, range_str)
if data_range_str:
self._watch_thread_limit(flying.values())
restargs['data_range'] = 'bytes=%s' % data_range_str
flying[blockid] = self._get_block_async(obj, **restargs)
for runid, thread in flying.items():
if (blockid + 1) == num_of_blocks:
......@@ -899,8 +921,7 @@ class PithosClient(PithosRestClient):
if_match=None,
if_none_match=None,
if_modified_since=None,
if_unmodified_since=None,
data_range=None):
if_unmodified_since=None):
"""
:param obj: (str) remote object path
......@@ -912,9 +933,6 @@ class PithosClient(PithosRestClient):
:param if_unmodified_since: (str) formated date
:param data_range: (str) from-to where from and to are integers
denoting file positions in bytes
:returns: (list)
"""
try:
......@@ -925,8 +943,7 @@ class PithosClient(PithosRestClient):
if_etag_match=if_match,
if_etag_not_match=if_none_match,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since,
data_range=data_range)
if_unmodified_since=if_unmodified_since)
except ClientError as err:
if err.status == 304 or err.status == 412:
return {}
......
......@@ -242,7 +242,7 @@ class PithosRestClient(TestCase):
pm = pm[:-2]
self.client.account_post(*(pm + args), **kwargs)
upd = pm[0]
self.assertEqual(SP.mock_calls[-1], call('update', iff=upd))
self.assertEqual(SP.mock_calls[-1], call('update', '', iff=upd))
expected = []
if pm[1]:
expected += [
......@@ -369,7 +369,7 @@ class PithosRestClient(TestCase):
self.client.container_post(*(pm + args), **kwargs)
upd, frmt = pm[:2]
self.assertEqual(SP.mock_calls[-2:], [
call('update', iff=upd),
call('update', '', iff=upd),
call('format', frmt, iff=frmt)])
qta, vrs, metas, ctype, clen, trenc = pm[2:]
prfx = 'X-Container-Meta-'
......@@ -637,7 +637,9 @@ class PithosRestClient(TestCase):
if pval:
perm_str += ';' if perm_str else ''
perm_str += '%s=%s' % (ptype, ','.join(pval))
exp += [call('X-Object-Sharing', perm_str)]
exp += [call('X-Object-Sharing', perm_str, iff=perms)]
else:
exp += [call('X-Object-Sharing', '', iff={})]
exp += [call('X-Object-Public', public, public is not None)]
for k, v in metas.items():
exp += [call('X-Object-Meta-%s' % k, v)]
......@@ -674,7 +676,7 @@ class PithosRestClient(TestCase):
format, update = pm[:2]
self.assertEqual(SP.mock_calls[-2:], [
call('format', format, iff=format),
call('update', iff=update)])
call('update', '', iff=update)])
(
im, inm, clen, ctype, crng, trenc, cenc,
condis, srcobj, srcacc, srcvrs, obytes, mnfs) = terms
......@@ -699,7 +701,9 @@ class PithosRestClient(TestCase):
if pval:
perm_str += ';' if perm_str else ''
perm_str += '%s=%s' % (ptype, ','.join(pval))
exp += [call('X-Object-Sharing', perm_str)]
exp += [call('X-Object-Sharing', perm_str, iff=perms)]
else:
exp += [call('X-Object-Sharing', '', iff={})]
exp += [call('X-Object-Public', public, public is not None)]
for k, v in metas.items():
exp += [call('X-Object-Meta-%s' % k, v)]
......@@ -737,6 +741,26 @@ class PithosRestClient(TestCase):
**kwargs))
class PithosMethods(TestCase):
def test__range_up(self):
from kamaki.clients.pithos import _range_up
for args, expected in (
((0, 100, 1000, '10'), '0-10'),
((0, 100, 1000, '-10'), ''),
((900, 1000, 1000, '-10'), '990-1000'),
((150, 250, 1000, '10'), ''),
((10, 200, 1000, '130-170'), '130-170'),
((150, 200, 1000, '130-170'), '150-170'),
((100, 150, 1000, '130-170'), '130-150'),
((200, 250, 1000, '130-170'), ''),
((100, 250, 1000, '30-170,200-270'), '100-170,200-250'),
((40, 950, 1000, '-170,200-270,50',), '830-950,200-270,40-50'),
((740, 900, 1000, '-170,200-270,50',), '830-900'),
((42, 333, 800, '100,50-200,-600',), '42-100,50-200,200-333')):
self.assertEqual(_range_up(*args), expected)
class PithosClient(TestCase):
files = []
......@@ -1379,7 +1403,6 @@ class PithosClient(TestCase):
self.assertEqual(r, {})
exp_args = dict(
hashmap=True,
data_range=None,
version=None,
if_etag_match=None,
if_etag_not_match=None,
......@@ -1390,11 +1413,9 @@ class PithosClient(TestCase):
if_match='if match',
if_none_match='if non match',
if_modified_since='some date here',
if_unmodified_since='some date here',
data_range='10-20')
if_unmodified_since='some date here')
with patch.object(
pithos.PithosClient, 'object_get',
return_value=FR()) as get:
pithos.PithosClient, 'object_get', return_value=FR()) as get:
r = self.client.get_object_hashmap(obj)
self.assertEqual(r, object_hashmap)
self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
......@@ -1811,5 +1832,8 @@ if __name__ == '__main__':
if not argv[1:] or argv[1] == 'PithosRestClient':
not_found = False
runTestCase(PithosRestClient, 'PithosRest Client', argv[2:])
if not argv[1:] or argv[1] == 'PithosMethods':
not_found = False
runTestCase(PithosRestClient, 'Pithos Methods', argv[2:])
if not_found:
print('TestCase %s not found' % argv[1])
......@@ -45,7 +45,8 @@ from kamaki.clients.cyclades.test import CycladesClient
from kamaki.clients.cyclades.test import CycladesRestClient
from kamaki.clients.image.test import ImageClient
from kamaki.clients.storage.test import StorageClient
from kamaki.clients.pithos.test import PithosClient, PithosRestClient
from kamaki.clients.pithos.test import (
PithosClient, PithosRestClient, PithosMethods)
class ClientError(TestCase):
......@@ -300,10 +301,7 @@ class Client(TestCase):
self.assertEqual(self.client.base_url, self.base_url)
self.assertEqual(self.client.token, self.token)
self.assert_dicts_are_equal(self.client.headers, {})
DATE_FORMATS = [
'%a %b %d %H:%M:%S %Y',
'%A, %d-%b-%y %H:%M:%S GMT',
'%a, %d %b %Y %H:%M:%S GMT']
DATE_FORMATS = ['%a %b %d %H:%M:%S %Y']
self.assertEqual(self.client.DATE_FORMATS, DATE_FORMATS)
def test__init_thread_limit(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment