Commit c4a93182 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'feature-file-range' into develop

parents ef04bdeb 51ccbc00
......@@ -46,4 +46,5 @@ Features:
New register args: --update-image-file=/local/path, --no-progress-bar
- Implement an astakos.post_user_catalogs call for getting usernames from a
list of user uuids [#4203]
- Use multiformed ranges in file/pithos [#4059]
......@@ -131,13 +131,23 @@ class RangeArgument(ValueArgument):
return getattr(self, '_value', self.default)
@value.setter
def value(self, newvalue):
if newvalue is None:
def value(self, newvalues):
if not newvalues:
self._value = self.default
return
(start, end) = newvalue.split('-')
(start, end) = (int(start), int(end))
self._value = '%s-%s' % (start, end)
self._value = ''
for newvalue in newvalues.split(','):
self._value = ('%s,' % self._value) if self._value else ''
start, sep, end = newvalue.partition('-')
if sep:
if start:
start, end = (int(start), int(end))
assert start <= end, 'Invalid range value %s' % newvalue
self._value += '%s-%s' % (int(start), int(end))
else:
self._value += '-%s' % int(end)
else:
self._value += '%s' % int(start)
# Command specs
......
......@@ -322,7 +322,7 @@ class Client(object):
MAX_THREADS = 7
DATE_FORMATS = ['%a %b %d %H:%M:%S %Y', ]
LOG_TOKEN = False
LOG_DATA = True
LOG_DATA = False
CONNECTION_RETRY_LIMIT = 0
def __init__(self, base_url, token):
......
......@@ -85,9 +85,7 @@ class Image(livetest.Generic):
f.close()
r = self.client.register(
self.imgname,
self.location,
params=dict(is_public=True))
self.imgname, self.location, params=dict(is_public=True))
self._imglist[self.imgname] = dict(
name=r['name'], id=r['id'])
self._imgdetails[self.imgname] = r
......
......@@ -174,12 +174,10 @@ class Pithos(livetest.Generic):
for format in self.client.DATE_FORMATS:
now_formated = self.now_unformated.strftime(format)
r1 = self.client.account_head(
if_modified_since=now_formated,
success=(204, 304, 412))
if_modified_since=now_formated, success=(204, 304, 412))
sc1 = r1.status_code
r2 = self.client.account_head(
if_unmodified_since=now_formated,
success=(204, 304, 412))
if_unmodified_since=now_formated, success=(204, 304, 412))
sc2 = r2.status_code
self.assertNotEqual(sc1, sc2)
......@@ -217,12 +215,10 @@ class Pithos(livetest.Generic):
for format in self.client.DATE_FORMATS:
now_formated = self.now_unformated.strftime(format)
r1 = self.client.account_get(
if_modified_since=now_formated,
success=(200, 304, 412))
if_modified_since=now_formated, success=(200, 304, 412))
sc1 = r1.status_code
r2 = self.client.account_get(
if_unmodified_since=now_formated,
success=(200, 304, 412))
if_unmodified_since=now_formated, success=(200, 304, 412))
sc2 = r2.status_code
self.assertNotEqual(sc1, sc2)
......@@ -258,8 +254,7 @@ class Pithos(livetest.Generic):
self.client.set_account_group(grpName, [u1, u3])
r = self.client.get_account_group()
self.assertEqual(
r['x-account-group-' + grpName],
'%s,%s' % (u1, u3))
r['x-account-group-' + grpName], '%s,%s' % (u1, u3))
except:
print('\tInvalid user id %s (it is ok, though)' % u3)
self.client.del_account_group(grpName)
......@@ -268,8 +263,7 @@ class Pithos(livetest.Generic):
mprefix = 'meta' + unicode(self.now)
self.client.set_account_meta({
mprefix + '1': 'v1',
mprefix + '2': 'v2'})
mprefix + '1': 'v1', mprefix + '2': 'v2'})
r = self.client.get_account_meta()
self.assertEqual(r['x-account-meta-' + mprefix + '1'], 'v1')
self.assertEqual(r['x-account-meta-' + mprefix + '2'], 'v2')
......@@ -312,12 +306,10 @@ class Pithos(livetest.Generic):
for format in self.client.DATE_FORMATS:
now_formated = self.now_unformated.strftime(format)
r1 = self.client.container_head(
if_modified_since=now_formated,
success=(204, 304, 412))
if_modified_since=now_formated, success=(204, 304, 412))
sc1 = r1.status_code
r2 = self.client.container_head(
if_unmodified_since=now_formated,
success=(204, 304, 412))
if_unmodified_since=now_formated, success=(204, 304, 412))
sc2 = r2.status_code
self.assertNotEqual(sc1, sc2)
......@@ -377,12 +369,10 @@ class Pithos(livetest.Generic):
for format in self.client.DATE_FORMATS:
now_formated = self.now_unformated.strftime(format)
r1 = self.client.container_get(
if_modified_since=now_formated,
success=(200, 304, 412))
if_modified_since=now_formated, success=(200, 304, 412))
sc1 = r1.status_code
r2 = self.client.container_get(
if_unmodified_since=now_formated,
success=(200, 304, 412))
if_unmodified_since=now_formated, success=(200, 304, 412))
sc2 = r2.status_code
self.assertNotEqual(sc1, sc2)
......@@ -550,9 +540,7 @@ class Pithos(livetest.Generic):
self.assertTrue(len(r[0]) > 1)
self.client.purge_container()
self.assertRaises(
ClientError,
self.client.get_object_versionlist,
'test')
ClientError, self.client.get_object_versionlist, 'test')
def _test_0080_recreate_deleted_data(self):
self._init_data()
......@@ -568,38 +556,32 @@ class Pithos(livetest.Generic):
r = self.client.object_head(obj)
self.assertEqual(r.status_code, 200)
etag = r.headers['etag']
real_version = r.headers['x-object-version']
r = self.client.object_head(obj, version=40)
self.assertEqual(r.headers['x-object-version'], '40')
self.assertRaises(
ClientError, self.client.object_head, obj, version=-10)
r = self.client.object_head(obj, version=real_version)
self.assertEqual(r.headers['x-object-version'], real_version)
r = self.client.object_head(obj, if_etag_match=etag)
self.assertEqual(r.status_code, 200)
r = self.client.object_head(
obj,
if_etag_not_match=etag,
success=(200, 412, 304))
obj, if_etag_not_match=etag, success=(200, 412, 304))
self.assertNotEqual(r.status_code, 200)
r = self.client.object_head(
obj,
version=40,
if_etag_match=etag,
success=412)
self.assertEqual(r.status_code, 412)
obj, version=real_version, if_etag_match=etag, success=200)
self.assertEqual(r.status_code, 200)
"""Check and if(un)modified_since"""
for format in self.client.DATE_FORMATS:
now_formated = self.now_unformated.strftime(format)
r1 = self.client.object_head(
obj,
if_modified_since=now_formated,
success=(200, 304, 412))
obj, if_modified_since=now_formated, success=(200, 304, 412))
sc1 = r1.status_code
r2 = self.client.object_head(
obj,
if_unmodified_since=now_formated,
success=(200, 304, 412))
obj, if_unmodified_since=now_formated, success=(200, 304, 412))
sc2 = r2.status_code
self.assertNotEqual(sc1, sc2)
......@@ -626,18 +608,13 @@ class Pithos(livetest.Generic):
rangestr = 'bytes=%s-%s' % (osize / 3, osize / 2)
r = self.client.object_get(
obj,
data_range=rangestr,
success=(200, 206))
obj, data_range=rangestr, success=(200, 206))
partsize = int(r.headers['content-length'])
self.assertTrue(0 < partsize and partsize <= 1 + osize / 3)
rangestr = 'bytes=%s-%s' % (osize / 3, osize / 2)
r = self.client.object_get(
obj,
data_range=rangestr,
if_range=True,
success=(200, 206))
obj, data_range=rangestr, if_range=True, success=(200, 206))
partsize = int(r.headers['content-length'])
self.assertTrue(0 < partsize and partsize <= 1 + osize / 3)
......@@ -651,14 +628,10 @@ class Pithos(livetest.Generic):
for format in self.client.DATE_FORMATS:
now_formated = self.now_unformated.strftime(format)
r1 = self.client.object_get(
obj,
if_modified_since=now_formated,
success=(200, 304, 412))
obj, if_modified_since=now_formated, success=(200, 304, 412))
sc1 = r1.status_code
r2 = self.client.object_get(
obj,
if_unmodified_since=now_formated,
success=(200, 304, 412))
obj, if_unmodified_since=now_formated, success=(200, 304, 412))
sc2 = r2.status_code
self.assertNotEqual(sc1, sc2)
......@@ -669,8 +642,7 @@ class Pithos(livetest.Generic):
src_f = self.create_large_file(f_size)
print('\tUploading...')
r = self.client.upload_object(
trg_fname, src_f,
container_info_cache=container_info_cache)
trg_fname, src_f, container_info_cache=container_info_cache)
print('\tDownloading...')
self.files.append(NamedTemporaryFile())
dnl_f = self.files[-1]
......@@ -686,8 +658,7 @@ class Pithos(livetest.Generic):
for pos in (0, f_size / 2, f_size - 256):
src_f.seek(pos)
tmp_s = self.client.download_to_string(
trg_fname,
range_str='%s-%s' % (pos, (pos + 128)))
trg_fname, range_str='%s-%s' % (pos, (pos + 128)))
self.assertEqual(tmp_s, src_f.read(len(tmp_s)))
print('\tUploading KiBs as strings...')
trg_fname = 'fromString_%s' % self.now
......@@ -704,8 +675,7 @@ class Pithos(livetest.Generic):
src_f = self.create_boring_file(42)
print('\tUploading boring file...')
self.client.upload_object(
trg_fname, src_f,
container_info_cache=container_info_cache)
trg_fname, src_f, container_info_cache=container_info_cache)
print('\tDownloading boring file...')
self.files.append(NamedTemporaryFile())
dnl_f = self.files[-1]
......@@ -783,9 +753,7 @@ class Pithos(livetest.Generic):
"""Check content_type and content_length"""
tmpdir = 'dir' + unicode(self.now)
r = self.client.object_put(
tmpdir,
content_type='application/directory',
content_length=0)
tmpdir, content_type='application/directory', content_length=0)
r = self.client.get_object_info(tmpdir)
self.assertEqual(r['content-type'], 'application/directory')
......@@ -1162,9 +1130,7 @@ class Pithos(livetest.Generic):
"""Check permissions"""
self.client.set_object_sharing(
obj,
read_permission=['u4', 'u5'],
write_permission=['u4'])
obj, read_permission=['u4', 'u5'], write_permission=['u4'])
r = self.client.get_object_sharing(obj)
self.assertTrue('read' in r)
self.assertTrue('u5' in r['read'])
......@@ -1249,12 +1215,10 @@ class Pithos(livetest.Generic):
content_type='application/octet-stream')
self.client.create_object_by_manifestation(
mobj,
content_type='application/octet-stream')
mobj, content_type='application/octet-stream')
r = self.client.object_post(
mobj,
manifest='%s/%s' % (self.client.container, mobj))
mobj, manifest='%s/%s' % (self.client.container, mobj))
r = self.client.object_get(mobj)
self.assertEqual(r.text, txt)
......@@ -1293,6 +1257,7 @@ class Pithos(livetest.Generic):
def create_large_file(self, size):
"""Create a large file at fs"""
print
self.files.append(NamedTemporaryFile())
f = self.files[-1]
Ki = size / 8
......
......@@ -52,17 +52,49 @@ def _pithos_hash(block, blockhash):
return h.hexdigest()
def _range_up(start, end, a_range):
if a_range:
(rstart, rend) = a_range.split('-')
(rstart, rend) = (int(rstart), int(rend))
if rstart > end or rend < start:
return (0, 0)
if rstart > start:
start = rstart
if rend < end:
end = rend
return (start, end)
def _range_up(start, end, max_value, a_range):
"""
:param start: (int) the window bottom
:param end: (int) the window top
:param max_value: (int) maximum accepted value
:param a_range: (str) a range string in the form X[,X'[,X''[...]]]
where X: x|x-y|-x where x < y and x, y natural numbers
:returns: (str) a range string cut-off for the start-end range
an empty response means this window is out of range
"""
assert start >= 0, '_range_up was called with start < 0'
assert end >= start, '_range_up was called with end < start'
assert end <= max_value, '_range_up was called with max_value < end'
if not a_range:
return '%s-%s' % (start, end)
selected = []
for some_range in a_range.split(','):
v0, sep, v1 = some_range.partition('-')
if v0:
v0 = int(v0)
if sep:
v1 = int(v1)
if v1 < start or v0 > end or v1 < v0:
continue
v0 = v0 if v0 > start else start
v1 = v1 if v1 < end else end
selected.append('%s-%s' % (v0, v1))
elif v0 < start:
continue
else:
v1 = v0 if v0 <= end else end
selected.append('%s-%s' % (start, v1))
else:
v1 = int(v1)
if max_value - v1 > end:
continue
v0 = (max_value - v1) if max_value - v1 > start else start
selected.append('%s-%s' % (v0, end))
return ','.join(selected)
class PithosClient(PithosRestClient):
......@@ -626,15 +658,18 @@ class PithosClient(PithosRestClient):
return (blocksize, blockhash, total_size, hashmap['hashes'], map_dict)
def _dump_blocks_sync(
self, obj, remote_hashes, blocksize, total_size, dst, range,
self, obj, remote_hashes, blocksize, total_size, dst, crange,
**args):
for blockid, blockhash in enumerate(remote_hashes):
if blockhash:
start = blocksize * blockid
is_last = start + blocksize > total_size
end = (total_size - 1) if is_last else (start + blocksize - 1)
(start, end) = _range_up(start, end, range)
args['data_range'] = 'bytes=%s-%s' % (start, end)
data_range = _range_up(start, end, total_size, crange)
if not data_range:
self._cb_next()
continue
args['data_range'] = 'bytes=%s' % data_range
r = self.object_get(obj, success=(200, 206), **args)
self._cb_next()
dst.write(r.content)
......@@ -680,9 +715,6 @@ class PithosClient(PithosRestClient):
flying = dict()
blockid_dict = dict()
offset = 0
if filerange is not None:
rstart = int(filerange.split('-')[0])
offset = rstart if blocksize > rstart else rstart % blocksize
self._init_thread_limit()
for block_hash, blockids in remote_hashes.items():
......@@ -699,12 +731,11 @@ class PithosClient(PithosRestClient):
**restargs)
end = total_size - 1 if (
key + blocksize > total_size) else key + blocksize - 1
start, end = _range_up(key, end, filerange)
if start == end:
data_range = _range_up(key, end, total_size, filerange)
if not data_range:
self._cb_next()
continue
restargs['async_headers'] = {
'Range': 'bytes=%s-%s' % (start, end)}
restargs['async_headers'] = {'Range': 'bytes=%s' % data_range}
flying[key] = self._get_block_async(obj, **restargs)
blockid_dict[key] = unsaved
......@@ -847,9 +878,10 @@ class PithosClient(PithosRestClient):
start = blocksize * blockid
is_last = start + blocksize > total_size
end = (total_size - 1) if is_last else (start + blocksize - 1)
(start, end) = _range_up(start, end, range_str)
if start < end:
data_range_str = _range_up(start, end, end, range_str)
if data_range_str:
self._watch_thread_limit(flying.values())
restargs['data_range'] = 'bytes=%s' % data_range_str
flying[blockid] = self._get_block_async(obj, **restargs)
for runid, thread in flying.items():
if (blockid + 1) == num_of_blocks:
......@@ -889,8 +921,7 @@ class PithosClient(PithosRestClient):
if_match=None,
if_none_match=None,
if_modified_since=None,
if_unmodified_since=None,
data_range=None):
if_unmodified_since=None):
"""
:param obj: (str) remote object path
......@@ -902,9 +933,6 @@ class PithosClient(PithosRestClient):
:param if_unmodified_since: (str) formated date
:param data_range: (str) from-to where from and to are integers
denoting file positions in bytes
:returns: (list)
"""
try:
......@@ -915,8 +943,7 @@ class PithosClient(PithosRestClient):
if_etag_match=if_match,
if_etag_not_match=if_none_match,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since,
data_range=data_range)
if_unmodified_since=if_unmodified_since)
except ClientError as err:
if err.status == 304 or err.status == 412:
return {}
......
......@@ -242,7 +242,7 @@ class PithosRestClient(TestCase):
pm = pm[:-2]
self.client.account_post(*(pm + args), **kwargs)
upd = pm[0]
self.assertEqual(SP.mock_calls[-1], call('update', iff=upd))
self.assertEqual(SP.mock_calls[-1], call('update', '', iff=upd))
expected = []
if pm[1]:
expected += [
......@@ -369,7 +369,7 @@ class PithosRestClient(TestCase):
self.client.container_post(*(pm + args), **kwargs)
upd, frmt = pm[:2]
self.assertEqual(SP.mock_calls[-2:], [
call('update', iff=upd),
call('update', '', iff=upd),
call('format', frmt, iff=frmt)])
qta, vrs, metas, ctype, clen, trenc = pm[2:]
prfx = 'X-Container-Meta-'
......@@ -637,7 +637,9 @@ class PithosRestClient(TestCase):
if pval:
perm_str += ';' if perm_str else ''
perm_str += '%s=%s' % (ptype, ','.join(pval))
exp += [call('X-Object-Sharing', perm_str)]
exp += [call('X-Object-Sharing', perm_str, iff=perms)]
else:
exp += [call('X-Object-Sharing', '', iff={})]
exp += [call('X-Object-Public', public, public is not None)]
for k, v in metas.items():
exp += [call('X-Object-Meta-%s' % k, v)]
......@@ -674,7 +676,7 @@ class PithosRestClient(TestCase):
format, update = pm[:2]
self.assertEqual(SP.mock_calls[-2:], [
call('format', format, iff=format),
call('update', iff=update)])
call('update', '', iff=update)])
(
im, inm, clen, ctype, crng, trenc, cenc,
condis, srcobj, srcacc, srcvrs, obytes, mnfs) = terms
......@@ -699,7 +701,9 @@ class PithosRestClient(TestCase):
if pval:
perm_str += ';' if perm_str else ''
perm_str += '%s=%s' % (ptype, ','.join(pval))
exp += [call('X-Object-Sharing', perm_str)]
exp += [call('X-Object-Sharing', perm_str, iff=perms)]
else:
exp += [call('X-Object-Sharing', '', iff={})]
exp += [call('X-Object-Public', public, public is not None)]
for k, v in metas.items():
exp += [call('X-Object-Meta-%s' % k, v)]
......@@ -737,6 +741,26 @@ class PithosRestClient(TestCase):
**kwargs))
class PithosMethods(TestCase):
def test__range_up(self):
from kamaki.clients.pithos import _range_up
for args, expected in (
((0, 100, 1000, '10'), '0-10'),
((0, 100, 1000, '-10'), ''),
((900, 1000, 1000, '-10'), '990-1000'),
((150, 250, 1000, '10'), ''),
((10, 200, 1000, '130-170'), '130-170'),
((150, 200, 1000, '130-170'), '150-170'),
((100, 150, 1000, '130-170'), '130-150'),
((200, 250, 1000, '130-170'), ''),
((100, 250, 1000, '30-170,200-270'), '100-170,200-250'),
((40, 950, 1000, '-170,200-270,50',), '830-950,200-270,40-50'),
((740, 900, 1000, '-170,200-270,50',), '830-900'),
((42, 333, 800, '100,50-200,-600',), '42-100,50-200,200-333')):
self.assertEqual(_range_up(*args), expected)
class PithosClient(TestCase):
files = []
......@@ -1379,7 +1403,6 @@ class PithosClient(TestCase):
self.assertEqual(r, {})
exp_args = dict(
hashmap=True,
data_range=None,
version=None,
if_etag_match=None,
if_etag_not_match=None,
......@@ -1390,11 +1413,9 @@ class PithosClient(TestCase):
if_match='if match',
if_none_match='if non match',
if_modified_since='some date here',
if_unmodified_since='some date here',
data_range='10-20')
if_unmodified_since='some date here')
with patch.object(
pithos.PithosClient, 'object_get',
return_value=FR()) as get:
pithos.PithosClient, 'object_get', return_value=FR()) as get:
r = self.client.get_object_hashmap(obj)
self.assertEqual(r, object_hashmap)
self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
......@@ -1811,5 +1832,8 @@ if __name__ == '__main__':
if not argv[1:] or argv[1] == 'PithosRestClient':
not_found = False
runTestCase(PithosRestClient, 'PithosRest Client', argv[2:])
if not argv[1:] or argv[1] == 'PithosMethods':
not_found = False
runTestCase(PithosRestClient, 'Pithos Methods', argv[2:])
if not_found:
print('TestCase %s not found' % argv[1])
......@@ -45,7 +45,8 @@ from kamaki.clients.cyclades.test import CycladesClient
from kamaki.clients.cyclades.test import CycladesRestClient
from kamaki.clients.image.test import ImageClient
from kamaki.clients.storage.test import StorageClient
from kamaki.clients.pithos.test import PithosClient, PithosRestClient
from kamaki.clients.pithos.test import (
PithosClient, PithosRestClient, PithosMethods)
class ClientError(TestCase):
......@@ -300,10 +301,7 @@ class Client(TestCase):
self.assertEqual(self.client.base_url, self.base_url)
self.assertEqual(self.client.token, self.token)
self.assert_dicts_are_equal(self.client.headers, {})
DATE_FORMATS = [
'%a %b %d %H:%M:%S %Y',
'%A, %d-%b-%y %H:%M:%S GMT',
'%a, %d %b %Y %H:%M:%S GMT']
DATE_FORMATS = ['%a %b %d %H:%M:%S %Y']
self.assertEqual(self.client.DATE_FORMATS, DATE_FORMATS)
def test__init_thread_limit(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment