Use multiformed ranges in kamaki pithos lib

Refs: #4059

Also, adjust corresponding unit and functional tests

kamaki/cli/commands/pithos.py

@@ -131,18 +131,23 @@ class RangeArgument(ValueArgument):
         return getattr(self, '_value', self.default)
 
     @value.setter
-    def value(self, newvalue):
-        if newvalue is None:
+    def value(self, newvalues):
+        if not newvalues:
             self._value = self.default
             return
-        start, sep, end = newvalue.partition('-')
-        if sep:
-            if start:
-                self._value = '%s-%s' % (int(start), int(end))
+        self._value = ''
+        for newvalue in newvalues.split(','):
+            self._value = ('%s,' % self._value) if self._value else ''
+            start, sep, end = newvalue.partition('-')
+            if sep:
+                if start:
+                    start, end = (int(start), int(end))
+                    assert start <= end, 'Invalid range value %s' % newvalue
+                    self._value += '%s-%s' % (int(start), int(end))
+                else:
+                    self._value += '-%s' % int(end)
             else:
-                self._value = '-%s' % int(end)
-        else:
-            self._value = '%s' % int(start)
+                self._value += '%s' % int(start)
 
 
 # Command specs

kamaki/clients/__init__.py

@@ -322,7 +322,7 @@ class Client(object):
     MAX_THREADS = 7
     DATE_FORMATS = ['%a %b %d %H:%M:%S %Y', ]
     LOG_TOKEN = False
-    LOG_DATA = True
+    LOG_DATA = False
     CONNECTION_RETRY_LIMIT = 0
 
     def __init__(self, base_url, token):

kamaki/clients/livetest/pithos.py

@@ -556,9 +556,12 @@ class Pithos(livetest.Generic):
         r = self.client.object_head(obj)
         self.assertEqual(r.status_code, 200)
         etag = r.headers['etag']
+        real_version = r.headers['x-object-version']
 
-        r = self.client.object_head(obj, version=40)
-        self.assertEqual(r.headers['x-object-version'], '40')
+        self.assertRaises(
+            ClientError, self.client.object_head, obj, version=-10)
+        r = self.client.object_head(obj, version=real_version)
+        self.assertEqual(r.headers['x-object-version'], real_version)
 
         r = self.client.object_head(obj, if_etag_match=etag)
         self.assertEqual(r.status_code, 200)
@@ -568,8 +571,8 @@ class Pithos(livetest.Generic):
         self.assertNotEqual(r.status_code, 200)
 
         r = self.client.object_head(
-            obj, version=40, if_etag_match=etag, success=412)
-        self.assertEqual(r.status_code, 412)
+            obj, version=real_version, if_etag_match=etag, success=200)
+        self.assertEqual(r.status_code, 200)
 
         """Check and if(un)modified_since"""
         for format in self.client.DATE_FORMATS:
@@ -1254,6 +1257,7 @@ class Pithos(livetest.Generic):
 
     def create_large_file(self, size):
         """Create a large file at fs"""
+        print
         self.files.append(NamedTemporaryFile())
         f = self.files[-1]
         Ki = size / 8

kamaki/clients/pithos/__init__.py

@@ -52,25 +52,49 @@ def _pithos_hash(block, blockhash):
     return h.hexdigest()
 
 
-def _range_up(start, end, a_range):
-    if a_range:
-        rstart, sep, rend = a_range.partition('-')
-        if rstart:
+def _range_up(start, end, max_value, a_range):
+    """
+    :param start: (int) the window bottom
+
+    :param end: (int) the window top
+
+    :param max_value: (int) maximum accepted value
+
+    :param a_range: (str) a range string in the form X[,X'[,X''[...]]]
+        where X: x|x-y|-x where x < y and x, y natural numbers
+
+    :returns: (str) a range string cut-off for the start-end range
+        an empty response means this window is out of range
+    """
+    assert start >= 0, '_range_up was called with start < 0'
+    assert end >= start, '_range_up was called with end < start'
+    assert end <= max_value, '_range_up was called with max_value < end'
+    if not a_range:
+        return '%s-%s' % (start, end)
+    selected = []
+    for some_range in a_range.split(','):
+        v0, sep, v1 = some_range.partition('-')
+        if v0:
+            v0 = int(v0)
             if sep:
-                rstart, rend = int(rstart), int(rend)
+                v1 = int(v1)
+                if v1 < start or v0 > end or v1 < v0:
+                    continue
+                v0 = v0 if v0 > start else start
+                v1 = v1 if v1 < end else end
+                selected.append('%s-%s' % (v0, v1))
+            elif v0 < start:
+                continue
             else:
-                rstart, rend = 0, int(rstart)
-        elif sep:
-            return (0, - int(rend))
+                v1 = v0 if v0 <= end else end
+                selected.append('%s-%s' % (start, v1))
         else:
-            return (0, 0)
-        if rstart > end or rend < start:
-            return (0, 0)
-        if rstart > start:
-            start = rstart
-        if rend < end:
-            end = rend
-    return (start, end)
+            v1 = int(v1)
+            if max_value - v1 > end:
+                continue
+            v0 = (max_value - v1) if max_value - v1 > start else start
+            selected.append('%s-%s' % (v0, end))
+    return ','.join(selected)
 
 
 class PithosClient(PithosRestClient):
@@ -641,9 +665,11 @@ class PithosClient(PithosRestClient):
                 start = blocksize * blockid
                 is_last = start + blocksize > total_size
                 end = (total_size - 1) if is_last else (start + blocksize - 1)
-                (start, end) = _range_up(start, end, crange)
-                args['data_range'] = 'bytes=%s-%s' % (
-                    (start, end) if end >= 0 else ('', - end))
+                data_range = _range_up(start, end, total_size, crange)
+                if not data_range:
+                    self._cb_next()
+                    continue
+                args['data_range'] = 'bytes=%s' % data_range
                 r = self.object_get(obj, success=(200, 206), **args)
                 self._cb_next()
                 dst.write(r.content)
@@ -689,9 +715,6 @@ class PithosClient(PithosRestClient):
         flying = dict()
         blockid_dict = dict()
         offset = 0
-        if filerange is not None:
-            rstart = int(filerange.split('-')[0])
-            offset = rstart if blocksize > rstart else rstart % blocksize
 
         self._init_thread_limit()
         for block_hash, blockids in remote_hashes.items():
@@ -708,13 +731,11 @@ class PithosClient(PithosRestClient):
                     **restargs)
                 end = total_size - 1 if (
                     key + blocksize > total_size) else key + blocksize - 1
-                start, end = _range_up(key, end, filerange)
-                if start == end:
+                data_range = _range_up(key, end, total_size, filerange)
+                if not data_range:
                     self._cb_next()
                     continue
-                restargs['async_headers'] = {
-                    'Range': 'bytes=%s-%s' % (
-                        (start, end) if end >= 0 else ('', - end))}
+                restargs['async_headers'] = {'Range': 'bytes=%s' % data_range}
                 flying[key] = self._get_block_async(obj, **restargs)
                 blockid_dict[key] = unsaved
 
@@ -857,9 +878,10 @@ class PithosClient(PithosRestClient):
                 start = blocksize * blockid
                 is_last = start + blocksize > total_size
                 end = (total_size - 1) if is_last else (start + blocksize - 1)
-                (start, end) = _range_up(start, end, range_str)
-                if start < end or end < 0:
+                data_range_str = _range_up(start, end, end, range_str)
+                if data_range_str:
                     self._watch_thread_limit(flying.values())
+                    restargs['data_range'] = 'bytes=%s' % data_range_str
                     flying[blockid] = self._get_block_async(obj, **restargs)
                 for runid, thread in flying.items():
                     if (blockid + 1) == num_of_blocks:
@@ -899,8 +921,7 @@ class PithosClient(PithosRestClient):
             if_match=None,
             if_none_match=None,
             if_modified_since=None,
-            if_unmodified_since=None,
-            data_range=None):
+            if_unmodified_since=None):
         """
         :param obj: (str) remote object path
 
@@ -912,9 +933,6 @@ class PithosClient(PithosRestClient):
 
         :param if_unmodified_since: (str) formated date
 
-        :param data_range: (str) from-to where from and to are integers
-            denoting file positions in bytes
-
         :returns: (list)
         """
         try:
@@ -925,8 +943,7 @@ class PithosClient(PithosRestClient):
                 if_etag_match=if_match,
                 if_etag_not_match=if_none_match,
                 if_modified_since=if_modified_since,
-                if_unmodified_since=if_unmodified_since,
-                data_range=data_range)
+                if_unmodified_since=if_unmodified_since)
         except ClientError as err:
             if err.status == 304 or err.status == 412:
                 return {}

kamaki/clients/pithos/test.py

@@ -242,7 +242,7 @@ class PithosRestClient(TestCase):
             pm = pm[:-2]
             self.client.account_post(*(pm + args), **kwargs)
             upd = pm[0]
-            self.assertEqual(SP.mock_calls[-1], call('update', iff=upd))
+            self.assertEqual(SP.mock_calls[-1], call('update', '', iff=upd))
             expected = []
             if pm[1]:
                 expected += [
@@ -369,7 +369,7 @@ class PithosRestClient(TestCase):
             self.client.container_post(*(pm + args), **kwargs)
             upd, frmt = pm[:2]
             self.assertEqual(SP.mock_calls[-2:], [
-                call('update', iff=upd),
+                call('update', '', iff=upd),
                 call('format', frmt, iff=frmt)])
             qta, vrs, metas, ctype, clen, trenc = pm[2:]
             prfx = 'X-Container-Meta-'
@@ -637,7 +637,9 @@ class PithosRestClient(TestCase):
                     if pval:
                         perm_str += ';' if perm_str else ''
                         perm_str += '%s=%s' % (ptype, ','.join(pval))
-                exp += [call('X-Object-Sharing', perm_str)]
+                exp += [call('X-Object-Sharing', perm_str, iff=perms)]
+            else:
+                exp += [call('X-Object-Sharing', '', iff={})]
             exp += [call('X-Object-Public', public, public is not None)]
             for k, v in metas.items():
                 exp += [call('X-Object-Meta-%s' % k, v)]
@@ -674,7 +676,7 @@ class PithosRestClient(TestCase):
             format, update = pm[:2]
             self.assertEqual(SP.mock_calls[-2:], [
                 call('format', format, iff=format),
-                call('update', iff=update)])
+                call('update', '', iff=update)])
             (
                 im, inm, clen, ctype, crng, trenc, cenc,
                 condis, srcobj, srcacc, srcvrs, obytes, mnfs) = terms
@@ -699,7 +701,9 @@ class PithosRestClient(TestCase):
                     if pval:
                         perm_str += ';' if perm_str else ''
                         perm_str += '%s=%s' % (ptype, ','.join(pval))
-                exp += [call('X-Object-Sharing', perm_str)]
+                exp += [call('X-Object-Sharing', perm_str, iff=perms)]
+            else:
+                exp += [call('X-Object-Sharing', '', iff={})]
             exp += [call('X-Object-Public', public, public is not None)]
             for k, v in metas.items():
                 exp += [call('X-Object-Meta-%s' % k, v)]
@@ -737,6 +741,26 @@ class PithosRestClient(TestCase):
                 **kwargs))
 
 
+class PithosMethods(TestCase):
+
+    def test__range_up(self):
+        from kamaki.clients.pithos import _range_up
+        for args, expected in (
+                ((0, 100, 1000, '10'), '0-10'),
+                ((0, 100, 1000, '-10'), ''),
+                ((900, 1000, 1000, '-10'), '990-1000'),
+                ((150, 250, 1000, '10'), ''),
+                ((10, 200, 1000, '130-170'), '130-170'),
+                ((150, 200, 1000, '130-170'), '150-170'),
+                ((100, 150, 1000, '130-170'), '130-150'),
+                ((200, 250, 1000, '130-170'), ''),
+                ((100, 250, 1000, '30-170,200-270'), '100-170,200-250'),
+                ((40, 950, 1000, '-170,200-270,50',), '830-950,200-270,40-50'),
+                ((740, 900, 1000, '-170,200-270,50',), '830-900'),
+                ((42, 333, 800, '100,50-200,-600',), '42-100,50-200,200-333')):
+            self.assertEqual(_range_up(*args), expected)
+
+
 class PithosClient(TestCase):
 
     files = []
@@ -1379,7 +1403,6 @@ class PithosClient(TestCase):
                 self.assertEqual(r, {})
         exp_args = dict(
             hashmap=True,
-            data_range=None,
             version=None,
             if_etag_match=None,
             if_etag_not_match=None,
@@ -1390,11 +1413,9 @@ class PithosClient(TestCase):
             if_match='if match',
             if_none_match='if non match',
             if_modified_since='some date here',
-            if_unmodified_since='some date here',
-            data_range='10-20')
+            if_unmodified_since='some date here')
         with patch.object(
-                pithos.PithosClient, 'object_get',
-                return_value=FR()) as get:
+                pithos.PithosClient, 'object_get', return_value=FR()) as get:
             r = self.client.get_object_hashmap(obj)
             self.assertEqual(r, object_hashmap)
             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
@@ -1811,5 +1832,8 @@ if __name__ == '__main__':
     if not argv[1:] or argv[1] == 'PithosRestClient':
         not_found = False
         runTestCase(PithosRestClient, 'PithosRest Client', argv[2:])
+    if not argv[1:] or argv[1] == 'PithosMethods':
+        not_found = False
+        runTestCase(PithosRestClient, 'Pithos Methods', argv[2:])
     if not_found:
         print('TestCase %s not found' % argv[1])

kamaki/clients/test.py

@@ -45,7 +45,8 @@ from kamaki.clients.cyclades.test import CycladesClient
 from kamaki.clients.cyclades.test import CycladesRestClient
 from kamaki.clients.image.test import ImageClient
 from kamaki.clients.storage.test import StorageClient
-from kamaki.clients.pithos.test import PithosClient, PithosRestClient
+from kamaki.clients.pithos.test import (
+    PithosClient, PithosRestClient, PithosMethods)
 
 
 class ClientError(TestCase):
@@ -300,10 +301,7 @@ class Client(TestCase):
         self.assertEqual(self.client.base_url, self.base_url)
         self.assertEqual(self.client.token, self.token)
         self.assert_dicts_are_equal(self.client.headers, {})
-        DATE_FORMATS = [
-            '%a %b %d %H:%M:%S %Y',
-            '%A, %d-%b-%y %H:%M:%S GMT',
-            '%a, %d %b %Y %H:%M:%S GMT']
+        DATE_FORMATS = ['%a %b %d %H:%M:%S %Y']
         self.assertEqual(self.client.DATE_FORMATS, DATE_FORMATS)
 
     def test__init_thread_limit(self):