Commit 49cc29b2 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Implement download_to_string in pithos client

The download_to_string method downloads a remote object from pithos into a
string, which is then returned.

Also, implemented unit and functional tests. Changelog is updated

Refs: #3608
parent 653e6193
......@@ -32,4 +32,5 @@ Features:
- Add a -l option to upload, for listing uploaded objects details [#3730]
- Add option to cache container info for upload_object [#3707]
- Add enumeration to all listing commands, make it optional [#3739]
- Add a download_to_string method in pithos client [#3608]
......@@ -677,10 +677,18 @@ class Pithos(livetest.Generic):
self.client.download_object(trg_fname, dnl_f)
print('\tCheck if files match...')
for pos in (0, f_size / 2, f_size - 20):
for pos in (0, f_size / 2, f_size - 128):
src_f.seek(pos)
dnl_f.seek(pos)
self.assertEqual(src_f.read(10), dnl_f.read(10))
self.assertEqual(src_f.read(64), dnl_f.read(64))
print('\tDownload KiBs to string and check again...')
for pos in (0, f_size / 2, f_size - 256):
src_f.seek(pos)
tmp_s = self.client.download_to_string(
trg_fname,
range_str='%s-%s' % (pos, (pos + 128)))
self.assertEqual(tmp_s, src_f.read(len(tmp_s)))
"""Upload a boring file"""
trg_fname = 'boringfile_%s' % self.now
......
......@@ -36,6 +36,7 @@ from threading import enumerate as activethreads
from os import fstat
from hashlib import new as newhashlib
from time import time
from StringIO import StringIO
from binascii import hexlify
......@@ -43,7 +44,6 @@ from kamaki.clients import SilentEvent, sendlog
from kamaki.clients.pithos.rest_api import PithosRestClient
from kamaki.clients.storage import ClientError
from kamaki.clients.utils import path4url, filter_in
from StringIO import StringIO
def _pithos_hash(block, blockhash):
......@@ -623,6 +623,71 @@ class PithosClient(PithosRestClient):
self._complete_cb()
def download_to_string(
self, obj,
download_cb=None,
version=None,
range_str=None,
if_match=None,
if_none_match=None,
if_modified_since=None,
if_unmodified_since=None):
"""Download an object to a string (multiple connections)
:param obj: (str) remote object path
:param download_cb: optional progress.bar object for downloading
:param version: (str) file version
:param range_str: (str) from, to are file positions (int) in bytes
:param if_match: (str)
:param if_none_match: (str)
:param if_modified_since: (str) formated date
:param if_unmodified_since: (str) formated date
:returns: (str) the whole object contents
"""
restargs = dict(
version=version,
data_range=None if range_str is None else 'bytes=%s' % range_str,
if_match=if_match,
if_none_match=if_none_match,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since)
(
blocksize,
blockhash,
total_size,
hash_list,
remote_hashes) = self._get_remote_blocks_info(obj, **restargs)
assert total_size >= 0
if download_cb:
self.progress_bar_gen = download_cb(len(hash_list))
self._cb_next()
ret = ''
for blockid, blockhash in enumerate(remote_hashes):
start = blocksize * blockid
is_last = start + blocksize > total_size
end = (total_size - 1) if is_last else (start + blocksize - 1)
(start, end) = _range_up(start, end, range_str)
if start == end:
continue
restargs['data_range'] = 'bytes=%s-%s' % (start, end)
r = self.object_get(obj, success=(200, 206), **restargs)
ret += r.content
self._cb_next()
self._complete_cb()
return ret
#Command Progress Bar method
def _cb_next(self, step=1):
if hasattr(self, 'progress_bar_gen'):
......
......@@ -1101,6 +1101,37 @@ class PithosClient(TestCase):
expected['permissions'] = expected.pop('sharing')
self.assertEqual(put.mock_calls[-1], call(obj, **expected))
@patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
@patch('%s.object_get' % pithos_pkg, return_value=FR())
def test_download_to_string(self, GET, GOH):
FR.content = 'some sample content'
num_of_blocks = len(object_hashmap['hashes'])
r = self.client.download_to_string(obj)
expected_content = FR.content * num_of_blocks
self.assertEqual(expected_content, r)
self.assertEqual(len(GET.mock_calls), num_of_blocks)
self.assertEqual(GET.mock_calls[-1][1], (obj,))
kwargs = dict(
version='version',
range_str='10-20',
if_match='if and only if',
if_none_match='if and only not',
if_modified_since='what if not?',
if_unmodified_since='this happens if not!')
expargs = dict(kwargs)
expargs.pop('range_str')
for k in expargs:
expargs[k] = None
GOH.assert_called_once_with(obj, **expargs)
r = self.client.download_to_string(obj, **kwargs)
expargs['data_range'] = 'bytes=%s' % kwargs['range_str']
for k, v in expargs.items():
self.assertEqual(
GET.mock_calls[-1][2][k],
v or kwargs.get(k))
@patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
@patch('%s.object_get' % pithos_pkg, return_value=FR())
def test_download_object(self, GET, GOH):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment