Minor bugfixes

kamaki/cli/commands/pithos_cli.py

@@ -586,6 +586,7 @@ class store_download(_store_container_command):
         super(self.__class__, self).main(container___path, path_is_optional=False)
 
         #setup output stream
+        parallel = False
         if local_path is None:
             out = stdout
         else:
@@ -600,8 +601,9 @@ class store_download(_store_container_command):
         download_cb = None if getattr(self.args, 'no_progress_bar') \
             else self.progress('Downloading')
 
+
         try:
-            self.client.download_object(self.path, out, download_cb, 
+            self.client.download_object(self.path, out, download_cb,
                 range=getattr(self.args, 'range'), version=getattr(self.args,'object_version'),
                 if_match=getattr(self.args, 'if_match'), overide=getattr(self.args, 'overide'),
                 if_none_match=getattr(self.args, 'if_none_match'),

kamaki/clients/__init__.py

@@ -84,6 +84,7 @@ class Client(object):
             #r = self._request(method, path, **kwargs)
             success = kwargs.pop('success', 200)
 
+            binary = kwargs.pop('binary', False)
             data = kwargs.pop('data', None)
             self.set_default_header('X-Auth-Token', self.token)
             #self.set_default_header('Accept', '*/*')
@@ -97,7 +98,7 @@ class Client(object):
 
             #kwargs.setdefault('verify', False)  # Disable certificate verification
             self.http_client.url = self.base_url + path
-            r = self.http_client.perform_request(method=method, data=data)
+            r = self.http_client.perform_request(method=method, data=data, binary=binary)
             #r = requests.request(method, url, headers=self.headers, data=data, **kwargs)
 
             req = self.http_client

kamaki/clients/connection/request.py

@@ -32,7 +32,6 @@
 # or implied, of GRNET S.A.
 
 import requests
-from copy import deepcopy
 from . import HTTPConnection, HTTPResponse, HTTPConnectionError, HTTPResponsePool
 from .pool import ObjectPool
 from urlparse import urlparse
@@ -44,22 +43,20 @@ requests.Response.status = property(_status)
 
 class HTTPRequestsResponse(HTTPResponse):
 
+	_get_content_only=False
+
 	def _get_response(self):
 		if self.prefetched:
 			return
 		r = self.request.response
 		try:
-			self.headers = deepcopy(r.headers)
-			self.text = deepcopy(r.text) \
-			if hasattr(r, 'text') else None
-			self.json = deepcopy(r.json) \
-			if hasattr(r, 'json') else None
-			self.content = deepcopy(r.content) \
-			if hasattr(r, 'content') else None
-			self.exception = deepcopy(r.exception) \
-			if hasattr(r, 'exception') else None
-			self.status = deepcopy(r.status)
-			self.status_code = deepcopy(r.status_code)
+			self.headers = r.headers
+			self.status = r.status
+			self.status_code = r.status_code
+			self.content = r.content if hasattr(r, 'content') else None
+			self.text = None if self._get_content_only else r.text
+			self.json = None if self._get_content_only else r.json
+			self.exception = r.exception if hasattr(r, 'exception') else None
 		except requests.ConnectionError as err:
 			raise HTTPConnectionError('Connection error', status=651, details=err.message)
 		except requests.HTTPError as err:
@@ -70,6 +67,7 @@ class HTTPRequestsResponse(HTTPResponse):
 			raise HTTPConnectionError('Invalid URL', status=404, details=err.message)
 		except requests.RequestException as err:
 			raise HTTPConnectionError('HTTP Request error', status=700, details=err.message)
+		self.prefetched=True
 
 	def release(self):
 		"""requests object handles this automatically"""
@@ -104,7 +102,8 @@ class HTTPRequest(HTTPConnection):
 			respool = self._pools[pool_key]
 		return respool.pool_get()
 
-	def perform_request(self, method=None, url=None, params=None, headers=None, data=None):
+	def perform_request(self, method=None, url=None, params=None, headers=None, data=None,
+		binary=False):
 		"""perform a request
 		Example: method='PUT' url='https://my.server:8080/path/to/service'
 			params={'update':None, 'format':'json'} headers={'X-Auth-Token':'s0m3t0k3n=='}
@@ -131,4 +130,6 @@ class HTTPRequest(HTTPConnection):
 		self._response_object = requests.request(self.method, self.url, headers=self.headers, data=data,
 			verify=self.verify, prefetch = False)
 		res.request = self._response_object.request
+		if binary:
+			res._get_content_only = True
 		return res

kamaki/clients/pithos.py

@@ -302,7 +302,7 @@ class PithosClient(StorageClient):
         success = kwargs.pop('success', 200)
         return self.head(path, *args, success=success, **kwargs)
 
-    def object_get(self, object, format='json', hashmap=False, version=None,
+    def object_get(self, object, format='json', hashmap=False, version=None, binary=False,
         data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None,
         if_modified_since = None, if_unmodified_since = None, *args, **kwargs):
         """ Full Pithos+ GET at object level
@@ -335,6 +335,8 @@ class PithosClient(StorageClient):
 
         path=path4url(self.account, self.container, object)
         success = kwargs.pop('success', 200)
+        if binary:
+            kwargs['binary'] = True
         return self.get(path, *args, success=success, **kwargs)
 
     def object_put(self, object, format='json', hashmap=False, delimiter = None, if_etag_match=None,
@@ -730,13 +732,15 @@ class PithosClient(StorageClient):
         self.object_put(object, format='json', hashmap=True, content_type=obj_content_type, 
             json=hashmap, success=201)
 
-    def download_object(self, obj, f, download_cb=None, version=None, overide=False, range=None,
-        if_match=None, if_none_match=None, if_modified_since=None, if_unmodified_since=None):
+    def download_object(self, obj, f, download_cb=None, version=None, overide=False,
+        range=None, if_match=None, if_none_match=None, if_modified_since=None,
+        if_unmodified_since=None):
         """overide is forcing the local file to become exactly as the remote, even if it is
         substantialy different
         """
 
         self.assert_container()
+        islocalfile = False if f.isatty() else True
 
         #retrieve object hashmap
         hashmap = self.get_object_hashmap(obj, version=version, if_match=if_match,
@@ -772,7 +776,7 @@ class PithosClient(StorageClient):
             download_gen.next()
 
         #load local file existing hashmap
-        if not f.isatty():
+        if islocalfile:
             hash_dict = {}
             from os import path
             if path.exists(f.name):
@@ -781,6 +785,7 @@ class PithosClient(StorageClient):
                 h = HashMap(blocksize, blockhash)
                 with_progress_bar = False if download_cb is None else True
                 h.load(f, with_progress_bar)
+                #resume if some blocks have been downloaded
                 for i, x in enumerate(h):
                     existing_hash = hexlify(x)
                     if existing_hash in map_dict:
@@ -791,9 +796,10 @@ class PithosClient(StorageClient):
                         raise ClientError(message='Local file is substantialy different',
                             status=600)
 
-        #download and print
+        #download and save/print
         for i, h in enumerate(map):
-            if not f.isatty() and h in hash_dict:
+            #if not islocalfile and h in hash_dict:
+            if h in hash_dict:
                 continue
             if download_cb is not None:
                 download_gen.next()
@@ -807,17 +813,20 @@ class PithosClient(StorageClient):
             if range is not None and end > custom_end:
                 end = custom_end
             data_range = 'bytes=%s-%s'%(start, end)
-            data = self.object_get(obj, data_range=data_range, success=(200, 206), version=version,
-                if_etag_match=if_match, if_etag_not_match=if_none_match,
+            r = self.object_get(obj, data_range=data_range, success=(200, 206), version=version,
+                if_etag_match=if_match, if_etag_not_match=if_none_match, binary=True,
                 if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since)
-            if not f.isatty():
+            if islocalfile:
                 f.seek(start)
-            f.write(data.content)
+            f.write(r.content)
+            f.flush()
+            r.release()
             #f.write(data.text.encode('utf-8'))
 
-        if overide and not f.isatty():
+        if overide and not islocalfile:
             f.truncate(total_size)
 
+
     def get_object_hashmap(self, obj, version=None, if_match=None, if_none_match=None,
         if_modified_since=None, if_unmodified_since=None):
         try:
@@ -828,8 +837,9 @@ class PithosClient(StorageClient):
             if err.status == 304 or err.status == 412:
                 return {}
             raise
-        from json import loads
-        return loads(r.text)
+        result = r.json
+        r.release()
+        return result
 
     def set_account_group(self, group, usernames):
         self.account_post(update=True, groups = {group:usernames})