Commit b1713259 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Minor bugfixes

parent f364f960
......@@ -586,6 +586,7 @@ class store_download(_store_container_command):
super(self.__class__, self).main(container___path, path_is_optional=False)
#setup output stream
parallel = False
if local_path is None:
out = stdout
else:
......@@ -600,8 +601,9 @@ class store_download(_store_container_command):
download_cb = None if getattr(self.args, 'no_progress_bar') \
else self.progress('Downloading')
try:
self.client.download_object(self.path, out, download_cb,
self.client.download_object(self.path, out, download_cb,
range=getattr(self.args, 'range'), version=getattr(self.args,'object_version'),
if_match=getattr(self.args, 'if_match'), overide=getattr(self.args, 'overide'),
if_none_match=getattr(self.args, 'if_none_match'),
......
......@@ -84,6 +84,7 @@ class Client(object):
#r = self._request(method, path, **kwargs)
success = kwargs.pop('success', 200)
binary = kwargs.pop('binary', False)
data = kwargs.pop('data', None)
self.set_default_header('X-Auth-Token', self.token)
#self.set_default_header('Accept', '*/*')
......@@ -97,7 +98,7 @@ class Client(object):
#kwargs.setdefault('verify', False) # Disable certificate verification
self.http_client.url = self.base_url + path
r = self.http_client.perform_request(method=method, data=data)
r = self.http_client.perform_request(method=method, data=data, binary=binary)
#r = requests.request(method, url, headers=self.headers, data=data, **kwargs)
req = self.http_client
......
......@@ -32,7 +32,6 @@
# or implied, of GRNET S.A.
import requests
from copy import deepcopy
from . import HTTPConnection, HTTPResponse, HTTPConnectionError, HTTPResponsePool
from .pool import ObjectPool
from urlparse import urlparse
......@@ -44,22 +43,20 @@ requests.Response.status = property(_status)
class HTTPRequestsResponse(HTTPResponse):
_get_content_only=False
def _get_response(self):
if self.prefetched:
return
r = self.request.response
try:
self.headers = deepcopy(r.headers)
self.text = deepcopy(r.text) \
if hasattr(r, 'text') else None
self.json = deepcopy(r.json) \
if hasattr(r, 'json') else None
self.content = deepcopy(r.content) \
if hasattr(r, 'content') else None
self.exception = deepcopy(r.exception) \
if hasattr(r, 'exception') else None
self.status = deepcopy(r.status)
self.status_code = deepcopy(r.status_code)
self.headers = r.headers
self.status = r.status
self.status_code = r.status_code
self.content = r.content if hasattr(r, 'content') else None
self.text = None if self._get_content_only else r.text
self.json = None if self._get_content_only else r.json
self.exception = r.exception if hasattr(r, 'exception') else None
except requests.ConnectionError as err:
raise HTTPConnectionError('Connection error', status=651, details=err.message)
except requests.HTTPError as err:
......@@ -70,6 +67,7 @@ class HTTPRequestsResponse(HTTPResponse):
raise HTTPConnectionError('Invalid URL', status=404, details=err.message)
except requests.RequestException as err:
raise HTTPConnectionError('HTTP Request error', status=700, details=err.message)
self.prefetched=True
def release(self):
"""requests object handles this automatically"""
......@@ -104,7 +102,8 @@ class HTTPRequest(HTTPConnection):
respool = self._pools[pool_key]
return respool.pool_get()
def perform_request(self, method=None, url=None, params=None, headers=None, data=None):
def perform_request(self, method=None, url=None, params=None, headers=None, data=None,
binary=False):
"""perform a request
Example: method='PUT' url='https://my.server:8080/path/to/service'
params={'update':None, 'format':'json'} headers={'X-Auth-Token':'s0m3t0k3n=='}
......@@ -131,4 +130,6 @@ class HTTPRequest(HTTPConnection):
self._response_object = requests.request(self.method, self.url, headers=self.headers, data=data,
verify=self.verify, prefetch = False)
res.request = self._response_object.request
if binary:
res._get_content_only = True
return res
......@@ -302,7 +302,7 @@ class PithosClient(StorageClient):
success = kwargs.pop('success', 200)
return self.head(path, *args, success=success, **kwargs)
def object_get(self, object, format='json', hashmap=False, version=None,
def object_get(self, object, format='json', hashmap=False, version=None, binary=False,
data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None,
if_modified_since = None, if_unmodified_since = None, *args, **kwargs):
""" Full Pithos+ GET at object level
......@@ -335,6 +335,8 @@ class PithosClient(StorageClient):
path=path4url(self.account, self.container, object)
success = kwargs.pop('success', 200)
if binary:
kwargs['binary'] = True
return self.get(path, *args, success=success, **kwargs)
def object_put(self, object, format='json', hashmap=False, delimiter = None, if_etag_match=None,
......@@ -730,13 +732,15 @@ class PithosClient(StorageClient):
self.object_put(object, format='json', hashmap=True, content_type=obj_content_type,
json=hashmap, success=201)
def download_object(self, obj, f, download_cb=None, version=None, overide=False, range=None,
if_match=None, if_none_match=None, if_modified_since=None, if_unmodified_since=None):
def download_object(self, obj, f, download_cb=None, version=None, overide=False,
range=None, if_match=None, if_none_match=None, if_modified_since=None,
if_unmodified_since=None):
"""overide is forcing the local file to become exactly as the remote, even if it is
substantialy different
"""
self.assert_container()
islocalfile = False if f.isatty() else True
#retrieve object hashmap
hashmap = self.get_object_hashmap(obj, version=version, if_match=if_match,
......@@ -772,7 +776,7 @@ class PithosClient(StorageClient):
download_gen.next()
#load local file existing hashmap
if not f.isatty():
if islocalfile:
hash_dict = {}
from os import path
if path.exists(f.name):
......@@ -781,6 +785,7 @@ class PithosClient(StorageClient):
h = HashMap(blocksize, blockhash)
with_progress_bar = False if download_cb is None else True
h.load(f, with_progress_bar)
#resume if some blocks have been downloaded
for i, x in enumerate(h):
existing_hash = hexlify(x)
if existing_hash in map_dict:
......@@ -791,9 +796,10 @@ class PithosClient(StorageClient):
raise ClientError(message='Local file is substantialy different',
status=600)
#download and print
#download and save/print
for i, h in enumerate(map):
if not f.isatty() and h in hash_dict:
#if not islocalfile and h in hash_dict:
if h in hash_dict:
continue
if download_cb is not None:
download_gen.next()
......@@ -807,17 +813,20 @@ class PithosClient(StorageClient):
if range is not None and end > custom_end:
end = custom_end
data_range = 'bytes=%s-%s'%(start, end)
data = self.object_get(obj, data_range=data_range, success=(200, 206), version=version,
if_etag_match=if_match, if_etag_not_match=if_none_match,
r = self.object_get(obj, data_range=data_range, success=(200, 206), version=version,
if_etag_match=if_match, if_etag_not_match=if_none_match, binary=True,
if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since)
if not f.isatty():
if islocalfile:
f.seek(start)
f.write(data.content)
f.write(r.content)
f.flush()
r.release()
#f.write(data.text.encode('utf-8'))
if overide and not f.isatty():
if overide and not islocalfile:
f.truncate(total_size)
def get_object_hashmap(self, obj, version=None, if_match=None, if_none_match=None,
if_modified_since=None, if_unmodified_since=None):
try:
......@@ -828,8 +837,9 @@ class PithosClient(StorageClient):
if err.status == 304 or err.status == 412:
return {}
raise
from json import loads
return loads(r.text)
result = r.json
r.release()
return result
def set_account_group(self, group, usernames):
self.account_post(update=True, groups = {group:usernames})
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment