Commit 56f0908a authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Improved incremental download, hashmap, versions

parent 98893eac
......@@ -299,7 +299,8 @@ class PithosClient(StorageClient):
return self.head(path, *args, success=success, **kwargs)
def object_get(self, object, format='json', hashmap=False, version=None,
data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None, if_modified_since = None, if_unmodified_since = None, *args, **kwargs):
data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None,
if_modified_since = None, if_unmodified_since = None, *args, **kwargs):
""" Full Pithos+ GET at object level
--- request parameters ---
@param format (string): json (default) or xml
......@@ -714,6 +715,65 @@ class PithosClient(StorageClient):
self.object_put(object, format='json', hashmap=True, content_type=obj_content_type,
json=hashmap, success=201)
def download_object(self, obj, f, download_cb=None):
self.assert_container()
#retrieve object hashmap
hashmap = self.get_object_hashmap(obj)
blocksize = int(hashmap['block_size'])
blockhash = hashmap['block_hash']
total_size = hashmap['bytes']
map = hashmap['hashes']
map_dict = {}
for h in map:
map_dict[h] = True
download_bars = len(map)
#load progress bar
if download_cb is not None:
download_gen = download_cb(total_size/blocksize + 1)
download_gen.next()
#load local file existing hashmap
if not f.isatty():
hash_dict = {}
index = 0
from os import path
if path.exists(f.name):
from binascii import hexlify
from .pithos_sh_lib.hashmap import HashMap
h = HashMap(blocksize, blockhash)
h.load(f)
for x in h:
existing_hash = hexlify(x)
if existing_hash not in map_dict:
raise ClientError(message='Local file is substantialy different',
status=600)
hash_dict[existing_hash] = index
index += 1
if download_cb:
download_gen.next()
#download and print
for i, h in enumerate(map):
if not f.isatty() and h in hash_dict:
continue
if download_cb is not None:
download_gen.next()
start = i*blocksize
end = start + blocksize -1 if start+blocksize < total_size else total_size -1
data_range = 'bytes=%s-%s'%(start, end)
data = self.object_get(obj, data_range=data_range, success=(200, 206))
if not f.isatty():
f.seek(start)
f.write(data.text)
def get_object_hashmap(self, obj, version=None):
r = self.object_get(obj, hashmap=True, version=version)
from json import loads
return loads(r.text)
def set_account_group(self, group, usernames):
self.account_post(update=True, groups = {group:usernames})
......
......@@ -39,13 +39,15 @@ from .pithos import PithosClient, ClientError
from .cli_utils import raiseCLIError
from kamaki.utils import print_dict, pretty_keys, print_list
from colors import bold
from sys import stdout
from sys import stdout, exit
import signal
from progress.bar import IncrementalBar
class ProgressBar(IncrementalBar):
suffix = '%(percent)d%% - %(eta)ds'
#suffix = '%(percent)d%% - %(eta)ds'
suffix = '%(percent)d%%'
class _pithos_init(object):
def main(self):
......@@ -377,7 +379,6 @@ class store_move(_store_container_command):
class store_append(_store_container_command):
"""Append local file to (existing) remote object"""
def main(self, local_path, container___path):
super(self.__class__, self).main(container___path, path_is_optional=False)
try:
......@@ -527,31 +528,58 @@ class store_upload(_store_container_command):
class store_download(_store_container_command):
"""Download a file"""
def main(self, container___path, local_path='-'):
def main(self, container___path, local_path=None):
super(self.__class__, self).main(container___path, path_is_optional=False)
#setup output stream
if local_path is None:
out = stdout
else:
try:
out = open(local_path, 'a+')
except IOError as err:
raise CLIError(message='Cannot write to file %s - %s'%(local_path,unicode(err)),
importance=1)
download_cb = self.progress('Downloading')
try:
f, size = self.client.get_object(self.path)
self.client.download_object(self.path, out, download_cb)
except ClientError as err:
raiseCLIError(err)
try:
out = open(local_path, 'w') if local_path != '-' else stdout
except IOError:
raise CLIError(message='Cannot write to file %s'%local_path, importance=1)
except KeyboardInterrupt:
print('\ndownload canceled by user')
if local_path is not None:
print('re-run command to resume')
blocksize = 4 * 1024 ** 2
nblocks = 1 + (size - 1) // blocksize
@command()
class store_hashmap(_store_container_command):
"""Get the hashmap of an object"""
cb = self.progress('Downloading blocks') if local_path != '-' else None
if cb:
gen = cb(nblocks)
gen.next()
def update_parser(self, parser):
super(self.__class__, self).update_parser(parser)
parser.add_argument('--range', action='store', dest='range', default=None,
help='show range of data')
parser.add_argument('--if-range', action='store', dest='if_range', default=None,
help='show range of data')
parser.add_argument('--if-match', action='store', dest='if_match', default=None,
help='show output if ETags match')
parser.add_argument('--if-none-match', action='store', dest='if_none_match', default=None,
help='show output if ETags don\'t match')
parser.add_argument('--if-modified-since', action='store', dest='if_modified_since',
default=None, help='show output if modified since then')
parser.add_argument('--if-unmodified-since', action='store', dest='if_unmodified_since',
default=None, help='show output if not modified since then')
parser.add_argument('--object-version', action='store', dest='object_version', default=None,
help='get the specific version')
data = f.read(blocksize)
while data:
out.write(data)
data = f.read(blocksize)
if cb:
gen.next()
def main(self, container___path):
super(self.__class__, self).main(container___path, path_is_optional=False)
try:
data = self.client.get_object_hashmap(self.path,
version=getattr(self.args, 'object_version'))
except ClientError as err:
raiseCLIError(err)
print_dict(data)
@command()
class store_delete(_store_container_command):
......
......@@ -37,6 +37,7 @@ from kamaki.cli import command, CLIError
from kamaki.utils import print_list
from .utils import dict2file, list2file
from .pithos_cli import _store_container_command, _store_account_command
from colors import bold
#from getpass import getuser
#from optparse import OptionParser
......@@ -80,11 +81,28 @@ def _build_args(arglist, attrs):
return args
@command()
class store_download(_pithos_sh_container_command):
class store_versions(_pithos_sh_container_command):
"""Get the version list of an object"""
def main(self, container___path):
super(store_versions, self).main(container___path)
try:
data = self.client.retrieve_object_versionlist(self.container, self.path)
except Fault as err:
raise CLIError(message=unicode(err), status=err.status)
from time import localtime, strftime
print('%s:%s version ids:'%(self.container,self.path))
for vitem in data['versions']:
t = localtime(float(vitem[1]))
vid = bold(unicode(vitem[0]))
print('\t%s \t(%s)'%(vid, strftime('%d-%m-%Y %H:%M:%S', t)))
@command()
class store_downloadz(_pithos_sh_container_command):
"""Download an object"""
def update_parser(self, parser):
super(store_download, self).update_parser(parser)
super(store_downloadz, self).update_parser(parser)
parser.add_argument('-l', action='store_true', dest='detail', default=False,
help='show detailed output')
parser.add_argument('--range', action='store', dest='range', default=None,
......@@ -101,17 +119,13 @@ class store_download(_pithos_sh_container_command):
default=None, help='show output if not modified since then')
parser.add_argument('--object-version', action='store', dest='object-version', default=None,
help='get the specific version')
parser.add_argument('--versionlist', action='store_true', dest='versionlist', default=False,
help='get the full object version list')
parser.add_argument('--hashmap', action='store_true', dest='hashmap', default=False,
help='get the object hashmap instead')
def main(self, container___path, outputFile=None):
super(store_download, self).main(container___path)
super(store_downloadz, self).main(container___path)
#prepare attributes and headers
attrs = ['if_match', 'if_none_match', 'if_modified_since',
'if_unmodified_since', 'hashmap']
'if_unmodified_since']
args = _build_args(self.args, attrs)
args['format'] = 'json' if hasattr(self.args,'detail') else 'text'
if getattr(self.args, 'range') is not None:
......@@ -120,27 +134,18 @@ class store_download(_pithos_sh_container_command):
args['if-range'] = 'If-Range:%s' % getattr(self.args, 'if_range')
#branch through options
if getattr(self.args,'versionlist'):
try:
args.pop('detail')
except KeyError:
pass
args.pop('format')
data=self.client.retrieve_object_versionlist(self.container, self.path, **args)
elif getattr(self.args,'object-version'):
data = self.client.retrieve_object_version(self.container, self.path,
version=getattr(self.args, 'object-version'), **args)
elif getattr(self.args, 'hashmap'):
try:
args.pop('detail')
except KeyError:
pass
data=self.client.retrieve_object_hashmap(self.container, self.path, **args)
elif outputFile is None:
cat(self.client, self.container, self.path)
else:
download(self.client, self.container, self.path, outputFile)
return
try:
if getattr(self.args,'object-version'):
data = self.client.retrieve_object_version(self.container, self.path,
version=getattr(self.args, 'object-version'), **args)
elif outputFile is None:
cat(self.client, self.container, self.path)
return
else:
download(self.client, self.container, self.path, outputFile)
return
except Fault as err:
raise CLIError(message=unicode(err), status=err.status, importance=err.status/100)
f = stdout if outputFile is None else open(outputFile, 'w')
if type(data) is dict:
......@@ -169,5 +174,8 @@ class store_sharers(_pithos_sh_account_command):
attrs = ['limit', 'marker']
args = _build_args(self.args, attrs)
args['format'] = 'json' if getattr(self.args, 'detail') else 'text'
print_list(self.client.list_shared_by_others(**args))
try:
print_list(self.client.list_shared_by_others(**args))
except Fault as err:
raise CLIError(message=unicode(err), status=err.status, importance=err.status/100)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment