Commit ca092af4 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Adjust up to store-overwrite

parent 1395c40e
......@@ -44,6 +44,10 @@ from argparse import RawDescriptionHelpFormatter
try:
from progress.bar import ShadyBar as KamakiProgressBar
except ImportError:
try:
from progress.bar import Bar as KamakiProgressBar
except ImportError:
pass
# progress not installed - pls, pip install progress
pass
......
......@@ -37,6 +37,7 @@ import logging
from kamaki.clients import ClientError
from kamaki.cli.errors import CLIError, raiseCLIError, CLISyntaxError
from kamaki.cli import _debug, kloger
from kamaki.cli.utils import format_size
sendlog = logging.getLogger('clients.send')
datasendlog = logging.getLogger('data.send')
......@@ -398,7 +399,8 @@ class pithos(object):
' 1. Set store.container variable (permanent)',
' /config set store.container <container>',
' 2. --container=<container> (temporary, overrides 1)',
' 3. Use the container:path format (temporary, overrides all)']
' 3. Use the container:path format (temporary, overrides all)',
'For a list of containers: /store list']
@classmethod
def connection(this, foo):
......@@ -429,9 +431,10 @@ class pithos(object):
return foo(self, *args, **kwargs)
except ClientError as ce:
if ce.status == 404 and 'container' in ('%s' % ce).lower():
cont = '%s or %s' if dst_cont else self.container
cont = '%s or %s' % (self.container, dst_cont)\
if dst_cont else self.container
raiseCLIError(ce,
'No container %s in account %s' % (
'Is container %s in account %s ?' % (
cont,
self.account),
details=this.container_howto)
......@@ -452,3 +455,54 @@ class pithos(object):
details=this.container_howto)
raise
return _raise
@classmethod
def object_size(this, foo):
def _raise(self, *args, **kwargs):
size = kwargs.get('size', None)
start = kwargs.get('start', 0)
end = kwargs.get('end', 0)
if size:
try:
size = int(size)
except ValueError as ve:
raiseCLIError(ve,
'Invalid file size %s ' % size,
details=['size must be a positive integer'],
importance=1)
else:
try:
start = int(start)
except ValueError as e:
raiseCLIError(e,
'Invalid start value %s in range' % start,
details=['size must be a positive integer'],
importance=1)
try:
end = int(end)
except ValueError as e:
raiseCLIError(e,
'Invalid end value %s in range' % end,
details=['size must be a positive integer'],
importance=1)
if start > end:
raiseCLIError(
'Invalid range %s-%s' % (start, end),
details=['size must be a positive integer'],
importance=1)
size = end - start
try:
return foo(self, *args, **kwargs)
except ClientError as ce:
err_msg = ('%s' % ce).lower()
if size and (ce.status == 416 or
(ce.status == 400 and\
'object length is smaller than range length' in err_msg)):
raiseCLIError(ce,
'Remote object %s:%s <= %s %s' % (
self.container,
self.path,
format_size(size),
('(%sB)' % size) if size >= 1024 else ''))
raise
return _raise
......@@ -696,75 +696,43 @@ class store_append(_store_container_command):
default=False)
)
def main(self, local_path, container___path):
super(self.__class__, self).main(
container___path,
path_is_optional=False)
progress_bar = self.arguments['progress_bar']
try:
upload_cb = progress_bar.get_generator('Appending blocks')
except Exception:
upload_cb = None
@errors.generic.all
@errors.pithos.connection
@errors.pithos.container
@errors.pithos.object_path
def _run(self, local_path):
(progress_bar, upload_cb) = self._safe_progress_bar('Appending')
try:
f = open(local_path, 'rb')
self.client.append_object(self.path, f, upload_cb)
except ClientError as err:
progress_bar.finish()
if err.status == 404:
if 'container' in ('%s' % err).lower():
raiseCLIError(
err,
'No container %s in account %s'\
% (self.container, self.account),
details=errors.pithos.container_howto)
elif 'object' in ('%s' % err).lower():
raiseCLIError(
err,
'No object %s in container %s'\
% (self.path, self.container),
details=errors.pithos.container_howto)
raise_connection_errors(err)
raiseCLIError(err)
except Exception as e:
progress_bar.finish()
raiseCLIError(e)
except Exception:
self._safe_progress_bar_finish(progress_bar)
raise
finally:
progress_bar.finish()
self._safe_progress_bar_finish(progress_bar)
def main(self, local_path, container___path):
super(self.__class__, self)._run(
container___path,
path_is_optional=False)
self._run(local_path)
@command(pithos_cmds)
class store_truncate(_store_container_command):
"""Truncate remote file up to a size (default is 0)"""
@errors.generic.all
@errors.pithos.connection
@errors.pithos.container
@errors.pithos.object_path
@errors.pithos.object_size
def _run(self, size=0):
self.client.truncate_object(self.path, size)
def main(self, container___path, size=0):
super(self.__class__, self).main(container___path)
try:
self.client.truncate_object(self.path, size)
except ClientError as err:
if err.status == 404:
if 'container' in ('%s' % err).lower():
raiseCLIError(
err,
'No container %s in account %s'\
% (self.container, self.account),
details=errors.pithos.container_howto)
elif 'object' in ('%s' % err).lower():
raiseCLIError(
err,
'No object %s in container %s'\
% (self.path, self.container),
details=errors.pithos.container_howto)
if err.status == 400 and\
'object length is smaller than range length'\
in ('%s' % err).lower():
raiseCLIError(err, 'Object %s:%s <= %sb' % (
self.container,
self.path,
size))
raise_connection_errors(err)
raiseCLIError(err)
except Exception as e:
raiseCLIError(e)
super(self.__class__, self)._run(container___path)
self._run(size=size)
@command(pithos_cmds)
......@@ -784,58 +752,40 @@ class store_overwrite(_store_container_command):
default=False)
)
def main(self, local_path, container____path__, start, end):
(start, end) = check_range(start, end)
super(self.__class__, self).main(container____path__)
try:
f = open(local_path, 'rb')
f.seek(0, 2)
f_size = f.tell()
f.seek(start, 0)
except Exception as e:
raiseCLIError(e)
progress_bar = self.arguments['progress_bar']
try:
upload_cb = progress_bar.get_generator(
'Overwriting %s blocks' % end - start)
except Exception:
upload_cb = None
def _open_file(self, local_path, start):
f = open(path.abspath(local_path), 'rb')
f.seek(0, 2)
f_size = f.tell()
f.seek(start, 0)
return (f, f_size)
@errors.generic.all
@errors.pithos.connection
@errors.pithos.container
@errors.pithos.object_path
@errors.pithos.object_size
def _run(self, local_path, start, end):
(start, end) = (int(start), int(end))
(f, f_size) = self._open_file(local_path, start)
(progress_bar, upload_cb) = self._safe_progress_bar(
'Overwrite %s bytes' % (end - start))
try:
self.path = self.path if self.path else path.basename(local_path)
self.client.overwrite_object(
obj=self.path,
start=start,
end=end,
source_file=f,
upload_cb=upload_cb)
except ClientError as err:
progress_bar.finish()
if (err.status == 400 and\
'content length does not match range' in ('%s' % err).lower()) or\
err.status == 416:
raiseCLIError(err, details=[
'Content size: %s' % f_size,
'Range: %s-%s (size: %s)' % (start, end, end - start)])
elif err.status == 404:
if 'container' in ('%s' % err).lower():
raiseCLIError(
err,
'No container %s in account %s'\
% (self.container, self.account),
details=errors.pithos.container_howto)
elif 'object' in ('%s' % err).lower():
raiseCLIError(
err,
'No object %s in container %s'\
% (self.path, self.container),
details=errors.pithos.container_howto)
raise_connection_errors(err)
raiseCLIError(err)
except Exception as e:
progress_bar.finish()
raiseCLIError(e)
except Exception:
self._safe_progress_bar_finish(progress_bar)
raise
finally:
progress_bar.finish()
self._safe_progress_bar_finish(progress_bar)
def main(self, local_path, container____path__, start, end):
super(self.__class__, self)._run(container____path__)
self.path = self.path if self.path else path.basename(local_path)
self._run(local_path=local_path, start=start, end=end)
@command(pithos_cmds)
......
......@@ -908,8 +908,13 @@ class PithosClient(PithosRestAPI):
:returns: (dict)
"""
r = self.object_head(obj, version=version)
return r.headers
try:
r = self.object_head(obj, version=version)
return r.headers
except ClientError as ce:
if ce.status == 404:
raise ClientError('Object not found', status=404)
raise
def get_object_meta(self, obj, version=None):
"""
......@@ -985,8 +990,9 @@ class PithosClient(PithosRestAPI):
filesize = fstat(source_file.fileno()).st_size
nblocks = 1 + (filesize - 1) // blocksize
offset = 0
if upload_cb is not None:
if upload_cb:
upload_gen = upload_cb(nblocks)
upload_gen.next()
for i in range(nblocks):
block = source_file.read(min(blocksize, filesize - offset))
offset += len(block)
......@@ -998,7 +1004,7 @@ class PithosClient(PithosRestAPI):
data=block)
r.release()
if upload_cb is not None:
if upload_cb:
upload_gen.next()
def truncate_object(self, obj, upto_bytes):
......@@ -1034,6 +1040,16 @@ class PithosClient(PithosRestAPI):
:param upload_db: progress.bar for uploading
"""
r = self.get_object_info(obj)
rf_size = int(r['content-length'])
if rf_size < int(start):
raise ClientError(
'Range start exceeds file size',
status=416)
elif rf_size < int(end):
raise ClientError(
'Range end exceeds file size',
status=416)
self._assert_container()
meta = self.get_container_info()
blocksize = int(meta['x-container-block-size'])
......@@ -1041,22 +1057,25 @@ class PithosClient(PithosRestAPI):
datasize = int(end) - int(start) + 1
nblocks = 1 + (datasize - 1) // blocksize
offset = 0
if upload_cb is not None:
if upload_cb:
upload_gen = upload_cb(nblocks)
upload_gen.next()
for i in range(nblocks):
block = source_file.read(min(blocksize,
filesize - offset,
datasize - offset))
offset += len(block)
r = self.object_post(obj,
update=True,
content_type='application/octet-stream',
content_length=len(block),
content_range='bytes %s-%s/*' % (start, end),
content_range='bytes %s-%s/*' % (
start + offset,
start + offset + len(block) - 1),
data=block)
offset += len(block)
r.release()
if upload_cb is not None:
if upload_cb:
upload_gen.next()
def copy_object(self, src_container, src_object, dst_container,
......
......@@ -34,7 +34,7 @@
# or implied, of GRNET S.A.
from setuptools import setup
#from sys import version_info
from sys import version_info
import collections
import kamaki
......@@ -42,8 +42,10 @@ import kamaki
optional = ['ansicolors',
'progress>=1.0.2']
requires = ['objpool',
'argparse']
requires = ['objpool']
if version_info[:1] == (2, 6):
requires.append('argparse')
if not hasattr(collections, "OrderedDict"): # Python 2.6
requires.append("ordereddict")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment