Commit 745d938b authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Tide up _upload_missing_blocks code + progress chk

progress check in upload: don't let progress bar raise exceptions
parent 7eda693f
......@@ -1046,7 +1046,7 @@ class store_upload(_store_container_command):
'Try to re-upload the file',
'For more error details, try kamaki store upload -d'])
raise_connection_errors(err)
raiseCLIError(err, '"%s" not accessible' % container____path__)
raiseCLIError(err, 'Failed to upload to %s' % container____path__)
except IOError as err:
try:
progress_bar.finish()
......
......@@ -178,9 +178,9 @@ class PithosClient(PithosRestAPI):
return event
def _put_block(self, data, hash):
#from random import randint
#if randint(0, 3):
# raise ClientError('BAD GATEWAY STUFF', 503)
from random import randint
if not randint(0, 7):
raise ClientError('BAD GATEWAY STUFF', 503)
r = self.container_post(update=True,
content_type='application/octet-stream',
content_length=len(data),
......@@ -264,29 +264,20 @@ class PithosClient(PithosRestAPI):
data = fileobj.read(bytes)
r = self._put_block_async(data, hash, upload_gen)
flying.append(r)
unfinished = []
current_fails = 0
for thread in flying:
unfinished = self._watch_thread_limit(unfinished)
unfinished = self._watch_thread_limit(flying)
for thread in set(flying).difference(unfinished):
if thread.exception:
failures.append(thread)
current_fails += 1
if isinstance(thread.exception, ClientError)\
and thread.exception.status == 502:
self.POOLSIZE = self._thread_limit
elif thread.isAlive():
unfinished.append(thread)
flying.append(thread)
elif upload_gen:
try:
upload_gen.next()
except:
pass
if upload_gen:
for i in range(len(unfinished), len(flying) - current_fails):
try:
upload_gen.next()
except:
pass
flying = unfinished
for thread in flying:
......@@ -364,8 +355,11 @@ class PithosClient(PithosRestAPI):
if upload_cb:
upload_gen = upload_cb(len(missing))
for i in range(len(missing), len(hashmap)):
upload_gen.next()
for i in range(len(missing), len(hashmap['hashes']) + 1):
try:
upload_gen.next()
except:
upload_gen = None
else:
upload_gen = None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment