More retries, dynamic upload thread limit

Even the slightest progress causes upload to retry
if 502 client error is received, # of threads are limited to current thread

kamaki/clients/__init__.py

@@ -120,6 +120,7 @@ class Client(object):
         self._elapsed_new = 0.0
 
     def _watch_thread_limit(self, threadlist):
+        recvlog.debug('# running threads: %s' % len(threadlist))
         if self._elapsed_old > self._elapsed_new\
         and self._thread_limit < self.POOL_SIZE:
             self._thread_limit += 1
@@ -202,7 +203,7 @@ class Client(object):
                 sendlog.info('\t%s: %s', key, val)
             sendlog.info('')
             if data:
-                sendlog.debug('%s', data)
+                sendlog.info(data[:256] + ' ...')
 
             recvlog.info('%d %s', r.status_code, r.status)
             for key, val in r.headers.items():

kamaki/clients/pithos.py

@@ -180,6 +180,9 @@ class PithosClient(PithosRestAPI):
         return event
 
     def _put_block(self, data, hash):
+        #from random import randint
+        #if randint(0,2):
+        #    raise ClientError('BAD GATEWAY STUFF', 502)
         r = self.container_post(update=True,
             content_type='application/octet-stream',
             content_length=len(data),
@@ -278,6 +281,9 @@ class PithosClient(PithosRestAPI):
 
                 if thread.exception:
                     failures.append(thread)
+                    if isinstance(thread.exception, ClientError)\
+                    and thread.exception.status == 502:
+                        self.POOLSIZE = self._thread_limit
                 elif thread.isAlive():
                     unfinished.append(thread)
             flying = unfinished
@@ -359,9 +365,11 @@ class PithosClient(PithosRestAPI):
                     hmap,
                     f,
                     upload_cb=upload_cb)
-                if missing and num_of_blocks > len(missing):
-                    num_of_blocks = len(missing)
-                    retries -= 1
+                if missing:
+                    if num_of_blocks == len(missing):
+                        retries -= 1
+                    else:
+                        num_of_blocks = len(missing)
                 else:
                     break
             if missing: