Commit 01d14153 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Add a Connection Retry to Client + ResponseManager

Usage:
cl.XxxClient(...)
cl.CONNECTION_RETRY_LIMIT = 5
cl.<methods>
parent c3c85430
...@@ -36,7 +36,7 @@ from urlparse import urlparse ...@@ -36,7 +36,7 @@ from urlparse import urlparse
from threading import Thread from threading import Thread
from json import dumps, loads from json import dumps, loads
from time import time from time import time
from httplib import ResponseNotReady from httplib import ResponseNotReady, HTTPException
from time import sleep from time import sleep
from random import random from random import random
from logging import getLogger from logging import getLogger
...@@ -185,10 +185,15 @@ class RequestManager(Logged): ...@@ -185,10 +185,15 @@ class RequestManager(Logged):
class ResponseManager(Logged): class ResponseManager(Logged):
"""Manage the http request and handle the response data, headers, etc.""" """Manage the http request and handle the response data, headers, etc."""
def __init__(self, request, poolsize=None): def __init__(self, request, poolsize=None, connection_retry_limit=0):
""" """
:param request: (RequestManager) :param request: (RequestManager)
:param poolsize: (int) the size of the connection pool
:param connection_retry_limit: (int)
""" """
self.CONNECTION_TRY_LIMIT = 1 + connection_retry_limit
self.request = request self.request = request
self._request_performed = False self._request_performed = False
self.poolsize = poolsize self.poolsize = poolsize
...@@ -198,39 +203,51 @@ class ResponseManager(Logged): ...@@ -198,39 +203,51 @@ class ResponseManager(Logged):
return return
pool_kw = dict(size=self.poolsize) if self.poolsize else dict() pool_kw = dict(size=self.poolsize) if self.poolsize else dict()
try: for retries in range(1, self.CONNECTION_TRY_LIMIT + 1):
with PooledHTTPConnection( try:
self.request.netloc, self.request.scheme, with PooledHTTPConnection(
**pool_kw) as connection: self.request.netloc, self.request.scheme,
self.request.LOG_TOKEN = self.LOG_TOKEN **pool_kw) as connection:
self.request.LOG_DATA = self.LOG_DATA self.request.LOG_TOKEN = self.LOG_TOKEN
r = self.request.perform(connection) self.request.LOG_DATA = self.LOG_DATA
recvlog.info('\n%s <-- %s <-- [req: %s]\n' % ( r = self.request.perform(connection)
self, r, self.request)) recvlog.info('\n%s <-- %s <-- [req: %s]\n' % (
self._request_performed = True self, r, self.request))
self._status_code, self._status = r.status, unquote(r.reason) self._request_performed = True
recvlog.info( self._status_code, self._status = r.status, unquote(
'%d %s\t[p: %s]' % (self.status_code, self.status, self)) r.reason)
self._headers = dict() recvlog.info(
for k, v in r.getheaders(): '%d %s\t[p: %s]' % (
if (not self.LOG_TOKEN) and k.lower() == 'x-auth-token': self.status_code, self.status, self))
continue self._headers = dict()
v = unquote(v) for k, v in r.getheaders():
self._headers[k] = v if (not self.LOG_TOKEN) and (
recvlog.info(' %s: %s\t[p: %s]' % (k, v, self)) k.lower() == 'x-auth-token'):
self._content = r.read() continue
recvlog.info('data size: %s\t[p: %s]' % ( v = unquote(v)
len(self._content) if self._content else 0, self._headers[k] = v
self)) recvlog.info(' %s: %s\t[p: %s]' % (k, v, self))
if self.LOG_DATA and self._content: self._content = r.read()
recvlog.info('%s\t[p: %s]' % (self._content, self)) recvlog.info('data size: %s\t[p: %s]' % (
except Exception as err: len(self._content) if self._content else 0,
from traceback import format_stack self))
recvlog.debug('\n'.join(['%s' % type(err)] + format_stack())) if self.LOG_DATA and self._content:
raise ClientError( recvlog.info('%s\t[p: %s]' % (self._content, self))
'Failed while http-connecting to %s (%s)' % ( break
self.request.url, except Exception as err:
err)) if isinstance(err, HTTPException):
if retries >= self.CONNECTION_TRY_LIMIT:
raise ClientError(
'Connection to %s failed %s times (%s: %s )' % (
self.request.url, retries, type(err), err))
else:
from traceback import format_stack
recvlog.debug(
'\n'.join(['%s' % type(err)] + format_stack()))
raise ClientError(
'Failed while http-connecting to %s (%s)' % (
self.request.url,
err))
@property @property
def status_code(self): def status_code(self):
...@@ -309,6 +326,7 @@ class Client(object): ...@@ -309,6 +326,7 @@ class Client(object):
'%a, %d %b %Y %H:%M:%S GMT'] '%a, %d %b %Y %H:%M:%S GMT']
LOG_TOKEN = False LOG_TOKEN = False
LOG_DATA = False LOG_DATA = False
CONNECTION_RETRY_LIMIT = 0
def __init__(self, base_url, token): def __init__(self, base_url, token):
assert base_url, 'No base_url for client %s' % self assert base_url, 'No base_url for client %s' % self
...@@ -395,7 +413,8 @@ class Client(object): ...@@ -395,7 +413,8 @@ class Client(object):
method, self.base_url, path, method, self.base_url, path,
data=data, headers=headers, params=params) data=data, headers=headers, params=params)
# req.log() # req.log()
r = ResponseManager(req) r = ResponseManager(
req, connection_retry_limit=self.CONNECTION_RETRY_LIMIT)
r.LOG_TOKEN, r.LOG_DATA = self.LOG_TOKEN, self.LOG_DATA r.LOG_TOKEN, r.LOG_DATA = self.LOG_TOKEN, self.LOG_DATA
finally: finally:
self.headers = dict() self.headers = dict()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment