Commit dc7cc899 authored by Giorgos Korfiatis's avatar Giorgos Korfiatis
Browse files

handle pithos calls retry

parent 94670581
......@@ -262,7 +262,11 @@ class PithosFileClient(FileClient):
def get_pithos_candidates(self, last_modified=None):
db = self.get_db()
objects = self.endpoint.list_objects()
try:
objects = self.endpoint.list_objects()
except ClientError as e:
logger.error(e)
return {}
self.objects = objects
upstream_all = {}
for obj in objects:
......@@ -292,7 +296,8 @@ class PithosFileClient(FileClient):
(last_modified, candidates))
return candidates
def notifier(self, callback=None, interval=2):
def notifier(self, callback=None):
interval = self.settings.pithos_list_interval
class PollPithosThread(utils.StoppableThread):
def run_body(this):
candidates = self.get_pithos_candidates(
......
......@@ -38,6 +38,8 @@ DEFAULT_CACHE_FETCH_NAME = 'fetched'
GLOBAL_SETTINGS_NAME = '.agkyra'
DEFAULT_DBNAME = "syncer.db"
DEFAULT_ACTION_MAX_WAIT = 10
DEFAULT_PITHOS_LIST_INTERVAL = 5
DEFAULT_CONNECTION_RETRY_LIMIT = 3
thread_local_data = threading.local()
......@@ -95,6 +97,13 @@ class SyncerSettings():
self.heartbeat = HeartBeat()
self.action_max_wait = kwargs.get("action_max_wait",
DEFAULT_ACTION_MAX_WAIT)
self.pithos_list_interval = kwargs.get("pithos_list_interval",
DEFAULT_PITHOS_LIST_INTERVAL)
self.connection_retry_limit = kwargs.get(
"connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
self.messager = Messager()
def get_db(self, initialize=False):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment