diff --git a/agkyra/agkyra/syncer/pithos_client.py b/agkyra/agkyra/syncer/pithos_client.py index af63487fb2d7fe8debd082d91ff3eec0eb10d245..24d1c0fa5818dcb771a5ce3588e5cd91bafbe3ed 100644 --- a/agkyra/agkyra/syncer/pithos_client.py +++ b/agkyra/agkyra/syncer/pithos_client.py @@ -262,7 +262,11 @@ class PithosFileClient(FileClient): def get_pithos_candidates(self, last_modified=None): db = self.get_db() - objects = self.endpoint.list_objects() + try: + objects = self.endpoint.list_objects() + except ClientError as e: + logger.error(e) + return {} self.objects = objects upstream_all = {} for obj in objects: @@ -292,7 +296,8 @@ class PithosFileClient(FileClient): (last_modified, candidates)) return candidates - def notifier(self, callback=None, interval=2): + def notifier(self, callback=None): + interval = self.settings.pithos_list_interval class PollPithosThread(utils.StoppableThread): def run_body(this): candidates = self.get_pithos_candidates( diff --git a/agkyra/agkyra/syncer/setup.py b/agkyra/agkyra/syncer/setup.py index 743c78c44a49d4bb3cc5b35c45155d43e4a91dab..b96b2104bb842953dc0bfa6965ec2c895c0ed6cb 100644 --- a/agkyra/agkyra/syncer/setup.py +++ b/agkyra/agkyra/syncer/setup.py @@ -38,6 +38,8 @@ DEFAULT_CACHE_FETCH_NAME = 'fetched' GLOBAL_SETTINGS_NAME = '.agkyra' DEFAULT_DBNAME = "syncer.db" DEFAULT_ACTION_MAX_WAIT = 10 +DEFAULT_PITHOS_LIST_INTERVAL = 5 +DEFAULT_CONNECTION_RETRY_LIMIT = 3 thread_local_data = threading.local() @@ -95,6 +97,13 @@ class SyncerSettings(): self.heartbeat = HeartBeat() self.action_max_wait = kwargs.get("action_max_wait", DEFAULT_ACTION_MAX_WAIT) + self.pithos_list_interval = kwargs.get("pithos_list_interval", + DEFAULT_PITHOS_LIST_INTERVAL) + + self.connection_retry_limit = kwargs.get( + "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT) + self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit + self.messager = Messager() def get_db(self, initialize=False):