From ede6421e6d2e318457753c474751dc1c5e5dc55c Mon Sep 17 00:00:00 2001
From: Giorgos Korfiatis <gkorf@grnet.gr>
Date: Thu, 7 May 2015 18:27:13 +0300
Subject: [PATCH] probe all in single transaction BUT check probe_candidates on
retry
---
agkyra/agkyra/syncer/localfs_client.py | 5 ++---
agkyra/agkyra/syncer/pithos_client.py | 7 +++----
agkyra/agkyra/syncer/syncer.py | 12 ++++++++----
3 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/agkyra/agkyra/syncer/localfs_client.py b/agkyra/agkyra/syncer/localfs_client.py
index 76d676f..731ae74 100644
--- a/agkyra/agkyra/syncer/localfs_client.py
+++ b/agkyra/agkyra/syncer/localfs_client.py
@@ -532,7 +532,7 @@ class LocalfsFileClient(FileClient):
final_part = parts[-1]
return exclude_pattern.match(final_part)
- def start_probing_file(self, objname, old_state, ref_state, callback=None):
+ def probe_file(self, objname, old_state, ref_state):
with self.probe_candidates.lock() as d:
cached_info = d.pop(objname, None)
if self.exclude_file(objname):
@@ -545,8 +545,7 @@ class LocalfsFileClient(FileClient):
if live_info is None:
return
live_state = old_state.set(info=live_info)
- if callback is not None:
- callback(live_state)
+ return live_state
def stage_file(self, source_state):
return LocalfsSourceHandle(self.settings, source_state)
diff --git a/agkyra/agkyra/syncer/pithos_client.py b/agkyra/agkyra/syncer/pithos_client.py
index 14818de..0080bb5 100644
--- a/agkyra/agkyra/syncer/pithos_client.py
+++ b/agkyra/agkyra/syncer/pithos_client.py
@@ -340,7 +340,7 @@ class PithosFileClient(FileClient):
PITHOS_TYPE: p_type,
}
- def start_probing_file(self, objname, old_state, ref_state, callback=None):
+ def probe_file(self, objname, old_state, ref_state):
info = old_state.info
with self.probe_candidates.lock() as d:
cached_info = d.pop(objname, None)
@@ -354,9 +354,8 @@ class PithosFileClient(FileClient):
else:
live_info = cached_info
if info != live_info:
- if callback is not None:
- live_state = old_state.set(info=live_info)
- callback(live_state)
+ live_state = old_state.set(info=live_info)
+ return live_state
def stage_file(self, source_state):
return PithosSourceHandle(self.settings, source_state)
diff --git a/agkyra/agkyra/syncer/syncer.py b/agkyra/agkyra/syncer/syncer.py
index ab0c830..0d8999c 100644
--- a/agkyra/agkyra/syncer/syncer.py
+++ b/agkyra/agkyra/syncer/syncer.py
@@ -97,6 +97,9 @@ class FileSyncer(object):
@transaction()
def probe_file(self, archive, objname):
+ return self._probe_file(archive, objname)
+
+ def _probe_file(self, archive, objname):
logger.info("Probing archive: %s, object: '%s'" % (archive, objname))
db = self.get_db()
client = self.clients[archive]
@@ -114,10 +117,10 @@ class FileSyncer(object):
logger.warning("Serial mismatch in probing archive: %s, "
"object: '%s'" % (archive, objname))
return
- client.start_probing_file(objname, db_state, ref_state,
- callback=self.update_file_state)
+ live_state = client.probe_file(objname, db_state, ref_state)
+ if live_state is not None:
+ self.update_file_state(live_state)
- @transaction()
def update_file_state(self, live_state):
db = self.get_db()
archive = live_state.archive
@@ -345,11 +348,12 @@ class FileSyncer(object):
return set(db.list_deciding(archives=archives,
sync=self.SYNC))
+ @transaction()
def probe_archive(self, archive, forced=False):
client = self.clients[archive]
candidates = client.list_candidate_files(forced=forced)
for objname in candidates:
- self.probe_file(archive, objname)
+ self._probe_file(archive, objname)
def decide_archive(self, archive):
for objname in self.list_deciding([archive]):
--
GitLab