diff --git a/agkyra/agkyra/syncer/localfs_client.py b/agkyra/agkyra/syncer/localfs_client.py index 0791c554efc6228a8e3a237657f692776bf3b523..dcd8fd0e1df540a23fa24469421588d41b7ec4cc 100644 --- a/agkyra/agkyra/syncer/localfs_client.py +++ b/agkyra/agkyra/syncer/localfs_client.py @@ -40,6 +40,10 @@ OS_NOT_A_DIR = 20 OS_NO_FILE_OR_DIR = 2 +exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"] +exclude_pattern = re.compile('|'.join(exclude_regexes)) + + class DirMissing(BaseException): pass @@ -462,8 +466,6 @@ class LocalfsFileClient(FileClient): self.ROOTPATH = settings.local_root_path self.CACHEPATH = settings.cache_path self.get_db = settings.get_db - self.exclude_files_exp = re.compile('.*\.tmp$') - self.exclude_dir_exp = re.compile(self.CACHEPATH) self.probe_candidates = common.LockedDict() def list_candidate_files(self, forced=False): @@ -478,14 +480,9 @@ class LocalfsFileClient(FileClient): for dirpath, dirnames, files in os.walk(self.ROOTPATH): rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH) logger.debug("'%s' '%s'" % (dirpath, rel_dirpath)) - # if self.exclude_dir_exp.match(dirpath): - # continue if rel_dirpath != '.': candidates[utils.to_standard_sep(rel_dirpath)] = None for filename in files: - # if self.exclude_files_exp.match(filename) or \ - # self.exclude_dir_exp.match(filename): - # continue if rel_dirpath == '.': prefix = "" else: @@ -502,13 +499,23 @@ class LocalfsFileClient(FileClient): local_path = utils.join_path(self.ROOTPATH, name) return local_path_changes(local_path, state) + def exclude_file(self, objname): + parts = objname.split(common.OBJECT_DIRSEP) + init_part = parts[0] + if init_part in [self.settings.cache_name]: + return True + final_part = parts[-1] + return exclude_pattern.match(final_part) + def start_probing_file(self, objname, old_state, ref_state, assumed_info=None, callback=None): - if old_state.serial != ref_state.serial: - logger.warning("Serial mismatch in probing path '%s'" % objname) - return cached_info = self.probe_candidates.pop(objname) + if self.exclude_file(objname): + logger.warning("Ignoring probe archive: %s, object: %s" % + (old_state.archive, objname)) + return + live_info = (self._local_path_changes(objname, old_state) if cached_info is None else cached_info) if live_info is None: diff --git a/agkyra/agkyra/syncer/pithos_client.py b/agkyra/agkyra/syncer/pithos_client.py index 676eaad49bb70c20aa7619a062ca72e659ccf227..759990e2fc68213a2a3ce46f1c46f28934e02c74 100644 --- a/agkyra/agkyra/syncer/pithos_client.py +++ b/agkyra/agkyra/syncer/pithos_client.py @@ -323,12 +323,12 @@ class PithosFileClient(FileClient): def start_probing_file(self, objname, old_state, ref_state, assumed_info=None, callback=None): + info = old_state.info + cached_info = self.probe_candidates.pop(objname) if exclude_pattern.match(objname): logger.warning("Ignoring probe archive: %s, object: '%s'" % (old_state.archive, objname)) return - info = old_state.info - cached_info = self.probe_candidates.pop(objname) if cached_info is None: obj = self.get_object(objname) live_info = self.get_object_live_info(obj) diff --git a/agkyra/agkyra/syncer/syncer.py b/agkyra/agkyra/syncer/syncer.py index 99fb3b3dfead91c7e3eda0fba54f5507fe835ac7..3491a41bcd179d89d6deeb9893a84e688236f9bd 100644 --- a/agkyra/agkyra/syncer/syncer.py +++ b/agkyra/agkyra/syncer/syncer.py @@ -16,7 +16,6 @@ import time import threading import logging -import re from agkyra.syncer import common from agkyra.syncer.setup import SyncerSettings @@ -27,9 +26,6 @@ from agkyra.syncer import messaging, utils logger = logging.getLogger(__name__) -exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"] -exclude_pattern = re.compile('|'.join(exclude_regexes)) - class FileSyncer(object): @@ -100,20 +96,8 @@ class FileSyncer(object): def get_next_message(self, block=False): return self.messager.get(block=block) - def exclude_file(self, objname): - parts = objname.split(common.OBJECT_DIRSEP) - init_part = parts[0] - if init_part in [self.settings.cache_name]: - return True - final_part = parts[-1] - return exclude_pattern.match(final_part) - @transaction() def probe_file(self, archive, objname, assumed_info=None): - if self.exclude_file(objname): - logger.warning("Ignoring probe archive: %s, object: %s" % - (archive, objname)) - return logger.info("Probing archive: %s, object: '%s'" % (archive, objname)) db = self.get_db() client = self.clients[archive] @@ -141,10 +125,6 @@ class FileSyncer(object): archive = live_state.archive objname = live_state.objname serial = live_state.serial - if self.exclude_file(objname): - logger.warning("Ignoring update archive: %s, object: %s" % - (archive, objname)) - return msg = messaging.UpdateMessage( archive=archive, objname=objname, serial=serial, logger=logger) self.messager.put(msg)