pop from candidates before excluding file

agkyra/agkyra/syncer/localfs_client.py

@@ -40,6 +40,10 @@ OS_NOT_A_DIR = 20
 OS_NO_FILE_OR_DIR = 2
 
 
+exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"]
+exclude_pattern = re.compile('|'.join(exclude_regexes))
+
+
 class DirMissing(BaseException):
     pass
 
@@ -462,8 +466,6 @@ class LocalfsFileClient(FileClient):
         self.ROOTPATH = settings.local_root_path
         self.CACHEPATH = settings.cache_path
         self.get_db = settings.get_db
-        self.exclude_files_exp = re.compile('.*\.tmp$')
-        self.exclude_dir_exp = re.compile(self.CACHEPATH)
         self.probe_candidates = common.LockedDict()
 
     def list_candidate_files(self, forced=False):
@@ -478,14 +480,9 @@ class LocalfsFileClient(FileClient):
         for dirpath, dirnames, files in os.walk(self.ROOTPATH):
             rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH)
             logger.debug("'%s' '%s'" % (dirpath, rel_dirpath))
-            # if self.exclude_dir_exp.match(dirpath):
-            #     continue
             if rel_dirpath != '.':
                 candidates[utils.to_standard_sep(rel_dirpath)] = None
             for filename in files:
-                # if self.exclude_files_exp.match(filename) or \
-                #         self.exclude_dir_exp.match(filename):
-                #     continue
                 if rel_dirpath == '.':
                     prefix = ""
                 else:
@@ -502,13 +499,23 @@ class LocalfsFileClient(FileClient):
         local_path = utils.join_path(self.ROOTPATH, name)
         return local_path_changes(local_path, state)
 
+    def exclude_file(self, objname):
+        parts = objname.split(common.OBJECT_DIRSEP)
+        init_part = parts[0]
+        if init_part in [self.settings.cache_name]:
+            return True
+        final_part = parts[-1]
+        return exclude_pattern.match(final_part)
+
     def start_probing_file(self, objname, old_state, ref_state,
                            assumed_info=None,
                            callback=None):
-        if old_state.serial != ref_state.serial:
-            logger.warning("Serial mismatch in probing path '%s'" % objname)
-            return
         cached_info = self.probe_candidates.pop(objname)
+        if self.exclude_file(objname):
+            logger.warning("Ignoring probe archive: %s, object: %s" %
+                           (old_state.archive, objname))
+            return
+
         live_info = (self._local_path_changes(objname, old_state)
                      if cached_info is None else cached_info)
         if live_info is None:

agkyra/agkyra/syncer/pithos_client.py

@@ -323,12 +323,12 @@ class PithosFileClient(FileClient):
     def start_probing_file(self, objname, old_state, ref_state,
                            assumed_info=None,
                            callback=None):
+        info = old_state.info
+        cached_info = self.probe_candidates.pop(objname)
         if exclude_pattern.match(objname):
             logger.warning("Ignoring probe archive: %s, object: '%s'" %
                            (old_state.archive, objname))
             return
-        info = old_state.info
-        cached_info = self.probe_candidates.pop(objname)
         if cached_info is None:
             obj = self.get_object(objname)
             live_info = self.get_object_live_info(obj)

agkyra/agkyra/syncer/syncer.py

@@ -16,7 +16,6 @@
 import time
 import threading
 import logging
-import re
 
 from agkyra.syncer import common
 from agkyra.syncer.setup import SyncerSettings
@@ -27,9 +26,6 @@ from agkyra.syncer import messaging, utils
 
 logger = logging.getLogger(__name__)
 
-exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"]
-exclude_pattern = re.compile('|'.join(exclude_regexes))
-
 
 class FileSyncer(object):
 
@@ -100,20 +96,8 @@ class FileSyncer(object):
     def get_next_message(self, block=False):
         return self.messager.get(block=block)
 
-    def exclude_file(self, objname):
-        parts = objname.split(common.OBJECT_DIRSEP)
-        init_part = parts[0]
-        if init_part in [self.settings.cache_name]:
-            return True
-        final_part = parts[-1]
-        return exclude_pattern.match(final_part)
-
     @transaction()
     def probe_file(self, archive, objname, assumed_info=None):
-        if self.exclude_file(objname):
-            logger.warning("Ignoring probe archive: %s, object: %s" %
-                           (archive, objname))
-            return
         logger.info("Probing archive: %s, object: '%s'" % (archive, objname))
         db = self.get_db()
         client = self.clients[archive]
@@ -141,10 +125,6 @@ class FileSyncer(object):
         archive = live_state.archive
         objname = live_state.objname
         serial = live_state.serial
-        if self.exclude_file(objname):
-            logger.warning("Ignoring update archive: %s, object: %s" %
-                           (archive, objname))
-            return
         msg = messaging.UpdateMessage(
             archive=archive, objname=objname, serial=serial, logger=logger)
         self.messager.put(msg)