Commit e7e1b4f4 authored by Giorgos Korfiatis's avatar Giorgos Korfiatis
Browse files

pop from candidates before excluding file

parent 18658a87
......@@ -40,6 +40,10 @@ OS_NOT_A_DIR = 20
OS_NO_FILE_OR_DIR = 2
exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"]
exclude_pattern = re.compile('|'.join(exclude_regexes))
class DirMissing(BaseException):
pass
......@@ -462,8 +466,6 @@ class LocalfsFileClient(FileClient):
self.ROOTPATH = settings.local_root_path
self.CACHEPATH = settings.cache_path
self.get_db = settings.get_db
self.exclude_files_exp = re.compile('.*\.tmp$')
self.exclude_dir_exp = re.compile(self.CACHEPATH)
self.probe_candidates = common.LockedDict()
def list_candidate_files(self, forced=False):
......@@ -478,14 +480,9 @@ class LocalfsFileClient(FileClient):
for dirpath, dirnames, files in os.walk(self.ROOTPATH):
rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH)
logger.debug("'%s' '%s'" % (dirpath, rel_dirpath))
# if self.exclude_dir_exp.match(dirpath):
# continue
if rel_dirpath != '.':
candidates[utils.to_standard_sep(rel_dirpath)] = None
for filename in files:
# if self.exclude_files_exp.match(filename) or \
# self.exclude_dir_exp.match(filename):
# continue
if rel_dirpath == '.':
prefix = ""
else:
......@@ -502,13 +499,23 @@ class LocalfsFileClient(FileClient):
local_path = utils.join_path(self.ROOTPATH, name)
return local_path_changes(local_path, state)
def exclude_file(self, objname):
parts = objname.split(common.OBJECT_DIRSEP)
init_part = parts[0]
if init_part in [self.settings.cache_name]:
return True
final_part = parts[-1]
return exclude_pattern.match(final_part)
def start_probing_file(self, objname, old_state, ref_state,
assumed_info=None,
callback=None):
if old_state.serial != ref_state.serial:
logger.warning("Serial mismatch in probing path '%s'" % objname)
return
cached_info = self.probe_candidates.pop(objname)
if self.exclude_file(objname):
logger.warning("Ignoring probe archive: %s, object: %s" %
(old_state.archive, objname))
return
live_info = (self._local_path_changes(objname, old_state)
if cached_info is None else cached_info)
if live_info is None:
......
......@@ -323,12 +323,12 @@ class PithosFileClient(FileClient):
def start_probing_file(self, objname, old_state, ref_state,
assumed_info=None,
callback=None):
info = old_state.info
cached_info = self.probe_candidates.pop(objname)
if exclude_pattern.match(objname):
logger.warning("Ignoring probe archive: %s, object: '%s'" %
(old_state.archive, objname))
return
info = old_state.info
cached_info = self.probe_candidates.pop(objname)
if cached_info is None:
obj = self.get_object(objname)
live_info = self.get_object_live_info(obj)
......
......@@ -16,7 +16,6 @@
import time
import threading
import logging
import re
from agkyra.syncer import common
from agkyra.syncer.setup import SyncerSettings
......@@ -27,9 +26,6 @@ from agkyra.syncer import messaging, utils
logger = logging.getLogger(__name__)
exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"]
exclude_pattern = re.compile('|'.join(exclude_regexes))
class FileSyncer(object):
......@@ -100,20 +96,8 @@ class FileSyncer(object):
def get_next_message(self, block=False):
return self.messager.get(block=block)
def exclude_file(self, objname):
parts = objname.split(common.OBJECT_DIRSEP)
init_part = parts[0]
if init_part in [self.settings.cache_name]:
return True
final_part = parts[-1]
return exclude_pattern.match(final_part)
@transaction()
def probe_file(self, archive, objname, assumed_info=None):
if self.exclude_file(objname):
logger.warning("Ignoring probe archive: %s, object: %s" %
(archive, objname))
return
logger.info("Probing archive: %s, object: '%s'" % (archive, objname))
db = self.get_db()
client = self.clients[archive]
......@@ -141,10 +125,6 @@ class FileSyncer(object):
archive = live_state.archive
objname = live_state.objname
serial = live_state.serial
if self.exclude_file(objname):
logger.warning("Ignoring update archive: %s, object: %s" %
(archive, objname))
return
msg = messaging.UpdateMessage(
archive=archive, objname=objname, serial=serial, logger=logger)
self.messager.put(msg)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment