Commit 263eaa01 authored by Giorgos Korfiatis's avatar Giorgos Korfiatis
Browse files

stage by copying

parent 17cf6a79
......@@ -43,6 +43,18 @@ class BusyError(SyncError):
pass
class NotStableBusyError(BusyError):
pass
class OpenBusyError(BusyError):
pass
class ChangedBusyError(BusyError):
pass
class ConflictError(SyncError):
pass
......
......@@ -20,6 +20,7 @@ import datetime
import psutil
import time
import filecmp
import shutil
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
......@@ -41,8 +42,9 @@ LOCAL_OTHER = 5
OS_FILE_EXISTS = 17
OS_NOT_A_DIR = 20
OS_NO_FILE_OR_DIR = 2
OS_IS_DIR = 21
DEFAULT_MTIME_PRECISION = 1e-7
DEFAULT_MTIME_PRECISION = 1e-4
exclude_regexes = ["\.#", "\.~", "~\$", "~.*\.tmp$", "\..*\.swp$"]
exclude_pattern = re.compile('|'.join(exclude_regexes))
......@@ -104,7 +106,12 @@ def eq_float(f1, f2):
def files_equal(f1, f2):
logger.debug("Comparing files: '%s', '%s'" % (f1, f2))
try:
return filecmp.cmp(f1, f2, shallow=False)
except OSError as e:
if e.errno in [OS_NO_FILE_OR_DIR, OS_NOT_A_DIR]:
return False
raise
def info_is_unhandled(info):
......@@ -192,6 +199,10 @@ def path_status(path):
return status
def info_of_regular_file(info):
return info and info[LOCALFS_TYPE] == common.T_FILE
def is_info_eq(info1, info2, unhandled_equal=True):
if {} in [info1, info2]:
return info1 == info2
......@@ -379,10 +390,10 @@ class LocalfsSourceHandle(object):
def get_path_in_cache(self, name):
return utils.join_path(self.cache_path, name)
def lock_file(self):
def copy_file(self):
fspath = self.fspath
if file_is_open(fspath):
raise common.BusyError("File '%s' is open. Aborting"
raise common.OpenBusyError("File '%s' is open. Aborting"
% fspath)
new_registered = self.register_stage_name(fspath)
stage_filename = self.stage_filename
......@@ -398,31 +409,53 @@ class LocalfsSourceHandle(object):
logger.info("Staging file '%s' to '%s'" % (self.objname, stage_path))
try:
os.rename(fspath, stage_path)
except OSError as e:
if e.errno in [OS_NO_FILE_OR_DIR, OS_NOT_A_DIR]:
logger.info("Source does not exist: '%s'" % fspath)
shutil.copy2(fspath, stage_path)
except IOError as e:
if e.errno in [OS_NO_FILE_OR_DIR, OS_IS_DIR]:
logger.info("Source is not a regural file: '%s'" % fspath)
self.unregister_stage_name(stage_filename)
return
else:
raise e
def stage_file(self):
self.lock_file()
if self.staged_path is not None:
if file_is_open(self.staged_path):
os.rename(self.staged_path, self.fspath)
self.copy_file()
live_info = get_live_info(self.fspath)
print "live_info =", self.objname, live_info
self.check_staged(live_info)
self.check_update_source_state(live_info)
def check_staged(self, live_info):
is_reg = info_of_regular_file(live_info)
print "is_reg", self.objname, is_reg
if self.staged_path is None:
if is_reg:
m = ("File '%s' is not in a stable state; unstaged"
% self.objname)
logger.warning(m)
raise common.NotStableBusyError(m)
return
if not is_reg:
os.unlink(self.staged_path)
self.unregister_stage_name(self.stage_filename)
logger.warning("File '%s' is open; unstaged" % self.objname)
raise common.BusyError("File '%s' is open. Undoing" %
self.staged_path)
logger.warning("Path '%s' is not a regular file; unstaged")
return
if path_status(self.staged_path) != LOCAL_FILE:
os.rename(self.staged_path, self.fspath)
if file_is_open(self.fspath):
os.unlink(self.staged_path)
self.unregister_stage_name(self.stage_filename)
logger.warning("Object '%s' is not a regular file; unstaged" %
self.objname)
self.check_update_source_state()
m = "File '%s' is open; unstaged" % self.objname
logger.warning(m)
raise common.OpenBusyError(m)
if not files_equal(self.staged_path, self.fspath):
os.unlink(self.staged_path)
self.unregister_stage_name(self.stage_filename)
m = "File '%s' contents have changed; unstaged" % self.objname
logger.warning(m)
raise common.ChangedBusyError(m)
def __init__(self, settings, source_state):
self.settings = settings
......@@ -438,7 +471,7 @@ class LocalfsSourceHandle(object):
self.stage_filename = None
self.staged_path = None
self.heartbeat = settings.heartbeat
if self.needs_staging():
if info_of_regular_file(self.source_state.info):
self.stage_file()
@transaction()
......@@ -446,10 +479,8 @@ class LocalfsSourceHandle(object):
db = self.get_db()
db.put_state(state)
def check_update_source_state(self):
live_info = local_path_changes(
self.staged_path, self.source_state)
if live_info is not None:
def check_update_source_state(self, live_info):
if not is_info_eq(live_info, self.source_state.info):
msg = messaging.LiveInfoUpdateMessage(
archive=self.SIGNATURE, objname=self.objname,
info=live_info, logger=logger)
......@@ -461,10 +492,6 @@ class LocalfsSourceHandle(object):
def get_synced_state(self):
return self.source_state
def needs_staging(self):
info = self.source_state.info
return info and info[LOCALFS_TYPE] == common.T_FILE
def info_is_dir(self):
try:
return self.source_state.info[LOCALFS_TYPE] == common.T_DIR
......@@ -488,11 +515,7 @@ class LocalfsSourceHandle(object):
if self.stage_filename is None:
return
staged_path = self.staged_path
try:
link_file(staged_path, self.fspath)
os.unlink(staged_path)
except common.ConflictError:
self.stash_staged_file()
self.unregister_stage_name(self.stage_filename)
......
......@@ -14,7 +14,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from agkyra.syncer.setup import SyncerSettings
from agkyra.syncer.localfs_client import LocalfsFileClient, LocalfsTargetHandle
from agkyra.syncer import localfs_client
from agkyra.syncer.pithos_client import PithosFileClient
from agkyra.syncer.syncer import FileSyncer
import agkyra.syncer.syncer
......@@ -88,7 +88,7 @@ class AgkyraTest(unittest.TestCase):
ignore_ssl=True)
cls.master = PithosFileClient(cls.settings)
cls.slave = LocalfsFileClient(cls.settings)
cls.slave = localfs_client.LocalfsFileClient(cls.settings)
cls.s = FileSyncer(cls.settings, cls.master, cls.slave)
cls.pithos = cls.master.endpoint
cls.pithos.create_container(cls.ID)
......@@ -618,7 +618,7 @@ class AgkyraTest(unittest.TestCase):
f.write("content")
state = self.db.get_state(self.s.SLAVE, fil)
handle = LocalfsTargetHandle(self.s.settings, state)
handle = localfs_client.LocalfsTargetHandle(self.s.settings, state)
hidden_filename = utils.join_path(
handle.cache_hide_name, utils.hash_string(handle.objname))
hidden_path = handle.get_path_in_cache(hidden_filename)
......@@ -687,6 +687,99 @@ class AgkyraTest(unittest.TestCase):
self.assert_message(messaging.CollisionMessage)
self.assert_message(messaging.SyncErrorMessage)
def test_014_staging(self):
fil = "f014"
d = "d014"
fln = "f014.link"
f_path = self.get_path(fil)
with open(f_path, "w") as f:
f.write("content")
fln_path = self.get_path(fln)
os.symlink(f_path, fln_path)
d_path = self.get_path(d)
os.mkdir(d_path)
self.s.probe_file(self.s.SLAVE, fil)
self.assert_message(messaging.UpdateMessage)
state = self.db.get_state(self.s.SLAVE, fil)
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
staged_path = handle.staged_path
self.assertTrue(localfs_client.files_equal(f_path, staged_path))
handle.unstage_file()
self.assertFalse(os.path.exists(staged_path))
with open(f_path, "w") as f:
f.write("content new")
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
self.assert_message(messaging.LiveInfoUpdateMessage)
self.assertTrue(localfs_client.files_equal(f_path, staged_path))
handle.unstage_file()
f = open(f_path, "r")
with self.assertRaises(common.OpenBusyError):
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
ftmp_path = self.get_path("f014tmp")
with open(ftmp_path, "w") as f:
f.write("tmp")
os.unlink(f_path)
os.symlink(ftmp_path, f_path)
state = self.db.get_state(self.s.SLAVE, fil)
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
self.assert_message(messaging.LiveInfoUpdateMessage)
self.assertIsNone(handle.staged_path)
self.s.probe_file(self.s.SLAVE, fln)
self.assert_message(messaging.UpdateMessage)
state = self.db.get_state(self.s.SLAVE, fln)
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
self.assertIsNone(handle.staged_path)
os.unlink(fln_path)
with open(fln_path, "w") as f:
f.write("reg file")
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
self.assertIsNone(handle.staged_path)
# try to stage now
handle.stage_file()
self.assert_message(messaging.LiveInfoUpdateMessage)
self.assertTrue(localfs_client.files_equal(
fln_path, handle.staged_path))
handle.unstage_file()
fmissing = "fmissing014"
fmissing_path = self.get_path(fmissing)
self.s.probe_file(self.s.SLAVE, fmissing)
state = self.db.get_state(self.s.SLAVE, fmissing)
handle = localfs_client.LocalfsSourceHandle(self.s.settings, state)
self.assertIsNone(handle.staged_path)
with open(fmissing_path, "w") as f:
f.write("ref file")
handle.copy_file()
self.assertIsNotNone(handle.staged_path)
live_info = localfs_client.get_live_info(handle.fspath)
handle.check_staged(live_info)
with open(fmissing_path, "w") as f:
f.write("ref file2")
with self.assertRaises(common.ChangedBusyError):
handle.check_staged(live_info)
# turn it into a dir
os.unlink(fmissing_path)
os.mkdir(fmissing_path)
handle.copy_file()
with self.assertRaises(common.NotStableBusyError):
handle.check_staged(live_info)
# info of dir
live_info = localfs_client.get_live_info(handle.fspath)
handle.check_staged(live_info)
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment