Commit 9d1b963f authored by Michael Hanselmann's avatar Michael Hanselmann

utils: Move code related to file locking into separate file

Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent 7831fc5f
......@@ -214,6 +214,7 @@ server_PYTHON = \
utils_PYTHON = \
lib/utils/__init__.py \
lib/utils/algo.py \
lib/utils/filelock.py \
lib/utils/hash.py \
lib/utils/log.py \
lib/utils/mlock.py \
......@@ -486,6 +487,7 @@ python_tests = \
test/ganeti.ssh_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils.algo_unittest.py \
test/ganeti.utils.filelock_unittest.py \
test/ganeti.utils.hash_unittest.py \
test/ganeti.utils.mlock_unittest.py \
test/ganeti.utils.retry_unittest.py \
......
......@@ -60,6 +60,7 @@ from ganeti.utils.mlock import * # pylint: disable-msg=W0401
from ganeti.utils.log import * # pylint: disable-msg=W0401
from ganeti.utils.hash import * # pylint: disable-msg=W0401
from ganeti.utils.wrapper import * # pylint: disable-msg=W0401
from ganeti.utils.filelock import * # pylint: disable-msg=W0401
#: when set to True, L{RunCmd} is disabled
......@@ -2549,21 +2550,6 @@ def RunInSeparateProcess(fn, *args):
return bool(exitcode)
def LockFile(fd):
"""Locks a file using POSIX locks.
@type fd: int
@param fd: the file descriptor we need to lock
"""
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError, err:
if err.errno == errno.EAGAIN:
raise errors.LockError("File already locked")
raise
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
"""Reads the watcher pause file.
......@@ -2658,139 +2644,6 @@ def GenerateSelfSignedSslCert(filename, common_name=constants.X509_CERT_CN,
WriteFile(filename, mode=0400, data=key_pem + cert_pem)
class FileLock(object):
"""Utility class for file locks.
"""
def __init__(self, fd, filename):
"""Constructor for FileLock.
@type fd: file
@param fd: File object
@type filename: str
@param filename: Path of the file opened at I{fd}
"""
self.fd = fd
self.filename = filename
@classmethod
def Open(cls, filename):
"""Creates and opens a file to be used as a file-based lock.
@type filename: string
@param filename: path to the file to be locked
"""
# Using "os.open" is necessary to allow both opening existing file
# read/write and creating if not existing. Vanilla "open" will truncate an
# existing file -or- allow creating if not existing.
return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
filename)
def __del__(self):
self.Close()
def Close(self):
"""Close the file and release the lock.
"""
if hasattr(self, "fd") and self.fd:
self.fd.close()
self.fd = None
def _flock(self, flag, blocking, timeout, errmsg):
"""Wrapper for fcntl.flock.
@type flag: int
@param flag: operation flag
@type blocking: bool
@param blocking: whether the operation should be done in blocking mode.
@type timeout: None or float
@param timeout: for how long the operation should be retried (implies
non-blocking mode).
@type errmsg: string
@param errmsg: error message in case operation fails.
"""
assert self.fd, "Lock was closed"
assert timeout is None or timeout >= 0, \
"If specified, timeout must be positive"
assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
# When a timeout is used, LOCK_NB must always be set
if not (timeout is None and blocking):
flag |= fcntl.LOCK_NB
if timeout is None:
self._Lock(self.fd, flag, timeout)
else:
try:
Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
args=(self.fd, flag, timeout))
except RetryTimeout:
raise errors.LockError(errmsg)
@staticmethod
def _Lock(fd, flag, timeout):
try:
fcntl.flock(fd, flag)
except IOError, err:
if timeout is not None and err.errno == errno.EAGAIN:
raise RetryAgain()
logging.exception("fcntl.flock failed")
raise
def Exclusive(self, blocking=False, timeout=None):
"""Locks the file in exclusive mode.
@type blocking: boolean
@param blocking: whether to block and wait until we
can lock the file or return immediately
@type timeout: int or None
@param timeout: if not None, the duration to wait for the lock
(in blocking mode)
"""
self._flock(fcntl.LOCK_EX, blocking, timeout,
"Failed to lock %s in exclusive mode" % self.filename)
def Shared(self, blocking=False, timeout=None):
"""Locks the file in shared mode.
@type blocking: boolean
@param blocking: whether to block and wait until we
can lock the file or return immediately
@type timeout: int or None
@param timeout: if not None, the duration to wait for the lock
(in blocking mode)
"""
self._flock(fcntl.LOCK_SH, blocking, timeout,
"Failed to lock %s in shared mode" % self.filename)
def Unlock(self, blocking=True, timeout=None):
"""Unlocks the file.
According to C{flock(2)}, unlocking can also be a nonblocking
operation::
To make a non-blocking request, include LOCK_NB with any of the above
operations.
@type blocking: boolean
@param blocking: whether to block and wait until we
can lock the file or return immediately
@type timeout: int or None
@param timeout: if not None, the duration to wait for the lock
(in blocking mode)
"""
self._flock(fcntl.LOCK_UN, blocking, timeout,
"Failed to unlock %s" % self.filename)
def SignalHandled(signums):
"""Signal Handled decoration.
......
#
#
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Utility functions for file-based locks.
"""
import fcntl
import errno
import os
import logging
from ganeti import errors
from ganeti.utils import retry
def LockFile(fd):
"""Locks a file using POSIX locks.
@type fd: int
@param fd: the file descriptor we need to lock
"""
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError, err:
if err.errno == errno.EAGAIN:
raise errors.LockError("File already locked")
raise
class FileLock(object):
"""Utility class for file locks.
"""
def __init__(self, fd, filename):
"""Constructor for FileLock.
@type fd: file
@param fd: File object
@type filename: str
@param filename: Path of the file opened at I{fd}
"""
self.fd = fd
self.filename = filename
@classmethod
def Open(cls, filename):
"""Creates and opens a file to be used as a file-based lock.
@type filename: string
@param filename: path to the file to be locked
"""
# Using "os.open" is necessary to allow both opening existing file
# read/write and creating if not existing. Vanilla "open" will truncate an
# existing file -or- allow creating if not existing.
return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
filename)
def __del__(self):
self.Close()
def Close(self):
"""Close the file and release the lock.
"""
if hasattr(self, "fd") and self.fd:
self.fd.close()
self.fd = None
def _flock(self, flag, blocking, timeout, errmsg):
"""Wrapper for fcntl.flock.
@type flag: int
@param flag: operation flag
@type blocking: bool
@param blocking: whether the operation should be done in blocking mode.
@type timeout: None or float
@param timeout: for how long the operation should be retried (implies
non-blocking mode).
@type errmsg: string
@param errmsg: error message in case operation fails.
"""
assert self.fd, "Lock was closed"
assert timeout is None or timeout >= 0, \
"If specified, timeout must be positive"
assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
# When a timeout is used, LOCK_NB must always be set
if not (timeout is None and blocking):
flag |= fcntl.LOCK_NB
if timeout is None:
self._Lock(self.fd, flag, timeout)
else:
try:
retry.Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
args=(self.fd, flag, timeout))
except retry.RetryTimeout:
raise errors.LockError(errmsg)
@staticmethod
def _Lock(fd, flag, timeout):
try:
fcntl.flock(fd, flag)
except IOError, err:
if timeout is not None and err.errno == errno.EAGAIN:
raise retry.RetryAgain()
logging.exception("fcntl.flock failed")
raise
def Exclusive(self, blocking=False, timeout=None):
"""Locks the file in exclusive mode.
@type blocking: boolean
@param blocking: whether to block and wait until we
can lock the file or return immediately
@type timeout: int or None
@param timeout: if not None, the duration to wait for the lock
(in blocking mode)
"""
self._flock(fcntl.LOCK_EX, blocking, timeout,
"Failed to lock %s in exclusive mode" % self.filename)
def Shared(self, blocking=False, timeout=None):
"""Locks the file in shared mode.
@type blocking: boolean
@param blocking: whether to block and wait until we
can lock the file or return immediately
@type timeout: int or None
@param timeout: if not None, the duration to wait for the lock
(in blocking mode)
"""
self._flock(fcntl.LOCK_SH, blocking, timeout,
"Failed to lock %s in shared mode" % self.filename)
def Unlock(self, blocking=True, timeout=None):
"""Unlocks the file.
According to C{flock(2)}, unlocking can also be a nonblocking
operation::
To make a non-blocking request, include LOCK_NB with any of the above
operations.
@type blocking: boolean
@param blocking: whether to block and wait until we
can lock the file or return immediately
@type timeout: int or None
@param timeout: if not None, the duration to wait for the lock
(in blocking mode)
"""
self._flock(fcntl.LOCK_UN, blocking, timeout,
"Failed to unlock %s" % self.filename)
#!/usr/bin/python
#
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Script for testing ganeti.utils.filelock"""
import os
import tempfile
import unittest
from ganeti import constants
from ganeti import utils
from ganeti import errors
import testutils
class _BaseFileLockTest:
"""Test case for the FileLock class"""
def testSharedNonblocking(self):
self.lock.Shared(blocking=False)
self.lock.Close()
def testExclusiveNonblocking(self):
self.lock.Exclusive(blocking=False)
self.lock.Close()
def testUnlockNonblocking(self):
self.lock.Unlock(blocking=False)
self.lock.Close()
def testSharedBlocking(self):
self.lock.Shared(blocking=True)
self.lock.Close()
def testExclusiveBlocking(self):
self.lock.Exclusive(blocking=True)
self.lock.Close()
def testUnlockBlocking(self):
self.lock.Unlock(blocking=True)
self.lock.Close()
def testSharedExclusiveUnlock(self):
self.lock.Shared(blocking=False)
self.lock.Exclusive(blocking=False)
self.lock.Unlock(blocking=False)
self.lock.Close()
def testExclusiveSharedUnlock(self):
self.lock.Exclusive(blocking=False)
self.lock.Shared(blocking=False)
self.lock.Unlock(blocking=False)
self.lock.Close()
def testSimpleTimeout(self):
# These will succeed on the first attempt, hence a short timeout
self.lock.Shared(blocking=True, timeout=10.0)
self.lock.Exclusive(blocking=False, timeout=10.0)
self.lock.Unlock(blocking=True, timeout=10.0)
self.lock.Close()
@staticmethod
def _TryLockInner(filename, shared, blocking):
lock = utils.FileLock.Open(filename)
if shared:
fn = lock.Shared
else:
fn = lock.Exclusive
try:
# The timeout doesn't really matter as the parent process waits for us to
# finish anyway.
fn(blocking=blocking, timeout=0.01)
except errors.LockError, err:
return False
return True
def _TryLock(self, *args):
return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name,
*args)
def testTimeout(self):
for blocking in [True, False]:
self.lock.Exclusive(blocking=True)
self.failIf(self._TryLock(False, blocking))
self.failIf(self._TryLock(True, blocking))
self.lock.Shared(blocking=True)
self.assert_(self._TryLock(True, blocking))
self.failIf(self._TryLock(False, blocking))
def testCloseShared(self):
self.lock.Close()
self.assertRaises(AssertionError, self.lock.Shared, blocking=False)
def testCloseExclusive(self):
self.lock.Close()
self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False)
def testCloseUnlock(self):
self.lock.Close()
self.assertRaises(AssertionError, self.lock.Unlock, blocking=False)
class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest):
TESTDATA = "Hello World\n" * 10
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.tmpfile = tempfile.NamedTemporaryFile()
utils.WriteFile(self.tmpfile.name, data=self.TESTDATA)
self.lock = utils.FileLock.Open(self.tmpfile.name)
# Ensure "Open" didn't truncate file
self.assertFileContent(self.tmpfile.name, self.TESTDATA)
def tearDown(self):
self.assertFileContent(self.tmpfile.name, self.TESTDATA)
testutils.GanetiTestCase.tearDown(self)
class TestFileLockWithFileObject(unittest.TestCase, _BaseFileLockTest):
def setUp(self):
self.tmpfile = tempfile.NamedTemporaryFile()
self.lock = utils.FileLock(open(self.tmpfile.name, "w"), self.tmpfile.name)
if __name__ == "__main__":
testutils.GanetiTestProgram()
......@@ -1165,122 +1165,6 @@ class TestTailFile(testutils.GanetiTestCase):
self.failUnlessEqual(TailFile(fname, lines=i), data[-i:])
class _BaseFileLockTest:
"""Test case for the FileLock class"""
def testSharedNonblocking(self):
self.lock.Shared(blocking=False)
self.lock.Close()
def testExclusiveNonblocking(self):
self.lock.Exclusive(blocking=False)
self.lock.Close()
def testUnlockNonblocking(self):
self.lock.Unlock(blocking=False)
self.lock.Close()
def testSharedBlocking(self):
self.lock.Shared(blocking=True)
self.lock.Close()
def testExclusiveBlocking(self):
self.lock.Exclusive(blocking=True)
self.lock.Close()
def testUnlockBlocking(self):
self.lock.Unlock(blocking=True)
self.lock.Close()
def testSharedExclusiveUnlock(self):
self.lock.Shared(blocking=False)
self.lock.Exclusive(blocking=False)
self.lock.Unlock(blocking=False)
self.lock.Close()
def testExclusiveSharedUnlock(self):
self.lock.Exclusive(blocking=False)
self.lock.Shared(blocking=False)
self.lock.Unlock(blocking=False)
self.lock.Close()
def testSimpleTimeout(self):
# These will succeed on the first attempt, hence a short timeout
self.lock.Shared(blocking=True, timeout=10.0)
self.lock.Exclusive(blocking=False, timeout=10.0)
self.lock.Unlock(blocking=True, timeout=10.0)
self.lock.Close()
@staticmethod
def _TryLockInner(filename, shared, blocking):
lock = utils.FileLock.Open(filename)
if shared:
fn = lock.Shared
else:
fn = lock.Exclusive
try:
# The timeout doesn't really matter as the parent process waits for us to
# finish anyway.
fn(blocking=blocking, timeout=0.01)
except errors.LockError, err:
return False
return True
def _TryLock(self, *args):
return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name,
*args)
def testTimeout(self):
for blocking in [True, False]:
self.lock.Exclusive(blocking=True)
self.failIf(self._TryLock(False, blocking))
self.failIf(self._TryLock(True, blocking))
self.lock.Shared(blocking=True)
self.assert_(self._TryLock(True, blocking))
self.failIf(self._TryLock(False, blocking))
def testCloseShared(self):
self.lock.Close()
self.assertRaises(AssertionError, self.lock.Shared, blocking=False)
def testCloseExclusive(self):
self.lock.Close()
self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False)
def testCloseUnlock(self):
self.lock.Close()
self.assertRaises(AssertionError, self.lock.Unlock, blocking=False)
class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest):
TESTDATA = "Hello World\n" * 10
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.tmpfile = tempfile.NamedTemporaryFile()
utils.WriteFile(self.tmpfile.name, data=self.TESTDATA)
self.lock = utils.FileLock.Open(self.tmpfile.name)
# Ensure "Open" didn't truncate file
self.assertFileContent(self.tmpfile.name, self.TESTDATA)
def tearDown(self):
self.assertFileContent(self.tmpfile.name, self.TESTDATA)
testutils.GanetiTestCase.tearDown(self)
class TestFileLockWithFileObject(unittest.TestCase, _BaseFileLockTest):
def setUp(self):
self.tmpfile = tempfile.NamedTemporaryFile()
self.lock = utils.FileLock(open(self.tmpfile.name, "w"), self.tmpfile.name)
class TestTimeFunctions(unittest.TestCase):
"""Test case for time functions"""
......