From 9d1b963f9ae6362dc9242bdb23627d979175dc8a Mon Sep 17 00:00:00 2001
From: Michael Hanselmann <hansmi@google.com>
Date: Mon, 10 Jan 2011 17:58:52 +0100
Subject: [PATCH] utils: Move code related to file locking into separate file
Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>
---
Makefile.am | 2 +
lib/utils/__init__.py | 149 +-------------------
lib/utils/filelock.py | 179 +++++++++++++++++++++++++
test/ganeti.utils.filelock_unittest.py | 152 +++++++++++++++++++++
test/ganeti.utils_unittest.py | 116 ----------------
5 files changed, 334 insertions(+), 264 deletions(-)
create mode 100644 lib/utils/filelock.py
create mode 100755 test/ganeti.utils.filelock_unittest.py
diff --git a/Makefile.am b/Makefile.am
index bbec35280..7c44969db 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -214,6 +214,7 @@ server_PYTHON = \
utils_PYTHON = \
lib/utils/__init__.py \
lib/utils/algo.py \
+ lib/utils/filelock.py \
lib/utils/hash.py \
lib/utils/log.py \
lib/utils/mlock.py \
@@ -486,6 +487,7 @@ python_tests = \
test/ganeti.ssh_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils.algo_unittest.py \
+ test/ganeti.utils.filelock_unittest.py \
test/ganeti.utils.hash_unittest.py \
test/ganeti.utils.mlock_unittest.py \
test/ganeti.utils.retry_unittest.py \
diff --git a/lib/utils/__init__.py b/lib/utils/__init__.py
index eb8d1b028..a7b603341 100644
--- a/lib/utils/__init__.py
+++ b/lib/utils/__init__.py
@@ -60,6 +60,7 @@ from ganeti.utils.mlock import * # pylint: disable-msg=W0401
from ganeti.utils.log import * # pylint: disable-msg=W0401
from ganeti.utils.hash import * # pylint: disable-msg=W0401
from ganeti.utils.wrapper import * # pylint: disable-msg=W0401
+from ganeti.utils.filelock import * # pylint: disable-msg=W0401
#: when set to True, L{RunCmd} is disabled
@@ -2549,21 +2550,6 @@ def RunInSeparateProcess(fn, *args):
return bool(exitcode)
-def LockFile(fd):
- """Locks a file using POSIX locks.
-
- @type fd: int
- @param fd: the file descriptor we need to lock
-
- """
- try:
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError, err:
- if err.errno == errno.EAGAIN:
- raise errors.LockError("File already locked")
- raise
-
-
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
"""Reads the watcher pause file.
@@ -2658,139 +2644,6 @@ def GenerateSelfSignedSslCert(filename, common_name=constants.X509_CERT_CN,
WriteFile(filename, mode=0400, data=key_pem + cert_pem)
-class FileLock(object):
- """Utility class for file locks.
-
- """
- def __init__(self, fd, filename):
- """Constructor for FileLock.
-
- @type fd: file
- @param fd: File object
- @type filename: str
- @param filename: Path of the file opened at I{fd}
-
- """
- self.fd = fd
- self.filename = filename
-
- @classmethod
- def Open(cls, filename):
- """Creates and opens a file to be used as a file-based lock.
-
- @type filename: string
- @param filename: path to the file to be locked
-
- """
- # Using "os.open" is necessary to allow both opening existing file
- # read/write and creating if not existing. Vanilla "open" will truncate an
- # existing file -or- allow creating if not existing.
- return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
- filename)
-
- def __del__(self):
- self.Close()
-
- def Close(self):
- """Close the file and release the lock.
-
- """
- if hasattr(self, "fd") and self.fd:
- self.fd.close()
- self.fd = None
-
- def _flock(self, flag, blocking, timeout, errmsg):
- """Wrapper for fcntl.flock.
-
- @type flag: int
- @param flag: operation flag
- @type blocking: bool
- @param blocking: whether the operation should be done in blocking mode.
- @type timeout: None or float
- @param timeout: for how long the operation should be retried (implies
- non-blocking mode).
- @type errmsg: string
- @param errmsg: error message in case operation fails.
-
- """
- assert self.fd, "Lock was closed"
- assert timeout is None or timeout >= 0, \
- "If specified, timeout must be positive"
- assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
-
- # When a timeout is used, LOCK_NB must always be set
- if not (timeout is None and blocking):
- flag |= fcntl.LOCK_NB
-
- if timeout is None:
- self._Lock(self.fd, flag, timeout)
- else:
- try:
- Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
- args=(self.fd, flag, timeout))
- except RetryTimeout:
- raise errors.LockError(errmsg)
-
- @staticmethod
- def _Lock(fd, flag, timeout):
- try:
- fcntl.flock(fd, flag)
- except IOError, err:
- if timeout is not None and err.errno == errno.EAGAIN:
- raise RetryAgain()
-
- logging.exception("fcntl.flock failed")
- raise
-
- def Exclusive(self, blocking=False, timeout=None):
- """Locks the file in exclusive mode.
-
- @type blocking: boolean
- @param blocking: whether to block and wait until we
- can lock the file or return immediately
- @type timeout: int or None
- @param timeout: if not None, the duration to wait for the lock
- (in blocking mode)
-
- """
- self._flock(fcntl.LOCK_EX, blocking, timeout,
- "Failed to lock %s in exclusive mode" % self.filename)
-
- def Shared(self, blocking=False, timeout=None):
- """Locks the file in shared mode.
-
- @type blocking: boolean
- @param blocking: whether to block and wait until we
- can lock the file or return immediately
- @type timeout: int or None
- @param timeout: if not None, the duration to wait for the lock
- (in blocking mode)
-
- """
- self._flock(fcntl.LOCK_SH, blocking, timeout,
- "Failed to lock %s in shared mode" % self.filename)
-
- def Unlock(self, blocking=True, timeout=None):
- """Unlocks the file.
-
- According to C{flock(2)}, unlocking can also be a nonblocking
- operation::
-
- To make a non-blocking request, include LOCK_NB with any of the above
- operations.
-
- @type blocking: boolean
- @param blocking: whether to block and wait until we
- can lock the file or return immediately
- @type timeout: int or None
- @param timeout: if not None, the duration to wait for the lock
- (in blocking mode)
-
- """
- self._flock(fcntl.LOCK_UN, blocking, timeout,
- "Failed to unlock %s" % self.filename)
-
-
def SignalHandled(signums):
"""Signal Handled decoration.
diff --git a/lib/utils/filelock.py b/lib/utils/filelock.py
new file mode 100644
index 000000000..58b32bcff
--- /dev/null
+++ b/lib/utils/filelock.py
@@ -0,0 +1,179 @@
+#
+#
+
+# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Utility functions for file-based locks.
+
+"""
+
+import fcntl
+import errno
+import os
+import logging
+
+from ganeti import errors
+from ganeti.utils import retry
+
+
+def LockFile(fd):
+ """Locks a file using POSIX locks.
+
+ @type fd: int
+ @param fd: the file descriptor we need to lock
+
+ """
+ try:
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError, err:
+ if err.errno == errno.EAGAIN:
+ raise errors.LockError("File already locked")
+ raise
+
+
+class FileLock(object):
+ """Utility class for file locks.
+
+ """
+ def __init__(self, fd, filename):
+ """Constructor for FileLock.
+
+ @type fd: file
+ @param fd: File object
+ @type filename: str
+ @param filename: Path of the file opened at I{fd}
+
+ """
+ self.fd = fd
+ self.filename = filename
+
+ @classmethod
+ def Open(cls, filename):
+ """Creates and opens a file to be used as a file-based lock.
+
+ @type filename: string
+ @param filename: path to the file to be locked
+
+ """
+ # Using "os.open" is necessary to allow both opening existing file
+ # read/write and creating if not existing. Vanilla "open" will truncate an
+ # existing file -or- allow creating if not existing.
+ return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
+ filename)
+
+ def __del__(self):
+ self.Close()
+
+ def Close(self):
+ """Close the file and release the lock.
+
+ """
+ if hasattr(self, "fd") and self.fd:
+ self.fd.close()
+ self.fd = None
+
+ def _flock(self, flag, blocking, timeout, errmsg):
+ """Wrapper for fcntl.flock.
+
+ @type flag: int
+ @param flag: operation flag
+ @type blocking: bool
+ @param blocking: whether the operation should be done in blocking mode.
+ @type timeout: None or float
+ @param timeout: for how long the operation should be retried (implies
+ non-blocking mode).
+ @type errmsg: string
+ @param errmsg: error message in case operation fails.
+
+ """
+ assert self.fd, "Lock was closed"
+ assert timeout is None or timeout >= 0, \
+ "If specified, timeout must be positive"
+ assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
+
+ # When a timeout is used, LOCK_NB must always be set
+ if not (timeout is None and blocking):
+ flag |= fcntl.LOCK_NB
+
+ if timeout is None:
+ self._Lock(self.fd, flag, timeout)
+ else:
+ try:
+ retry.Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
+ args=(self.fd, flag, timeout))
+ except retry.RetryTimeout:
+ raise errors.LockError(errmsg)
+
+ @staticmethod
+ def _Lock(fd, flag, timeout):
+ try:
+ fcntl.flock(fd, flag)
+ except IOError, err:
+ if timeout is not None and err.errno == errno.EAGAIN:
+ raise retry.RetryAgain()
+
+ logging.exception("fcntl.flock failed")
+ raise
+
+ def Exclusive(self, blocking=False, timeout=None):
+ """Locks the file in exclusive mode.
+
+ @type blocking: boolean
+ @param blocking: whether to block and wait until we
+ can lock the file or return immediately
+ @type timeout: int or None
+ @param timeout: if not None, the duration to wait for the lock
+ (in blocking mode)
+
+ """
+ self._flock(fcntl.LOCK_EX, blocking, timeout,
+ "Failed to lock %s in exclusive mode" % self.filename)
+
+ def Shared(self, blocking=False, timeout=None):
+ """Locks the file in shared mode.
+
+ @type blocking: boolean
+ @param blocking: whether to block and wait until we
+ can lock the file or return immediately
+ @type timeout: int or None
+ @param timeout: if not None, the duration to wait for the lock
+ (in blocking mode)
+
+ """
+ self._flock(fcntl.LOCK_SH, blocking, timeout,
+ "Failed to lock %s in shared mode" % self.filename)
+
+ def Unlock(self, blocking=True, timeout=None):
+ """Unlocks the file.
+
+ According to C{flock(2)}, unlocking can also be a nonblocking
+ operation::
+
+ To make a non-blocking request, include LOCK_NB with any of the above
+ operations.
+
+ @type blocking: boolean
+ @param blocking: whether to block and wait until we
+ can lock the file or return immediately
+ @type timeout: int or None
+ @param timeout: if not None, the duration to wait for the lock
+ (in blocking mode)
+
+ """
+ self._flock(fcntl.LOCK_UN, blocking, timeout,
+ "Failed to unlock %s" % self.filename)
diff --git a/test/ganeti.utils.filelock_unittest.py b/test/ganeti.utils.filelock_unittest.py
new file mode 100755
index 000000000..29e7f6446
--- /dev/null
+++ b/test/ganeti.utils.filelock_unittest.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python
+#
+
+# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for testing ganeti.utils.filelock"""
+
+import os
+import tempfile
+import unittest
+
+from ganeti import constants
+from ganeti import utils
+from ganeti import errors
+
+import testutils
+
+
+class _BaseFileLockTest:
+ """Test case for the FileLock class"""
+
+ def testSharedNonblocking(self):
+ self.lock.Shared(blocking=False)
+ self.lock.Close()
+
+ def testExclusiveNonblocking(self):
+ self.lock.Exclusive(blocking=False)
+ self.lock.Close()
+
+ def testUnlockNonblocking(self):
+ self.lock.Unlock(blocking=False)
+ self.lock.Close()
+
+ def testSharedBlocking(self):
+ self.lock.Shared(blocking=True)
+ self.lock.Close()
+
+ def testExclusiveBlocking(self):
+ self.lock.Exclusive(blocking=True)
+ self.lock.Close()
+
+ def testUnlockBlocking(self):
+ self.lock.Unlock(blocking=True)
+ self.lock.Close()
+
+ def testSharedExclusiveUnlock(self):
+ self.lock.Shared(blocking=False)
+ self.lock.Exclusive(blocking=False)
+ self.lock.Unlock(blocking=False)
+ self.lock.Close()
+
+ def testExclusiveSharedUnlock(self):
+ self.lock.Exclusive(blocking=False)
+ self.lock.Shared(blocking=False)
+ self.lock.Unlock(blocking=False)
+ self.lock.Close()
+
+ def testSimpleTimeout(self):
+ # These will succeed on the first attempt, hence a short timeout
+ self.lock.Shared(blocking=True, timeout=10.0)
+ self.lock.Exclusive(blocking=False, timeout=10.0)
+ self.lock.Unlock(blocking=True, timeout=10.0)
+ self.lock.Close()
+
+ @staticmethod
+ def _TryLockInner(filename, shared, blocking):
+ lock = utils.FileLock.Open(filename)
+
+ if shared:
+ fn = lock.Shared
+ else:
+ fn = lock.Exclusive
+
+ try:
+ # The timeout doesn't really matter as the parent process waits for us to
+ # finish anyway.
+ fn(blocking=blocking, timeout=0.01)
+ except errors.LockError, err:
+ return False
+
+ return True
+
+ def _TryLock(self, *args):
+ return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name,
+ *args)
+
+ def testTimeout(self):
+ for blocking in [True, False]:
+ self.lock.Exclusive(blocking=True)
+ self.failIf(self._TryLock(False, blocking))
+ self.failIf(self._TryLock(True, blocking))
+
+ self.lock.Shared(blocking=True)
+ self.assert_(self._TryLock(True, blocking))
+ self.failIf(self._TryLock(False, blocking))
+
+ def testCloseShared(self):
+ self.lock.Close()
+ self.assertRaises(AssertionError, self.lock.Shared, blocking=False)
+
+ def testCloseExclusive(self):
+ self.lock.Close()
+ self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False)
+
+ def testCloseUnlock(self):
+ self.lock.Close()
+ self.assertRaises(AssertionError, self.lock.Unlock, blocking=False)
+
+
+class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest):
+ TESTDATA = "Hello World\n" * 10
+
+ def setUp(self):
+ testutils.GanetiTestCase.setUp(self)
+
+ self.tmpfile = tempfile.NamedTemporaryFile()
+ utils.WriteFile(self.tmpfile.name, data=self.TESTDATA)
+ self.lock = utils.FileLock.Open(self.tmpfile.name)
+
+ # Ensure "Open" didn't truncate file
+ self.assertFileContent(self.tmpfile.name, self.TESTDATA)
+
+ def tearDown(self):
+ self.assertFileContent(self.tmpfile.name, self.TESTDATA)
+
+ testutils.GanetiTestCase.tearDown(self)
+
+
+class TestFileLockWithFileObject(unittest.TestCase, _BaseFileLockTest):
+ def setUp(self):
+ self.tmpfile = tempfile.NamedTemporaryFile()
+ self.lock = utils.FileLock(open(self.tmpfile.name, "w"), self.tmpfile.name)
+
+
+if __name__ == "__main__":
+ testutils.GanetiTestProgram()
diff --git a/test/ganeti.utils_unittest.py b/test/ganeti.utils_unittest.py
index baaca37be..e7b0298f8 100755
--- a/test/ganeti.utils_unittest.py
+++ b/test/ganeti.utils_unittest.py
@@ -1165,122 +1165,6 @@ class TestTailFile(testutils.GanetiTestCase):
self.failUnlessEqual(TailFile(fname, lines=i), data[-i:])
-class _BaseFileLockTest:
- """Test case for the FileLock class"""
-
- def testSharedNonblocking(self):
- self.lock.Shared(blocking=False)
- self.lock.Close()
-
- def testExclusiveNonblocking(self):
- self.lock.Exclusive(blocking=False)
- self.lock.Close()
-
- def testUnlockNonblocking(self):
- self.lock.Unlock(blocking=False)
- self.lock.Close()
-
- def testSharedBlocking(self):
- self.lock.Shared(blocking=True)
- self.lock.Close()
-
- def testExclusiveBlocking(self):
- self.lock.Exclusive(blocking=True)
- self.lock.Close()
-
- def testUnlockBlocking(self):
- self.lock.Unlock(blocking=True)
- self.lock.Close()
-
- def testSharedExclusiveUnlock(self):
- self.lock.Shared(blocking=False)
- self.lock.Exclusive(blocking=False)
- self.lock.Unlock(blocking=False)
- self.lock.Close()
-
- def testExclusiveSharedUnlock(self):
- self.lock.Exclusive(blocking=False)
- self.lock.Shared(blocking=False)
- self.lock.Unlock(blocking=False)
- self.lock.Close()
-
- def testSimpleTimeout(self):
- # These will succeed on the first attempt, hence a short timeout
- self.lock.Shared(blocking=True, timeout=10.0)
- self.lock.Exclusive(blocking=False, timeout=10.0)
- self.lock.Unlock(blocking=True, timeout=10.0)
- self.lock.Close()
-
- @staticmethod
- def _TryLockInner(filename, shared, blocking):
- lock = utils.FileLock.Open(filename)
-
- if shared:
- fn = lock.Shared
- else:
- fn = lock.Exclusive
-
- try:
- # The timeout doesn't really matter as the parent process waits for us to
- # finish anyway.
- fn(blocking=blocking, timeout=0.01)
- except errors.LockError, err:
- return False
-
- return True
-
- def _TryLock(self, *args):
- return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name,
- *args)
-
- def testTimeout(self):
- for blocking in [True, False]:
- self.lock.Exclusive(blocking=True)
- self.failIf(self._TryLock(False, blocking))
- self.failIf(self._TryLock(True, blocking))
-
- self.lock.Shared(blocking=True)
- self.assert_(self._TryLock(True, blocking))
- self.failIf(self._TryLock(False, blocking))
-
- def testCloseShared(self):
- self.lock.Close()
- self.assertRaises(AssertionError, self.lock.Shared, blocking=False)
-
- def testCloseExclusive(self):
- self.lock.Close()
- self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False)
-
- def testCloseUnlock(self):
- self.lock.Close()
- self.assertRaises(AssertionError, self.lock.Unlock, blocking=False)
-
-
-class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest):
- TESTDATA = "Hello World\n" * 10
-
- def setUp(self):
- testutils.GanetiTestCase.setUp(self)
-
- self.tmpfile = tempfile.NamedTemporaryFile()
- utils.WriteFile(self.tmpfile.name, data=self.TESTDATA)
- self.lock = utils.FileLock.Open(self.tmpfile.name)
-
- # Ensure "Open" didn't truncate file
- self.assertFileContent(self.tmpfile.name, self.TESTDATA)
-
- def tearDown(self):
- self.assertFileContent(self.tmpfile.name, self.TESTDATA)
-
- testutils.GanetiTestCase.tearDown(self)
-
-
-class TestFileLockWithFileObject(unittest.TestCase, _BaseFileLockTest):
- def setUp(self):
- self.tmpfile = tempfile.NamedTemporaryFile()
- self.lock = utils.FileLock(open(self.tmpfile.name, "w"), self.tmpfile.name)
-
-
class TestTimeFunctions(unittest.TestCase):
"""Test case for time functions"""
--
GitLab