utils: Move code related to file locking into separate file

Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>

Makefile.am

@@ -214,6 +214,7 @@ server_PYTHON = \
 utils_PYTHON = \
 	lib/utils/__init__.py \
 	lib/utils/algo.py \
+	lib/utils/filelock.py \
 	lib/utils/hash.py \
 	lib/utils/log.py \
 	lib/utils/mlock.py \
@@ -486,6 +487,7 @@ python_tests = \
 	test/ganeti.ssh_unittest.py \
 	test/ganeti.uidpool_unittest.py \
 	test/ganeti.utils.algo_unittest.py \
+	test/ganeti.utils.filelock_unittest.py \
 	test/ganeti.utils.hash_unittest.py \
 	test/ganeti.utils.mlock_unittest.py \
 	test/ganeti.utils.retry_unittest.py \

lib/utils/__init__.py

@@ -60,6 +60,7 @@ from ganeti.utils.mlock import * # pylint: disable-msg=W0401
 from ganeti.utils.log import * # pylint: disable-msg=W0401
 from ganeti.utils.hash import * # pylint: disable-msg=W0401
 from ganeti.utils.wrapper import * # pylint: disable-msg=W0401
+from ganeti.utils.filelock import * # pylint: disable-msg=W0401
 
 
 #: when set to True, L{RunCmd} is disabled
@@ -2549,21 +2550,6 @@ def RunInSeparateProcess(fn, *args):
   return bool(exitcode)
 
 
-def LockFile(fd):
-  """Locks a file using POSIX locks.
-
-  @type fd: int
-  @param fd: the file descriptor we need to lock
-
-  """
-  try:
-    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except IOError, err:
-    if err.errno == errno.EAGAIN:
-      raise errors.LockError("File already locked")
-    raise
-
-
 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
   """Reads the watcher pause file.
 
@@ -2658,139 +2644,6 @@ def GenerateSelfSignedSslCert(filename, common_name=constants.X509_CERT_CN,
   WriteFile(filename, mode=0400, data=key_pem + cert_pem)
 
 
-class FileLock(object):
-  """Utility class for file locks.
-
-  """
-  def __init__(self, fd, filename):
-    """Constructor for FileLock.
-
-    @type fd: file
-    @param fd: File object
-    @type filename: str
-    @param filename: Path of the file opened at I{fd}
-
-    """
-    self.fd = fd
-    self.filename = filename
-
-  @classmethod
-  def Open(cls, filename):
-    """Creates and opens a file to be used as a file-based lock.
-
-    @type filename: string
-    @param filename: path to the file to be locked
-
-    """
-    # Using "os.open" is necessary to allow both opening existing file
-    # read/write and creating if not existing. Vanilla "open" will truncate an
-    # existing file -or- allow creating if not existing.
-    return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
-               filename)
-
-  def __del__(self):
-    self.Close()
-
-  def Close(self):
-    """Close the file and release the lock.
-
-    """
-    if hasattr(self, "fd") and self.fd:
-      self.fd.close()
-      self.fd = None
-
-  def _flock(self, flag, blocking, timeout, errmsg):
-    """Wrapper for fcntl.flock.
-
-    @type flag: int
-    @param flag: operation flag
-    @type blocking: bool
-    @param blocking: whether the operation should be done in blocking mode.
-    @type timeout: None or float
-    @param timeout: for how long the operation should be retried (implies
-                    non-blocking mode).
-    @type errmsg: string
-    @param errmsg: error message in case operation fails.
-
-    """
-    assert self.fd, "Lock was closed"
-    assert timeout is None or timeout >= 0, \
-      "If specified, timeout must be positive"
-    assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
-
-    # When a timeout is used, LOCK_NB must always be set
-    if not (timeout is None and blocking):
-      flag |= fcntl.LOCK_NB
-
-    if timeout is None:
-      self._Lock(self.fd, flag, timeout)
-    else:
-      try:
-        Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
-              args=(self.fd, flag, timeout))
-      except RetryTimeout:
-        raise errors.LockError(errmsg)
-
-  @staticmethod
-  def _Lock(fd, flag, timeout):
-    try:
-      fcntl.flock(fd, flag)
-    except IOError, err:
-      if timeout is not None and err.errno == errno.EAGAIN:
-        raise RetryAgain()
-
-      logging.exception("fcntl.flock failed")
-      raise
-
-  def Exclusive(self, blocking=False, timeout=None):
-    """Locks the file in exclusive mode.
-
-    @type blocking: boolean
-    @param blocking: whether to block and wait until we
-        can lock the file or return immediately
-    @type timeout: int or None
-    @param timeout: if not None, the duration to wait for the lock
-        (in blocking mode)
-
-    """
-    self._flock(fcntl.LOCK_EX, blocking, timeout,
-                "Failed to lock %s in exclusive mode" % self.filename)
-
-  def Shared(self, blocking=False, timeout=None):
-    """Locks the file in shared mode.
-
-    @type blocking: boolean
-    @param blocking: whether to block and wait until we
-        can lock the file or return immediately
-    @type timeout: int or None
-    @param timeout: if not None, the duration to wait for the lock
-        (in blocking mode)
-
-    """
-    self._flock(fcntl.LOCK_SH, blocking, timeout,
-                "Failed to lock %s in shared mode" % self.filename)
-
-  def Unlock(self, blocking=True, timeout=None):
-    """Unlocks the file.
-
-    According to C{flock(2)}, unlocking can also be a nonblocking
-    operation::
-
-      To make a non-blocking request, include LOCK_NB with any of the above
-      operations.
-
-    @type blocking: boolean
-    @param blocking: whether to block and wait until we
-        can lock the file or return immediately
-    @type timeout: int or None
-    @param timeout: if not None, the duration to wait for the lock
-        (in blocking mode)
-
-    """
-    self._flock(fcntl.LOCK_UN, blocking, timeout,
-                "Failed to unlock %s" % self.filename)
-
-
 def SignalHandled(signums):
   """Signal Handled decoration.
 

lib/utils/filelock.py

+#
+#
+
+# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Utility functions for file-based locks.
+
+"""
+
+import fcntl
+import errno
+import os
+import logging
+
+from ganeti import errors
+from ganeti.utils import retry
+
+
+def LockFile(fd):
+  """Locks a file using POSIX locks.
+
+  @type fd: int
+  @param fd: the file descriptor we need to lock
+
+  """
+  try:
+    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  except IOError, err:
+    if err.errno == errno.EAGAIN:
+      raise errors.LockError("File already locked")
+    raise
+
+
+class FileLock(object):
+  """Utility class for file locks.
+
+  """
+  def __init__(self, fd, filename):
+    """Constructor for FileLock.
+
+    @type fd: file
+    @param fd: File object
+    @type filename: str
+    @param filename: Path of the file opened at I{fd}
+
+    """
+    self.fd = fd
+    self.filename = filename
+
+  @classmethod
+  def Open(cls, filename):
+    """Creates and opens a file to be used as a file-based lock.
+
+    @type filename: string
+    @param filename: path to the file to be locked
+
+    """
+    # Using "os.open" is necessary to allow both opening existing file
+    # read/write and creating if not existing. Vanilla "open" will truncate an
+    # existing file -or- allow creating if not existing.
+    return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
+               filename)
+
+  def __del__(self):
+    self.Close()
+
+  def Close(self):
+    """Close the file and release the lock.
+
+    """
+    if hasattr(self, "fd") and self.fd:
+      self.fd.close()
+      self.fd = None
+
+  def _flock(self, flag, blocking, timeout, errmsg):
+    """Wrapper for fcntl.flock.
+
+    @type flag: int
+    @param flag: operation flag
+    @type blocking: bool
+    @param blocking: whether the operation should be done in blocking mode.
+    @type timeout: None or float
+    @param timeout: for how long the operation should be retried (implies
+                    non-blocking mode).
+    @type errmsg: string
+    @param errmsg: error message in case operation fails.
+
+    """
+    assert self.fd, "Lock was closed"
+    assert timeout is None or timeout >= 0, \
+      "If specified, timeout must be positive"
+    assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
+
+    # When a timeout is used, LOCK_NB must always be set
+    if not (timeout is None and blocking):
+      flag |= fcntl.LOCK_NB
+
+    if timeout is None:
+      self._Lock(self.fd, flag, timeout)
+    else:
+      try:
+        retry.Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
+                    args=(self.fd, flag, timeout))
+      except retry.RetryTimeout:
+        raise errors.LockError(errmsg)
+
+  @staticmethod
+  def _Lock(fd, flag, timeout):
+    try:
+      fcntl.flock(fd, flag)
+    except IOError, err:
+      if timeout is not None and err.errno == errno.EAGAIN:
+        raise retry.RetryAgain()
+
+      logging.exception("fcntl.flock failed")
+      raise
+
+  def Exclusive(self, blocking=False, timeout=None):
+    """Locks the file in exclusive mode.
+
+    @type blocking: boolean
+    @param blocking: whether to block and wait until we
+        can lock the file or return immediately
+    @type timeout: int or None
+    @param timeout: if not None, the duration to wait for the lock
+        (in blocking mode)
+
+    """
+    self._flock(fcntl.LOCK_EX, blocking, timeout,
+                "Failed to lock %s in exclusive mode" % self.filename)
+
+  def Shared(self, blocking=False, timeout=None):
+    """Locks the file in shared mode.
+
+    @type blocking: boolean
+    @param blocking: whether to block and wait until we
+        can lock the file or return immediately
+    @type timeout: int or None
+    @param timeout: if not None, the duration to wait for the lock
+        (in blocking mode)
+
+    """
+    self._flock(fcntl.LOCK_SH, blocking, timeout,
+                "Failed to lock %s in shared mode" % self.filename)
+
+  def Unlock(self, blocking=True, timeout=None):
+    """Unlocks the file.
+
+    According to C{flock(2)}, unlocking can also be a nonblocking
+    operation::
+
+      To make a non-blocking request, include LOCK_NB with any of the above
+      operations.
+
+    @type blocking: boolean
+    @param blocking: whether to block and wait until we
+        can lock the file or return immediately
+    @type timeout: int or None
+    @param timeout: if not None, the duration to wait for the lock
+        (in blocking mode)
+
+    """
+    self._flock(fcntl.LOCK_UN, blocking, timeout,
+                "Failed to unlock %s" % self.filename)

test/ganeti.utils.filelock_unittest.py

+#!/usr/bin/python
+#
+
+# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for testing ganeti.utils.filelock"""
+
+import os
+import tempfile
+import unittest
+
+from ganeti import constants
+from ganeti import utils
+from ganeti import errors
+
+import testutils
+
+
+class _BaseFileLockTest:
+  """Test case for the FileLock class"""
+
+  def testSharedNonblocking(self):
+    self.lock.Shared(blocking=False)
+    self.lock.Close()
+
+  def testExclusiveNonblocking(self):
+    self.lock.Exclusive(blocking=False)
+    self.lock.Close()
+
+  def testUnlockNonblocking(self):
+    self.lock.Unlock(blocking=False)
+    self.lock.Close()
+
+  def testSharedBlocking(self):
+    self.lock.Shared(blocking=True)
+    self.lock.Close()
+
+  def testExclusiveBlocking(self):
+    self.lock.Exclusive(blocking=True)
+    self.lock.Close()
+
+  def testUnlockBlocking(self):
+    self.lock.Unlock(blocking=True)
+    self.lock.Close()
+
+  def testSharedExclusiveUnlock(self):
+    self.lock.Shared(blocking=False)
+    self.lock.Exclusive(blocking=False)
+    self.lock.Unlock(blocking=False)
+    self.lock.Close()
+
+  def testExclusiveSharedUnlock(self):
+    self.lock.Exclusive(blocking=False)
+    self.lock.Shared(blocking=False)
+    self.lock.Unlock(blocking=False)
+    self.lock.Close()
+
+  def testSimpleTimeout(self):
+    # These will succeed on the first attempt, hence a short timeout
+    self.lock.Shared(blocking=True, timeout=10.0)
+    self.lock.Exclusive(blocking=False, timeout=10.0)
+    self.lock.Unlock(blocking=True, timeout=10.0)
+    self.lock.Close()
+
+  @staticmethod
+  def _TryLockInner(filename, shared, blocking):
+    lock = utils.FileLock.Open(filename)
+
+    if shared:
+      fn = lock.Shared
+    else:
+      fn = lock.Exclusive
+
+    try:
+      # The timeout doesn't really matter as the parent process waits for us to
+      # finish anyway.
+      fn(blocking=blocking, timeout=0.01)
+    except errors.LockError, err:
+      return False
+
+    return True
+
+  def _TryLock(self, *args):
+    return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name,
+                                      *args)
+
+  def testTimeout(self):
+    for blocking in [True, False]:
+      self.lock.Exclusive(blocking=True)
+      self.failIf(self._TryLock(False, blocking))
+      self.failIf(self._TryLock(True, blocking))
+
+      self.lock.Shared(blocking=True)
+      self.assert_(self._TryLock(True, blocking))
+      self.failIf(self._TryLock(False, blocking))
+
+  def testCloseShared(self):
+    self.lock.Close()
+    self.assertRaises(AssertionError, self.lock.Shared, blocking=False)
+
+  def testCloseExclusive(self):
+    self.lock.Close()
+    self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False)
+
+  def testCloseUnlock(self):
+    self.lock.Close()
+    self.assertRaises(AssertionError, self.lock.Unlock, blocking=False)
+
+
+class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest):
+  TESTDATA = "Hello World\n" * 10
+
+  def setUp(self):
+    testutils.GanetiTestCase.setUp(self)
+
+    self.tmpfile = tempfile.NamedTemporaryFile()
+    utils.WriteFile(self.tmpfile.name, data=self.TESTDATA)
+    self.lock = utils.FileLock.Open(self.tmpfile.name)
+
+    # Ensure "Open" didn't truncate file
+    self.assertFileContent(self.tmpfile.name, self.TESTDATA)
+
+  def tearDown(self):
+    self.assertFileContent(self.tmpfile.name, self.TESTDATA)
+
+    testutils.GanetiTestCase.tearDown(self)
+
+
+class TestFileLockWithFileObject(unittest.TestCase, _BaseFileLockTest):
+  def setUp(self):
+    self.tmpfile = tempfile.NamedTemporaryFile()
+    self.lock = utils.FileLock(open(self.tmpfile.name, "w"), self.tmpfile.name)
+
+
+if __name__ == "__main__":
+  testutils.GanetiTestProgram()

test/ganeti.utils_unittest.py

@@ -1165,122 +1165,6 @@ class TestTailFile(testutils.GanetiTestCase):
       self.failUnlessEqual(TailFile(fname, lines=i), data[-i:])
 
 
-class _BaseFileLockTest:
-  """Test case for the FileLock class"""
-
-  def testSharedNonblocking(self):
-    self.lock.Shared(blocking=False)
-    self.lock.Close()
-
-  def testExclusiveNonblocking(self):
-    self.lock.Exclusive(blocking=False)
-    self.lock.Close()
-
-  def testUnlockNonblocking(self):
-    self.lock.Unlock(blocking=False)
-    self.lock.Close()
-
-  def testSharedBlocking(self):
-    self.lock.Shared(blocking=True)
-    self.lock.Close()
-
-  def testExclusiveBlocking(self):
-    self.lock.Exclusive(blocking=True)
-    self.lock.Close()
-
-  def testUnlockBlocking(self):
-    self.lock.Unlock(blocking=True)
-    self.lock.Close()
-
-  def testSharedExclusiveUnlock(self):
-    self.lock.Shared(blocking=False)
-    self.lock.Exclusive(blocking=False)
-    self.lock.Unlock(blocking=False)
-    self.lock.Close()
-
-  def testExclusiveSharedUnlock(self):
-    self.lock.Exclusive(blocking=False)
-    self.lock.Shared(blocking=False)
-    self.lock.Unlock(blocking=False)
-    self.lock.Close()
-
-  def testSimpleTimeout(self):
-    # These will succeed on the first attempt, hence a short timeout
-    self.lock.Shared(blocking=True, timeout=10.0)
-    self.lock.Exclusive(blocking=False, timeout=10.0)
-    self.lock.Unlock(blocking=True, timeout=10.0)
-    self.lock.Close()
-
-  @staticmethod
-  def _TryLockInner(filename, shared, blocking):
-    lock = utils.FileLock.Open(filename)
-
-    if shared:
-      fn = lock.Shared
-    else:
-      fn = lock.Exclusive
-
-    try:
-      # The timeout doesn't really matter as the parent process waits for us to
-      # finish anyway.
-      fn(blocking=blocking, timeout=0.01)
-    except errors.LockError, err:
-      return False
-
-    return True
-
-  def _TryLock(self, *args):
-    return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name,
-                                      *args)
-
-  def testTimeout(self):
-    for blocking in [True, False]:
-      self.lock.Exclusive(blocking=True)
-      self.failIf(self._TryLock(False, blocking))
-      self.failIf(self._TryLock(True, blocking))
-
-      self.lock.Shared(blocking=True)
-      self.assert_(self._TryLock(True, blocking))
-      self.failIf(self._TryLock(False, blocking))
-
-  def testCloseShared(self):
-    self.lock.Close()
-    self.assertRaises(AssertionError, self.lock.Shared, blocking=False)
-
-  def testCloseExclusive(self):
-    self.lock.Close()
-    self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False)
-
-  def testCloseUnlock(self):
-    self.lock.Close()
-    self.assertRaises(AssertionError, self.lock.Unlock, blocking=False)
-
-
-class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest):
-  TESTDATA = "Hello World\n" * 10
-
-  def setUp(self):
-    testutils.GanetiTestCase.setUp(self)