Add separate module for backported language functionality

utils.py, where they were before, is already huge.
Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Guido Trotter <ultrotter@google.com>

Makefile.am

@@ -93,6 +93,7 @@ pkgpython_PYTHON = \
 	lib/bootstrap.py \
 	lib/cli.py \
 	lib/cmdlib.py \
+	lib/compat.py \
 	lib/config.py \
 	lib/constants.py \
 	lib/daemon.py \
@@ -321,6 +322,7 @@ python_tests = \
 	test/ganeti.bdev_unittest.py \
 	test/ganeti.cli_unittest.py \
 	test/ganeti.cmdlib_unittest.py \
+	test/ganeti.compat_unittest.py \
 	test/ganeti.confd.client_unittest.py \
 	test/ganeti.config_unittest.py \
 	test/ganeti.constants_unittest.py \

lib/bdev.py

@@ -32,6 +32,7 @@ from ganeti import utils
 from ganeti import errors
 from ganeti import constants
 from ganeti import objects
+from ganeti import compat
 
 
 # Size of reads in _CanReadDevice
@@ -388,7 +389,7 @@ class LogicalVolume(BlockDev):
     pvs_info.reverse()
 
     pvlist = [ pv[1] for pv in pvs_info ]
-    if utils.any(pvlist, lambda v: ":" in v):
+    if compat.any(pvlist, lambda v: ":" in v):
       _ThrowError("Some of your PVs have the invalid character ':' in their"
                   " name, this is not supported - please filter them out"
                   " in lvm.conf using either 'filter' or 'preferred_names'")
@@ -462,7 +463,7 @@ class LogicalVolume(BlockDev):
     """
     if (not cls._VALID_NAME_RE.match(name) or
         name in cls._INVALID_NAMES or
-        utils.any(cls._INVALID_SUBSTRINGS, lambda x: x in name)):
+        compat.any(cls._INVALID_SUBSTRINGS, lambda x: x in name)):
       _ThrowError("Invalid LVM name '%s'", name)
 
   def Remove(self):

lib/cli.py

@@ -37,6 +37,7 @@ from ganeti import luxi
 from ganeti import ssconf
 from ganeti import rpc
 from ganeti import ssh
+from ganeti import compat
 
 from optparse import (OptionParser, TitledHelpFormatter,
                       Option, OptionValueError)
@@ -2116,7 +2117,7 @@ class JobExecutor(object):
         ToStdout("Submitted jobs %s", utils.CommaJoin(ok_jobs))
 
     # first, remove any non-submitted jobs
-    self.jobs, failures = utils.partition(self.jobs, lambda x: x[1])
+    self.jobs, failures = compat.partition(self.jobs, lambda x: x[1])
     for idx, _, jid, name in failures:
       ToStderr("Failed to submit job for %s: %s", name, jid)
       results.append((idx, False, jid))

lib/compat.py

+#
+#
+
+# Copyright (C) 2010 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Module containing backported language/library functionality.
+
+"""
+
+import itertools
+
+try:
+  import functools
+except ImportError:
+  functools = None
+
+
+def all(seq, pred=bool): # pylint: disable-msg=W0622
+  """Returns True if pred(x) is True for every element in the iterable.
+
+  Please note that this function provides a C{pred} parameter which isn't
+  available in the version included in Python 2.5 and above.
+
+  """
+  for _ in itertools.ifilterfalse(pred, seq):
+    return False
+  return True
+
+
+def any(seq, pred=bool): # pylint: disable-msg=W0622
+  """Returns True if pred(x) is True for at least one element in the iterable.
+
+  Please note that this function provides a C{pred} parameter which isn't
+  available in the version included in Python 2.5 and above.
+
+  """
+  for _ in itertools.ifilter(pred, seq):
+    return True
+  return False
+
+
+def partition(seq, pred=bool): # pylint: disable-msg=W0622
+  """Partition a list in two, based on the given predicate.
+
+  """
+  return (list(itertools.ifilter(pred, seq)),
+          list(itertools.ifilterfalse(pred, seq)))
+
+
+# Even though we're using Python's built-in "partial" function if available,
+# this one is always defined for testing.
+def _partial(func, *args, **keywords): # pylint: disable-msg=W0622
+  """Decorator with partial application of arguments and keywords.
+
+  This function was copied from Python's documentation.
+
+  """
+  def newfunc(*fargs, **fkeywords):
+    newkeywords = keywords.copy()
+    newkeywords.update(fkeywords)
+    return func(*(args + fargs), **newkeywords) # pylint: disable-msg=W0142
+
+  newfunc.func = func
+  newfunc.args = args
+  newfunc.keywords = keywords
+  return newfunc
+
+
+if functools is None:
+  partial = _partial
+else:
+  partial = functools.partial

lib/confd/client.py

@@ -61,6 +61,7 @@ from ganeti import daemon # contains AsyncUDPSocket
 from ganeti import errors
 from ganeti import confd
 from ganeti import ssconf
+from ganeti import compat
 
 
 class ConfdAsyncUDPClient(daemon.AsyncUDPSocket):
@@ -581,7 +582,7 @@ class ConfdCountingCallback:
     """Have all the registered queries received at least an answer?
 
     """
-    return utils.all(self._answers.values())
+    return compat.all(self._answers.values())
 
   def _HandleExpire(self, up):
     # if we have no answer we have received none, before the expiration.

lib/locking.py

@@ -33,6 +33,7 @@ import errno
 
 from ganeti import errors
 from ganeti import utils
+from ganeti import compat
 
 
 def ssynchronized(lock, shared=0):
@@ -1216,7 +1217,7 @@ class GanetiLockManager:
     """
     # This way of checking only works if LEVELS[i] = i, which we check for in
     # the test cases.
-    return utils.any((self._is_owned(l) for l in LEVELS[level + 1:]))
+    return compat.any((self._is_owned(l) for l in LEVELS[level + 1:]))
 
   def _BGL_owned(self): # pylint: disable-msg=C0103
     """Check if the current thread owns the BGL.

lib/utils.py

@@ -55,11 +55,6 @@ except ImportError:
   import sha
   sha1 = sha.new
 
-try:
-  import functools
-except ImportError:
-  functools = None
-
 from ganeti import errors
 from ganeti import constants
 
@@ -1492,45 +1487,6 @@ def FirstFree(seq, base=0):
   return None
 
 
-def all(seq, pred=bool): # pylint: disable-msg=W0622
-  "Returns True if pred(x) is True for every element in the iterable"
-  for _ in itertools.ifilterfalse(pred, seq):
-    return False
-  return True
-
-
-def any(seq, pred=bool): # pylint: disable-msg=W0622
-  "Returns True if pred(x) is True for at least one element in the iterable"
-  for _ in itertools.ifilter(pred, seq):
-    return True
-  return False
-
-
-# Even though we're using Python's built-in "partial" function if available,
-# this one is always defined for testing.
-def _partial(func, *args, **keywords): # pylint: disable-msg=W0622
-  """Decorator with partial application of arguments and keywords.
-
-  This function was copied from Python's documentation.
-
-  """
-  def newfunc(*fargs, **fkeywords):
-    newkeywords = keywords.copy()
-    newkeywords.update(fkeywords)
-    return func(*(args + fargs), **newkeywords) # pylint: disable-msg=W0142
-
-  newfunc.func = func
-  newfunc.args = args
-  newfunc.keywords = keywords
-  return newfunc
-
-
-if functools is None:
-  partial = _partial
-else:
-  partial = functools.partial
-
-
 def SingleWaitForFdCondition(fdobj, event, timeout):
   """Waits for a condition to occur on the socket.
 
@@ -1622,12 +1578,6 @@ def WaitForFdCondition(fdobj, event, timeout):
   return result
 
 
-def partition(seq, pred=bool): # # pylint: disable-msg=W0622
-  "Partition a list in two, based on the given predicate"
-  return (list(itertools.ifilter(pred, seq)),
-          list(itertools.ifilterfalse(pred, seq)))
-
-
 def UniqueSequence(seq):
   """Returns a list with unique elements.
 

scripts/gnt-cluster

@@ -40,6 +40,7 @@ from ganeti import bootstrap
 from ganeti import ssh
 from ganeti import objects
 from ganeti import uidpool
+from ganeti import compat
 
 
 @UsesRPC
@@ -423,7 +424,7 @@ def VerifyDisks(opts, args):
 
   if missing:
     for iname, ival in missing.iteritems():
-      all_missing = utils.all(ival, lambda x: x[0] in bad_nodes)
+      all_missing = compat.all(ival, lambda x: x[0] in bad_nodes)
       if all_missing:
         ToStdout("Instance %s cannot be verified as it lives on"
                  " broken nodes", iname)

test/ganeti.compat_unittest.py

+#!/usr/bin/python
+#
+
+# Copyright (C) 2010 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for unittesting the compat module"""
+
+import unittest
+
+from ganeti import compat
+
+import testutils
+
+
+class TestPartial(testutils.GanetiTestCase):
+  def test(self):
+    # Test standard version
+    self._Test(compat.partial)
+
+    # Test our version
+    self._Test(compat._partial)
+
+  def _Test(self, fn):
+    def _TestFunc1(x, power=2):
+      return x ** power
+
+    cubic = fn(_TestFunc1, power=3)
+    self.assertEqual(cubic(1), 1)
+    self.assertEqual(cubic(3), 27)
+    self.assertEqual(cubic(4), 64)
+
+    def _TestFunc2(*args, **kwargs):
+      return (args, kwargs)
+
+    self.assertEqualValues(fn(_TestFunc2, "Hello", "World")("Foo"),
+                           (("Hello", "World", "Foo"), {}))
+
+    self.assertEqualValues(fn(_TestFunc2, "Hello", xyz=123)("Foo"),
+                           (("Hello", "Foo"), {"xyz": 123}))
+
+    self.assertEqualValues(fn(_TestFunc2, xyz=123)("Foo", xyz=999),
+                           (("Foo", ), {"xyz": 999,}))
+
+
+if __name__ == "__main__":
+  testutils.GanetiTestProgram()

test/ganeti.utils_unittest.py

@@ -1592,32 +1592,5 @@ class TestLineSplitter(unittest.TestCase):
                              "", "x"])
 
 
-class TestPartial(testutils.GanetiTestCase):
-  def test(self):
-    self._Test(utils.partial)
-    self._Test(utils._partial)
-
-  def _Test(self, fn):
-    def _TestFunc1(x, power=2):
-      return x ** power
-
-    cubic = fn(_TestFunc1, power=3)
-    self.assertEqual(cubic(1), 1)
-    self.assertEqual(cubic(3), 27)
-    self.assertEqual(cubic(4), 64)
-
-    def _TestFunc2(*args, **kwargs):
-      return (args, kwargs)
-
-    self.assertEqualValues(fn(_TestFunc2, "Hello", "World")("Foo"),
-                           (("Hello", "World", "Foo"), {}))
-
-    self.assertEqualValues(fn(_TestFunc2, "Hello", xyz=123)("Foo"),
-                           (("Hello", "Foo"), {"xyz": 123}))
-
-    self.assertEqualValues(fn(_TestFunc2, xyz=123)("Foo", xyz=999),
-                           (("Foo", ), {"xyz": 999,}))
-
-
 if __name__ == '__main__':
   testutils.GanetiTestProgram()

tools/burnin

@@ -37,6 +37,7 @@ from ganeti import cli
 from ganeti import errors
 from ganeti import utils
 from ganeti import hypervisor
+from ganeti import compat
 
 from ganeti.confd import client as confd_client
 
@@ -982,7 +983,7 @@ class Burner(object):
         self.BurnReplaceDisks2()
 
       if (opts.disk_template in constants.DTS_GROWABLE and
-          utils.any(self.disk_growth, lambda n: n > 0)):
+          compat.any(self.disk_growth, lambda n: n > 0)):
         self.BurnGrowDisks()
 
       if opts.do_failover and opts.disk_template in constants.DTS_NET_MIRROR: